From 82450eb84f96c66818c069c1b287f0633c2f46d5 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:31:02 +0200 Subject: [PATCH 01/49] test: stage optional joined browser filter proof in disposable KVM --- docs/IMPLEMENTATION_STATUS.md | 21 +- tests/integration/browser-filter-bundle.py | 371 ++++++++++++++++++ tests/integration/browser-filter-delivery.py | 320 +++++++++++++++ tests/integration/browser-filter-delivery.sh | 88 +++++ .../content-provider-filter-smoke.sh | 5 + tests/integration/kvm-alpha-topology.sh | 38 +- tests/integration/run-alpha-topology-vm.sh | 68 +++- .../integration/test-browser-filter-bundle.py | 228 +++++++++++ .../test-browser-filter-delivery.py | 171 ++++++++ .../integration/test-browser-filter-runner.py | 51 +++ 10 files changed, 1355 insertions(+), 6 deletions(-) create mode 100644 tests/integration/browser-filter-bundle.py create mode 100644 tests/integration/browser-filter-delivery.py create mode 100644 tests/integration/browser-filter-delivery.sh create mode 100644 tests/integration/test-browser-filter-bundle.py create mode 100644 tests/integration/test-browser-filter-delivery.py create mode 100644 tests/integration/test-browser-filter-runner.py diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 873d068d5..a696f3cc2 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -11,7 +11,7 @@ detailed evidence ledger, including historical failures—not an installation gu [Original v1 scorecard](#fixed-alpha-v1-scorecard) · [Completion requirements](#definition-of-done) -Last updated: 2026-10-07 +Last updated: 2026-10-08 ## Current integration and active work @@ -30,6 +30,10 @@ Browser [PR10](https://github.com/VOLPAROSSA/volparossa-browser/pull/10) adds an actual-upstream-source regression for Consent-O-Matic's navigation-status reset: the original source fails and a one-character in-memory correction passes. The signed extension is unchanged; no upstream contribution has yet been submitted. +The revised contribution approach uses the original extension's voluntary site +reporting, not a maintained fork or automatic submission of browsing data. +Such reports can reveal the reported site's URL; they are not described as +guaranteed anonymous. Bundling, settings and cooperation tests stay in Browser. The new `content filter-snapshot` command reuses the existing protected public-text fetch path with an independently selected publisher, name and exact manifest hash. @@ -105,6 +109,21 @@ same-account trust, source binding and expiry responsibilities; it does not clai extension identity, cross-restart trusted time, publication governance or the unfinished Firefox startup/resume filtering barrier. +The optional combined Browser trial now has a fixed-inventory input transfer +and an explicit hook in the protected-provider fixture. It reuses the real +agent's private cache after the cold peer fetch, then requires actual Gecko +broker retrieval, original signed-uBO asset loading, engine request probes, +revocation and sticky opt-out. Only an explicitly supplied, hash-verified bundle +can enable it. The browser and broker run as the disposable worker in nested +namespaces; neither receives permission to read the agent cache directly. + +The Browser fixture passes 84 focused JavaScript checks and 24 Python checks; +12 core transfer checks and five runner argument checks also pass. These are +preparatory component checks, **not a passed combined runtime trial**. The hook +must still prove real control access after namespace mapping, end-to-end browser +behavior and full cleanup. Automatic startup enrollment, native admission +registration and expiry handling for already loaded rules remain open. + ### Bank and Transaction layer implementation The 2026-10-04 user extension adds VOLPAROSSA Bank and a reusable Transaction-layer. diff --git a/tests/integration/browser-filter-bundle.py b/tests/integration/browser-filter-bundle.py new file mode 100644 index 000000000..d7e8c6ba1 --- /dev/null +++ b/tests/integration/browser-filter-bundle.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Fixed Browser filter-trial inputs; hash, transfer and check without executing them. + +The independently retained INPUTS.json hash is transfer authority, not permission +to run Firefox. Capture neither imports Browser scripts nor rewrites signed XPIs. +Only the separate reviewed guest hook may execute the verified fixture. +""" +import ast +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import sys +import tarfile + +KIND = "volparossa-browser-filter-delivery-inputs" +STAGE = "build/firefox-filter-runtime/" +OUTPUT = "build/filter-delivery" +SCRIPTS = frozenset(("smoke_filter_delivery", "smoke_ubo_actor", "smoke_filter_selection", + "smoke_filter_broker", "smoke_privacy", "smoke_consent", "smoke_browser_startup", + "smoke_compute_model", "stage_firefox", "bundle_extensions", "consent_fixture")) +MODULES = ("Owner", "Journal", "Selection", "Actor", "ActorContract", + "VolparossaFilterSelectionParent", "VolparossaFilterSelectionChild", "Contract", + "Frame", "Clock", "Session", "Gecko", "AssetChannel", "AssetRedirect") +SOURCE_FILES = frozenset({*("scripts/" + name + ".py" for name in SCRIPTS), + *("integration/filters/" + name + ".sys.mjs" for name in MODULES), + "tests/fixtures/filter_delivery.sys.mjs", "defaults/privacy.json", "defaults/extensions.lock.json"}) +# Exact retained Debian ESR stage, including all untouched extension packages/notices. +# A new runtime layout requires review of this allowlist, not recursive adoption. +RUNTIME_FILES = frozenset(( + "application.ini", "browser/chrome/icons/default/default128.png", + "browser/chrome/icons/default/default16.png", "browser/chrome/icons/default/default32.png", + "browser/chrome/icons/default/default48.png", "browser/chrome/icons/default/default64.png", + "browser/defaults/preferences/debugger.js", "browser/defaults/preferences/firefox-branding.js", + "browser/defaults/preferences/firefox.js", "browser/defaults/preferences/vendor.js", + "browser/defaults/syspref/firefox-esr.js", "browser/extensions/langpack-nl@firefox-esr.mozilla.org.xpi", + "browser/omni.ja", "crashhelper", "crashreporter", "defaults/pref/channel-prefs.js", + "defaults/pref/volparossa-autoconfig.js", "dependentlibs.list", "distribution/extensions.lock.json", + "distribution/extensions/ATBC@EasonWong.xpi", "distribution/extensions/gdpr@cavi.au.dk.xpi", + "distribution/extensions/jid1-BoFifL9Vbdl2zQ@jetpack.xpi", + "distribution/extensions/uBlock0@raymondhill.net.xpi", + "distribution/licenses/Adaptive-Tab-Bar-Colour-MIT.txt", "distribution/licenses/Consent-O-Matic-MIT.txt", + "firefox-bin", "firefox-esr", "fonts/TwemojiMozilla.ttf", "glxtest", + "gmp-clearkey/0.1/libclearkey.so", "gmp-clearkey/0.1/manifest.json", + "libfreeblpriv3.so", "libgkcodecs.so", "liblgpllibs.so", "libmozavcodec.so", "libmozavutil.so", + "libmozgtk.so", "libmozsandbox.so", "libmozsqlite3.so", "libmozwayland.so", "libnss3.so", + "libnssutil3.so", "libsmime3.so", "libsoftokn3.so", "libssl3.so", "libxul.so", "omni.ja", + "pingsender", "platform.ini", "vaapitest", "volparossa-staging.json", "volparossa.cfg")) +EXECUTABLES = frozenset(STAGE + name for name in + ("firefox-esr", "firefox-bin", "crashhelper", "crashreporter", "glxtest", "vaapitest", "pingsender")) +FILES = SOURCE_FILES | frozenset(STAGE + name for name in RUNTIME_FILES) +MAX_FILES = 128 +MAX_MANIFEST = 65536 +MAX_SOURCE = 1048576 +MAX_FILE = 256 * 1048576 +MAX_TOTAL = 384 * 1048576 +MAX_ARCHIVE = 385 * 1048576 + + +def require(condition): + if not condition: + raise ValueError("browser_filter_bundle_rejected") + + +def hex64(value): + return type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def mode(name): + return 0o700 if name in EXECUTABLES else 0o600 + + +def bound(name): + return MAX_FILE if name.startswith(STAGE) else MAX_SOURCE + + +def canonical(path, exists=True): + require(path.is_absolute() and path != Path("/") and path.resolve(strict=exists) == path) + + +def identity(info): + return (info.st_dev, info.st_ino, info.st_size, info.st_mode, info.st_uid, info.st_gid, + info.st_nlink, info.st_mtime_ns, info.st_ctime_ns) + + +def safe_info(path, maximum, *, normalized=None): + canonical(path) + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid in (0, os.geteuid()) + and not info.st_mode & 0o7002 and 0 < info.st_size <= maximum) + # The retained workspace stage has 0664 XPI/config files. Accept only the + # invoking owner's own group, then remove group access in the copied tree. + require(not info.st_mode & 0o020 or (info.st_uid == os.geteuid() and info.st_gid == os.getegid())) + if normalized is not None: + require(stat.S_IMODE(info.st_mode) == normalized) + return info + + +def open_checked(path, maximum, *, normalized=None): + info = safe_info(path, maximum, normalized=normalized) + stream = os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") + try: + require(identity(os.fstat(stream.fileno())) == identity(info)) + except BaseException: + stream.close() + raise + return stream, info + + +def fingerprint(path, maximum, *, normalized=None): + source, before = open_checked(path, maximum, normalized=normalized) + with source: + digest = hashlib.file_digest(source, "sha256").hexdigest() + require(identity(os.fstat(source.fileno())) == identity(before)) + require(identity(path.lstat()) == identity(before)) + return dict(bytes=before.st_size, sha256=digest) + + +def unique_pairs(pairs): + result = {} + for key, value in pairs: + require(key not in result) + result[key] = value + return result + + +def encode(value): + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + + +def manifest(raw, expected): + require(hex64(expected) and 0 < len(raw) <= MAX_MANIFEST + and hashlib.sha256(raw).hexdigest() == expected) + value = json.loads(raw, object_pairs_hook=unique_pairs) + require(type(value) is dict and set(value) == {"version", "kind", "files"} + and type(value["version"]) is int and value["version"] == 1 and value["kind"] == KIND + and type(value["files"]) is dict and set(value["files"]) == FILES + and len(value["files"]) <= MAX_FILES and encode(value) == raw) + total = 0 + for name, record in value["files"].items(): + require(type(record) is dict and set(record) == {"bytes", "mode", "sha256"} + and type(record["bytes"]) is int and 0 < record["bytes"] <= bound(name) + and type(record["mode"]) is int and record["mode"] == mode(name) + and hex64(record["sha256"])) + total += record["bytes"] + require(total <= MAX_TOTAL) + return value + + +def directories(files): + return {str(parent) for name in files for parent in Path(name).parents if parent != Path(".")} + + +def inventory_tree(root, files, *, allow_output=False, normalized=False): + canonical(root) + wanted_dirs = directories(files) + pending, seen = [(root, "")], set() + count = 0 + while pending: + directory, relative = pending.pop() + info = directory.lstat() + require(stat.S_ISDIR(info.st_mode) and not stat.S_ISLNK(info.st_mode) + and info.st_uid in (0, os.geteuid()) and not info.st_mode & 0o7002) + if normalized: + require(stat.S_IMODE(info.st_mode) == 0o700) + with os.scandir(directory) as entries: + for entry in entries: + count += 1 + require(count <= MAX_FILES * 4) + name = relative + entry.name + if allow_output and name == OUTPUT: + # Private runtime outputs are outside this input inventory. + require(entry.is_dir(follow_symlinks=False)) + continue + if entry.is_dir(follow_symlinks=False): + require(name in wanted_dirs) + pending.append((Path(entry.path), name + "/")) + else: + require(name in files and entry.is_file(follow_symlinks=False)) + seen.add(name) + require(seen == set(files)) + + +def python_closure(root): + """Parse the fixed reviewed closure; never import or evaluate its contents.""" + edges = {} + for name in sorted(SCRIPTS): + source, info = open_checked(root / "scripts" / (name + ".py"), MAX_SOURCE) + with source: + raw = source.read(MAX_SOURCE + 1) + require(len(raw) == info.st_size and identity(os.fstat(source.fileno())) == identity(info)) + local = set() + for node in ast.walk(ast.parse(raw)): + if isinstance(node, ast.Import): + imports = [alias.name.split(".")[0] for alias in node.names] + elif isinstance(node, ast.ImportFrom): + require(node.level == 0 and node.module is not None) + imports = [node.module.split(".")[0]] + else: + continue + for imported in imports: + require(imported in SCRIPTS or imported in sys.stdlib_module_names) + if imported in SCRIPTS: + local.add(imported) + edges[name] = local + reached, pending = set(), ["smoke_filter_delivery"] + while pending: + name = pending.pop() + if name not in reached: + reached.add(name) + pending.extend(edges[name]) + require(reached == SCRIPTS) + + +def header(name, size, permissions): + entry = tarfile.TarInfo(name) + entry.size, entry.mode = size, permissions + return entry.tobuf(format=tarfile.USTAR_FORMAT, encoding="utf-8", errors="strict") + + +def new_path(path): + canonical(path, exists=False) + canonical(path.parent) + require(not path.exists() and not path.is_symlink()) + + +def capture(root, stage, archive): + canonical(root) + inventory_tree(stage, RUNTIME_FILES) + python_closure(root) + selected = {} + for name in sorted(FILES): + source = stage / name[len(STAGE):] if name.startswith(STAGE) else root / name + selected[name] = dict(**fingerprint(source, bound(name)), mode=mode(name)) + if name in EXECUTABLES: + require(source.stat().st_mode & 0o100) + raw = encode(dict(version=1, kind=KIND, files=selected)) + expected = hashlib.sha256(raw).hexdigest() + manifest(raw, expected) + new_path(archive) + fd = os.open(archive, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as output: + for name in ("INPUTS.json", *sorted(FILES)): + record = selected.get(name, dict(bytes=len(raw), mode=0o600, sha256=expected)) + output.write(header(name, record["bytes"], record["mode"])) + if name == "INPUTS.json": + source, before = io.BytesIO(raw), None + else: + path = stage / name[len(STAGE):] if name.startswith(STAGE) else root / name + source, before = open_checked(path, record["bytes"]) + require(before.st_size == record["bytes"]) + with source: + digest, remaining = hashlib.sha256(), record["bytes"] + while remaining: + chunk = source.read(min(65536, remaining)) + require(bool(chunk)) + output.write(chunk) + digest.update(chunk) + remaining -= len(chunk) + require(not source.read(1) and digest.hexdigest() == record["sha256"]) + if before is not None: + require(identity(os.fstat(source.fileno())) == identity(before)) + output.write(b"\0" * (-record["bytes"] % 512)) + output.write(b"\0" * 1024) + output.flush() + os.fsync(output.fileno()) + return checked_archive(archive, expected) + + +def read_exact(source, length): + raw = source.read(length) + require(len(raw) == length) + return raw + + +def read_header(source, name, record): + require(read_exact(source, 512) == header(name, record["bytes"], record["mode"])) + + +def checked_archive(archive, expected, root=None): + source, before = open_checked(archive, MAX_ARCHIVE) + require(hex64(expected) and before.st_size % 512 == 0) + with source: + first = read_exact(source, 512) + entry = tarfile.TarInfo.frombuf(first, "utf-8", "strict") + require(entry.name == "INPUTS.json" and entry.type == tarfile.REGTYPE + and 0 < entry.size <= MAX_MANIFEST and first == header("INPUTS.json", entry.size, 0o600)) + raw = read_exact(source, entry.size) + require(read_exact(source, -entry.size % 512) == b"\0" * (-entry.size % 512)) + selected = manifest(raw, expected) + if root is not None: + new_path(root) + root.mkdir(mode=0o700) + for name, record in sorted(selected["files"].items()): + read_header(source, name, record) + target = None + try: + if root is not None: + path = root / name + directory = root + for part in Path(name).parts[:-1]: + directory /= part + directory.mkdir(mode=0o700, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + target = os.fdopen(fd, "wb") + digest, remaining = hashlib.sha256(), record["bytes"] + while remaining: + chunk = read_exact(source, min(65536, remaining)) + digest.update(chunk) + if target is not None: + target.write(chunk) + remaining -= len(chunk) + require(digest.hexdigest() == record["sha256"]) + require(read_exact(source, -record["bytes"] % 512) == b"\0" * (-record["bytes"] % 512)) + if target is not None: + target.flush() + os.fchmod(target.fileno(), record["mode"]) + finally: + if target is not None: + target.close() + # No hidden PAX/GNU entries, appended tar, nonzero padding or trailing payload. + require(read_exact(source, 1024) == b"\0" * 1024 and not source.read(1)) + require(identity(os.fstat(source.fileno())) == identity(before)) + source.seek(0) + archive_hash = hashlib.file_digest(source, "sha256").hexdigest() + require(identity(os.fstat(source.fileno())) == identity(before)) + if root is not None: + # A failed/partial unpack never carries the accepted manifest marker. + fd = os.open(root / "INPUTS.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as output: + output.write(raw) + return dict(manifest_sha256=expected, archive_sha256=archive_hash, + files=len(selected["files"]), bytes=sum(item["bytes"] for item in selected["files"].values())) + + +def verify_tree(root, expected): + source, before = open_checked(root / "INPUTS.json", MAX_MANIFEST, normalized=0o600) + with source: + raw = source.read(MAX_MANIFEST + 1) + require(identity(os.fstat(source.fileno())) == identity(before)) + selected = manifest(raw, expected) + inventory_tree(root, FILES | {"INPUTS.json"}, allow_output=True, normalized=True) + for name, record in selected["files"].items(): + actual = fingerprint(root / name, record["bytes"], normalized=record["mode"]) + require(actual == {key: record[key] for key in ("bytes", "sha256")}) + return dict(manifest_sha256=expected, files=len(FILES), private_output_inspected=False) + + +def main(): + require(len(sys.argv) >= 2) + action = sys.argv[1] + if action in ("capture", "unpack"): + require(len(sys.argv) == 5) + first, second = map(Path, sys.argv[2:4]) + result = capture(first, second, Path(sys.argv[4])) if action == "capture" else checked_archive(first, sys.argv[4], second) + else: + require(action in ("verify", "verify-tree") and len(sys.argv) == 4) + result = (checked_archive if action == "verify" else verify_tree)(Path(sys.argv[2]), sys.argv[3]) + print(json.dumps(dict(version=1, action=action, transferred=True, runtime_executed=False, **result), sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, TypeError, SyntaxError, tarfile.TarError, UnicodeError): + print('{"transferred":false,"failure":"browser_filter_bundle_rejected","runtime_executed":false}') + raise SystemExit(1) diff --git a/tests/integration/browser-filter-delivery.py b/tests/integration/browser-filter-delivery.py new file mode 100644 index 000000000..232f925bd --- /dev/null +++ b/tests/integration/browser-filter-delivery.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Closed joined-proof checker/launcher, only for the disposable provider fixture. + +No cache seeding, agent mutation, replacement broker reply or model execution. +The enclosing topology retains ownership of source/runtime/output cleanup. +""" +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import stat +import sys +import time + +HERE = Path(__file__).parent +FILTER = runpy.run_path(str(HERE / "content-provider-filter-smoke.py")) +BUNDLE = runpy.run_path(str(HERE / "browser-filter-bundle.py")) +TEXT, SHA, NAME = FILTER["TEXT"], FILTER["SHA"], FILTER["NAME"] +OPERATIONS = ("open", "fetch", "enroll", "suspend", "optOut", "retry", "close") +CASES = ("baseline", "enrolled", "suspended", "opted_out", "retry") +SCOPE = dict(real_core_broker=True, real_gecko_unix_ipc=True, positive_verified_publication=True, + original_signed_ubo=True, actual_background_asset_request=True, actual_engine_probes=True, + existing_agent_required=True, browser_sessions=1, protected_peer_fetch_proven=False, + cold_cache_proven=False, native_admission_registered=False, production_default_enrollment=False, + startup_resume_stale_filter_barrier=False, real_os_expiry=False) +MAX_REPORT = 65536 +REPORT_KEYS = {"schema", "passed", "scope", "operation", "receipts", "probes", "module_sha256", "driver_sha256", + "broker_sha256", "expected", "runtime_version", "runtime_source_stamp", "runtime_sha256", "extension_packages", + "failure", "host_read_only", "interfaces", "browser_exit_status", "broker_cleanup", "broker_socket_removed", + "temporary_profiles_removed", "logs", "failure_phase"} + + +def require(condition): + if not condition: raise ValueError("browser_filter_delivery_rejected") + + +def same(value, expected): + require(type(value) is type(expected)) + if type(expected) is dict: + require(set(value) == set(expected)) + for key in expected: same(value[key], expected[key]) + elif type(expected) is list: + require(len(value) == len(expected)) + for item, other in zip(value, expected): same(item, other) + else: require(value == expected) + + +def hex64(value): + return type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def raw_file(path, limit=MAX_REPORT): + require(path.resolve(strict=True) == path) + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and not info.st_mode & 0o022 + and 0 <= info.st_size <= limit) + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as stream: + opened = os.fstat(stream.fileno()) + require((opened.st_dev, opened.st_ino, opened.st_size) == (info.st_dev, info.st_ino, info.st_size)) + raw = stream.read(limit + 1) + require(len(raw) == info.st_size) + return raw + + +def read(path): + return json.loads(raw_file(path), object_pairs_hook=BUNDLE["unique_pairs"]) + + +def write_new(path, raw): + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o400), "wb") as stream: + stream.write(raw); stream.flush(); os.fsync(stream.fileno()) + + +def private_root(root): + require(root.name == "browser-filter-delivery" and root.resolve(strict=True) == root) + info = root.lstat() + require(stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700 + and info.st_uid == os.getuid() and info.st_gid == os.getgid()) + + +def selection(value): + require(type(value) is dict and set(value) == {"name", "content_type", "bytes", "sha256", "rules", "publisher_key", "manifest_id"}) + same({key: value[key] for key in ("name", "content_type", "bytes", "sha256", "rules")}, + dict(name=NAME, content_type="text/plain", bytes=len(TEXT), sha256=SHA, rules=2)) + require(all(hex64(value[key]) and value[key] != "0" * 64 for key in ("publisher_key", "manifest_id"))) + return value + + +def prepare(root, publication_expiry): + private_root(root) + require({path.name for path in root.iterdir()} == {"selection.json", "home"}) + require(not list((root / "home").iterdir())) + chosen = selection(read(root / "selection.json")) + now = int(time.time()) + require(type(publication_expiry) is int and now < publication_expiry <= 2**53 - 1) + expected = dict(publisher_key=chosen["publisher_key"], name=chosen["name"], manifest_id=chosen["manifest_id"], + authority_expires_unix_seconds=min(publication_expiry, now + 600)) + write_new(root / "expected.json", json.dumps(expected, sort_keys=True, separators=(",", ":")).encode()) + return dict(version=1, expected=expected, created_unix_seconds=now, maximum_authority_seconds=600) + + +def cleanup(root): + require(root.name == "browser-filter-delivery" and not root.is_symlink()) + if not root.exists(): return dict(directory_removed=True, files_removed=True) + private_root(root) + require({path.name for path in root.iterdir()} <= {"selection.json", "expected.json", "home"}) + files = [] + for path in root.iterdir(): + info = path.lstat() + require(info.st_uid == os.getuid() and info.st_gid == os.getgid()) + if path.name == "home": + require(stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700 and not list(path.iterdir())) + else: + require(stat.S_ISREG(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o400 + and info.st_nlink == 1 and info.st_size <= 1024) + files.append(path) + for path in files: path.unlink() + if (root / "home").exists(): (root / "home").rmdir() + root.rmdir() + return dict(directory_removed=True, files_removed=True) + + +def process_boundary(parent, client, uid, control): + status = dict(line.split(":", 1) for line in Path("/proc/self/status").read_text().splitlines()) + require(uid == 985 and os.readlink("/proc/self/ns/net") == client != parent + and all(int(value) == uid for value in status["Uid"].split()) + and all(int(value) == control for value in status["Gid"].split()) + and status["Groups"].strip() == "" + and all(int(status[key], 16) == 0 for key in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")) + and int(status["NoNewPrivs"]) == 1) + return dict(user_uid=uid, primary_control_gid=control, supplementary_groups_empty=True, + client_namespace=True, outside_parent_namespace=True, all_capabilities_dropped=True, no_new_privileges=True) + + +def launch(arguments): + directory, input_directory, binary, binary_sha, control_socket, cache, parent, client, uid, group = arguments + source, root = Path(directory), Path(input_directory) + private_root(root) + require(source == root.parent / "browser-filter-source" and source.resolve(strict=True) == source + and Path.home() == root / "home" and hex64(binary_sha)) + boundary = process_boundary(parent, client, int(uid), int(group)) + print(json.dumps(dict(version=1, boundary=boundary), sort_keys=True), flush=True) + # Replace this inspected process; no changed credential or hidden wrapper + # exists between the observation and the reviewed Browser driver. + os.execv(sys.executable, [sys.executable, "-B", str(source / "scripts/smoke_filter_delivery.py"), + "--execute", "--stage", str(source / "build/firefox-filter-runtime"), + "--output", str(source / "build/filter-delivery"), "--broker", binary, "--broker-sha256", binary_sha, + "--control-socket", control_socket, "--agent-cache", cache, "--expected", str(root / "expected.json")]) + + +def cache_state(path, uid, gid): + require(path.name == "filter-cache" and path.parent.name == "state-client" and path.resolve(strict=True) == path) + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700 + and info.st_uid == uid and info.st_gid == gid) + return dict(device=info.st_dev, inode=info.st_ino, mode="0700", user_uid=uid, group_gid=gid) + + +def operation_receipt(operation, expected): + choice = "opted-out" if operation in ("optOut", "retry", "close") else "eligible" + state = "fresh" if operation in ("open", "fetch") else "active" if operation == "enroll" else "suspended" + result = None + if operation in ("enroll", "suspend", "optOut", "retry"): + result = dict(schema=1, choice=choice, state=state, attempted=operation != "retry", receipt=None) + if operation != "retry": + result["receipt"] = dict(selected=operation == "enroll", imported=operation == "enroll", + preserved=True, freshReload=True, reloadEvents=2) + return dict(schema=1, ok=True, phase=operation, status=dict(schema=1, choice=choice, state=state, + closed=operation == "close", failed=False), result=result, + delivery=None if operation == "open" else dict(bytes=len(TEXT), sha256=SHA, rules=2, manifest_id=expected["manifest_id"]), + asset=None if operation in ("open", "fetch") else dict(closed=True, accepted=1, completed=1, denied=0), + broker_revoked=operation in ("suspend", "optOut", "retry", "close")) + + +def validate_browser(report, outer, expected, manifest, stage, binary_sha): + fixed = dict(schema=1, passed=True, scope=SCOPE, operation="complete", failure=None, + expected=expected, broker_sha256=binary_sha, host_read_only=True, interfaces=["lo"], browser_exit_status=0, + broker_cleanup=dict(started=True, reaped=True, graceful=True), broker_socket_removed=True, + temporary_profiles_removed=True, + receipts={operation: operation_receipt(operation, expected) for operation in OPERATIONS}, + probes={case: {kind: 0 if case == "enrolled" and kind != "neutral" else 1 for kind in ("ads", "tracker", "neutral")} + for case in CASES}) + files = manifest["files"] + fixed.update(driver_sha256=files["scripts/smoke_filter_delivery.py"]["sha256"], + module_sha256={name: record["sha256"] for name, record in files.items() + if name.startswith("integration/filters/") or name == "tests/fixtures/filter_delivery.sys.mjs"}, + runtime_version=stage["version"], runtime_source_stamp=stage["source_stamp"], + runtime_sha256=stage["runtime_sha256"], extension_packages=stage["extensions"]["packages"]) + require(type(report) is dict and set(report) == set(fixed) | {"logs"}) + for key, value in fixed.items(): same(report[key], value) + logs = report["logs"] + require(type(logs) is dict and set(logs) == {"browser", "broker"}) + for value in logs.values(): + require(type(value) is dict and set(value) == {"bytes", "sha256"} and type(value["bytes"]) is int + and 0 <= value["bytes"] <= 4 * 1024 * 1024 and hex64(value["sha256"])) + same(outer, dict(schema=1, temporary_profiles_removed=True, broker_copy_removed=True, logs=logs)) + + +def validate_cold(evidence): + selection(evidence["input"]) + FILTER["validate_application"](evidence, evidence["cold"]["application"], False) + FILTER["validate_packets"](evidence, evidence["cold"], False) + route, peers, layout = evidence["selected_route"], evidence["expected_peers"], evidence["layout"] + paths, slots = route["paths"], route["benchmark_slots"] + require(route["transport"] == "mptcp" and len(paths) == len(slots) == 2 + and re.fullmatch(r"[0-9a-f]{32}", route["route_context_id"]) + and len({path["relay_peer_id"] for path in paths}) == 2 + and {path["exit_peer_id"] for path in paths} == {peers["exit"]} + and all(path["route_context_id"] == route["route_context_id"] for path in paths) + and [slot["relay_peer_id"] for slot in slots] == [path["relay_peer_id"] for path in paths] + and all(slot["relay_node"] in FILTER["ROLES"][1:4] and peers[slot["relay_node"]] == slot["relay_peer_id"] for slot in slots) + and {peers[node] for node in layout["provider_nodes"]}.isdisjoint( + {peers["client"], peers["exit"], layout["control_relay_peer_id"]} | {path["relay_peer_id"] for path in paths}) + and any(evidence["cold"]["privacy"][slot["relay_node"]]["client_leg_wireguard_data_datagrams"] > 0 + and evidence["cold"]["privacy"][slot["relay_node"]]["exit_leg_wireguard_data_datagrams"] > 0 for slot in slots)) + same(evidence["publisher_cleanup"], dict(directory_removed=True, files_removed=True)) + + +def retain(work): + """Preserve original closed receipts on failure too; absence is not success.""" + output = work / "client-fixtures/browser-filter-source/build/filter-delivery" + retained = {} + for name, keys in (("report", REPORT_KEYS), ("outer-cleanup", {"schema", "temporary_profiles_removed", "broker_copy_removed", "logs"})): + path = output / (name + ".json") + if not path.exists() and not path.is_symlink(): continue + raw = raw_file(path) + value = json.loads(raw, object_pairs_hook=BUNDLE["unique_pairs"]) + require(type(value) is dict and set(value) <= keys and type(value.get("schema")) is int and value["schema"] == 1) + target = work / ("browser-filter-delivery-original-" + name + ".json") + if target.exists(): require(raw_file(target) == raw) + else: write_new(target, raw) + retained[name] = hashlib.sha256(raw).hexdigest() + return dict(version=1, originals_sha256=retained, successful_native_execution_claimed=False) + + +def evidence(work, manifest_sha, binary_sha, uid, original_gid, control_gid): + source = work / "client-fixtures/browser-filter-source" + output = source / "build/filter-delivery" + manifest_raw = raw_file(source / "INPUTS.json") + manifest = BUNDLE["manifest"](manifest_raw, manifest_sha) + stage_path = source / "build/firefox-filter-runtime/volparossa-staging.json" + require(hashlib.sha256(raw_file(stage_path)).hexdigest() == manifest["files"]["build/firefox-filter-runtime/volparossa-staging.json"]["sha256"]) + stage = read(stage_path) + prepared = read(work / "browser-filter-delivery-prepared.json") + require(type(prepared) is dict and set(prepared) == {"version", "expected", "created_unix_seconds", "maximum_authority_seconds"}) + require(type(prepared["created_unix_seconds"]) is int and prepared["created_unix_seconds"] > 0) + same(prepared["version"], 1); same(prepared["maximum_authority_seconds"], 600) + selected = selection(read(work / "content-provider-filter-input.json")) + publish = read(work / "content-provider-filter-publish.json") + expected = dict(publisher_key=selected["publisher_key"], name=NAME, manifest_id=selected["manifest_id"], + authority_expires_unix_seconds=min(publish["expires_unix_seconds"], prepared["created_unix_seconds"] + 600)) + require(expected["authority_expires_unix_seconds"] > prepared["created_unix_seconds"]) + same(prepared["expected"], expected) + report, outer = read(output / "report.json"), read(output / "outer-cleanup.json") + validate_browser(report, outer, expected, manifest, stage, binary_sha) + same(read(work / "browser-filter-delivery-launch.json"), dict(version=1, boundary=dict(user_uid=uid, + primary_control_gid=control_gid, supplementary_groups_empty=True, client_namespace=True, + outside_parent_namespace=True, all_capabilities_dropped=True, no_new_privileges=True))) + for phase in ("before", "after"): + same(read(work / f"browser-filter-delivery-tree-{phase}.json"), dict(version=1, action="verify-tree", + transferred=True, runtime_executed=False, manifest_sha256=manifest_sha, + files=len(manifest["files"]), private_output_inspected=False)) + before, after = (read(work / f"browser-filter-delivery-cache-{phase}.json") for phase in ("before", "after")) + same(before, after) + same(before, cache_state(work / "state-client/filter-cache", before["user_uid"], before["group_gid"])) + require(uid == 985 and uid != before["user_uid"] and control_gid != before["group_gid"]) + same(read(work / "browser-filter-delivery-input-cleanup.json"), dict(directory_removed=True, files_removed=True)) + require(not (work / "client-fixtures/browser-filter-delivery").exists()) + for name in ("profile", "config", "cache", "runtime", "tmp", "appdata", "broker-binary"): + require(not (output / name).exists() and not (output / name).is_symlink()) + for name, value in report["logs"].items(): + raw = raw_file(output / (name + ".log"), 4 * 1024 * 1024) + same(value, dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest())) + cold = dict(input=selected, publish=publish, layout=read(work / "content-provider-layout.json"), + expected_peers=read(work / "a01-expected-peers.json"), + publisher_cleanup=read(work / "content-provider-filter-publisher-cleanup.json"), + selected_route=read(work / "content-provider-filter-selection.json"), + isolation=dict(agent_cache=str(work / "state-client/filter-cache"), user_uid=uid, user_gid=original_gid, control_gid=control_gid), + cold=dict(application=read(work / "content-provider-filter-cold-consumer.json"), + control_privacy=read(work / "content-provider-filter-cold-control.json"), + privacy={role: read(work / f"content-provider-filter-cold-privacy-{role}.json") for role in FILTER["ROLES"]})) + validate_cold(cold) + originals = {} + for name in ("report", "outer-cleanup"): + raw = raw_file(output / (name + ".json")) + require(raw_file(work / ("browser-filter-delivery-original-" + name + ".json")) == raw) + originals[name] = hashlib.sha256(raw).hexdigest() + return dict(version=1, kind="volparossa-browser-filter-delivery", success=True, + bundle_manifest_sha256=manifest_sha, core_broker_sha256=binary_sha, originals_sha256=originals, + publication=selected, protected_cold=dict(peer_bytes=cold["cold"]["application"]["receipt"]["peer_bytes"], + providers_used=cold["cold"]["application"]["receipt"]["providers_used"], privacy_captures_validated=True), + browser=dict(receipts=report["receipts"], probes=report["probes"], exit_status=0), + scope=dict(same_publication_joined=True, broker_reuse_requested=True, broker_cache_only_proven=False, + new_broker_peer_transfer_proven=False, native_admission_registered=False, production_default_enrollment=False, + startup_resume_stale_filter_barrier=False, two_subflow_bulk_aggregation=False), + cleanup=dict(browser_profile_removed=True, broker_reaped=True, broker_socket_removed=True, + fixture_authority_removed=True, separate_owner_input_removed=True, agent_cache_identity_preserved=True, + agent_cache_private=True, staged_inputs_removed_by_enclosing_topology=False)) + + +def main(): + action, *args = sys.argv[1:] + if action == "prepare" and len(args) == 2: result = prepare(Path(args[0]), int(args[1])) + elif action == "cleanup" and len(args) == 1: result = cleanup(Path(args[0])) + elif action == "cache-state" and len(args) == 3: result = cache_state(Path(args[0]), int(args[1]), int(args[2])) + elif action == "launch" and len(args) == 10: return launch(args) + elif action == "retain" and len(args) == 1: result = retain(Path(args[0])) + elif action == "evidence" and len(args) == 6: + result = evidence(Path(args[0]), args[1], args[2], *map(int, args[3:])) + else: raise ValueError("browser_filter_delivery_rejected") + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + + +if __name__ == "__main__": + try: main() + except (Exception, KeyboardInterrupt): + print("browser_filter_delivery_rejected", file=sys.stderr); raise SystemExit(1) from None diff --git a/tests/integration/browser-filter-delivery.sh b/tests/integration/browser-filter-delivery.sh new file mode 100644 index 000000000..c02d3c583 --- /dev/null +++ b/tests/integration/browser-filter-delivery.sh @@ -0,0 +1,88 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only +# Explicit optional continuation of the ORIGINAL protected cold filter fetch. +# All new paths are below the disposable topology's exact WORK directory. +# shellcheck disable=SC2154,SC2034 + +browser_filter_private() { + setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/browser-filter-delivery.py" "$@" +} + +browser_filter_delivery_cleanup() { + [ -f "$WORK/bin/browser-filter-delivery.py" ] || return 0 + browser_filter_private cleanup "$WORK/client-fixtures/browser-filter-delivery" \ + >"$WORK/browser-filter-delivery-input-cleanup.json" + # The driver removes its profiles and owned broker; the enclosing topology + # owns final removal of the staged source/runtime/report tree under WORK. +} + +browser_filter_delivery_run() { + if [ -z "${browser_filter_bundle:-}" ] || [ -z "${browser_filter_manifest_sha256:-}" ]; then + fail BROWSER_FILTER_INPUTS_MISSING + fi + PHASE=browser-filter-delivery-prepare + printf '%s\n' 'Browser filter trial: stage verified inputs only under the disposable WORK; reuse the actual private agent cache; run capless in the Client namespace; remove only the fixture grant/profile.' + browser_filter_directory=$WORK/client-fixtures/browser-filter-source + browser_filter_input=$WORK/client-fixtures/browser-filter-delivery + for browser_filter_new in "$browser_filter_directory" "$browser_filter_input"; do + if [ -e "$browser_filter_new" ] || [ -L "$browser_filter_new" ]; then fail BROWSER_FILTER_PATH_EXISTS; fi + done + install -o root -g root -m 0555 "$source_directory/tests/integration/browser-filter-delivery.py" \ + "$WORK/bin/browser-filter-delivery.py" || fail BROWSER_FILTER_DRIVER_UNAVAILABLE + install -o root -g root -m 0555 "$source_directory/tests/integration/browser-filter-bundle.py" \ + "$WORK/bin/browser-filter-bundle.py" || fail BROWSER_FILTER_BUNDLE_DRIVER_UNAVAILABLE + install -o root -g root -m 0600 "$browser_filter_bundle" "$WORK/browser-filter-inputs.tar" \ + || fail BROWSER_FILTER_ARCHIVE_COPY_FAILED + python3 -B "$WORK/bin/browser-filter-bundle.py" unpack \ + "$WORK/browser-filter-inputs.tar" "$browser_filter_directory" "$browser_filter_manifest_sha256" \ + >"$WORK/browser-filter-delivery-unpack.json" || fail BROWSER_FILTER_UNPACK_FAILED + # Unpack has already rejected symlinks, hardlinks, unknown paths and modes. + # This is the fresh, validated input tree only, never existing agent state. + chown -R "$WORKER_UID:$site_control_gid" "$browser_filter_directory" || fail BROWSER_FILTER_OWNER_FAILED + install -d -o "$WORKER_UID" -g "$site_control_gid" -m 0700 "$browser_filter_input" "$browser_filter_input/home" + install -o "$WORKER_UID" -g "$site_control_gid" -m 0400 "$WORK/content-provider-filter-input.json" \ + "$browser_filter_input/selection.json" || fail BROWSER_FILTER_SELECTION_UNAVAILABLE + browser_filter_expiry=$(jq -er '.expires_unix_seconds | select(type == "number" and . > 0 and . == floor)' \ + "$WORK/content-provider-filter-publish.json") || fail BROWSER_FILTER_EXPIRY_INVALID + browser_filter_private prepare "$browser_filter_input" "$browser_filter_expiry" \ + >"$WORK/browser-filter-delivery-prepared.json" || fail BROWSER_FILTER_AUTHORITY_FAILED + browser_filter_broker_sha256=$(sha256sum "$binary_directory/volparossa" | awk '{print $1}') + python3 -B "$WORK/bin/browser-filter-delivery.py" cache-state "$filter_cache" "$AGENT_UID" "$AGENT_GID" \ + >"$WORK/browser-filter-delivery-cache-before.json" || fail BROWSER_FILTER_CACHE_INVALID + setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/browser-filter-bundle.py" verify-tree \ + "$browser_filter_directory" "$browser_filter_manifest_sha256" \ + >"$WORK/browser-filter-delivery-tree-before.json" || fail BROWSER_FILTER_SOURCE_CHANGED + PHASE=browser-filter-delivery-native + browser_filter_status=0 + timeout --signal=TERM --kill-after=10s 500s ip netns exec "$CLIENT" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- env HOME="$browser_filter_input/home" python3 -B "$WORK/bin/browser-filter-delivery.py" launch \ + "$browser_filter_directory" "$browser_filter_input" "$binary_directory/volparossa" \ + "$browser_filter_broker_sha256" "$WORK/runtime-client/control/agent.sock" "$filter_cache" \ + "$site_parent_ns" "$site_client_ns" "$WORKER_UID" "$site_control_gid" \ + >"$WORK/browser-filter-delivery-launch.json" 2>"$WORK/browser-filter-delivery.err" \ + || browser_filter_status=$? + PHASE=browser-filter-delivery-verify + python3 -B "$WORK/bin/browser-filter-delivery.py" retain "$WORK" \ + >"$WORK/browser-filter-delivery-retained.json" || fail BROWSER_FILTER_RECEIPT_RETENTION_FAILED + browser_filter_delivery_cleanup || fail BROWSER_FILTER_INPUT_CLEANUP_FAILED + [ "$browser_filter_status" -eq 0 ] || fail BROWSER_FILTER_NATIVE_FAILED + setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/browser-filter-bundle.py" verify-tree \ + "$browser_filter_directory" "$browser_filter_manifest_sha256" \ + >"$WORK/browser-filter-delivery-tree-after.json" || fail BROWSER_FILTER_SOURCE_CHANGED + python3 -B "$WORK/bin/browser-filter-delivery.py" cache-state "$filter_cache" "$AGENT_UID" "$AGENT_GID" \ + >"$WORK/browser-filter-delivery-cache-after.json" || fail BROWSER_FILTER_CACHE_INVALID + if setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$filter_cache"; then fail BROWSER_FILTER_AGENT_CACHE_EXPOSED; fi + python3 -B "$WORK/bin/browser-filter-delivery.py" evidence "$WORK" "$browser_filter_manifest_sha256" \ + "$browser_filter_broker_sha256" "$WORKER_UID" "$WORKER_GID" "$site_control_gid" \ + >"$WORK/browser-filter-delivery-evidence.json" || fail BROWSER_FILTER_EVIDENCE_INVALID +} diff --git a/tests/integration/content-provider-filter-smoke.sh b/tests/integration/content-provider-filter-smoke.sh index dd5a93589..597e134ec 100644 --- a/tests/integration/content-provider-filter-smoke.sh +++ b/tests/integration/content-provider-filter-smoke.sh @@ -120,6 +120,11 @@ content_provider_filter_cold_run() { >"$WORK/content-provider-filter-publisher-cleanup.json" || fail CONTENT_FILTER_PUBLISHER_CLEANUP_FAILED [ ! -e "$filter_publisher" ] || fail CONTENT_FILTER_PUBLISHER_FILES_REMAIN content_provider_filter_consume cold + if [ -n "${browser_filter_bundle:-}" ]; then + # shellcheck source=tests/integration/browser-filter-delivery.sh + . "$source_directory/tests/integration/browser-filter-delivery.sh" + browser_filter_delivery_run + fi filter_cache_before=$(stat -Lc '%d:%i' "$filter_cache") || fail CONTENT_FILTER_CACHE_IDENTITY_UNAVAILABLE } diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index ab15e9c2a..f6cd4ada9 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -58,6 +58,9 @@ output_directory= expected_commit= code_bundle= code_manifest_sha256= +browser_filter_bundle= +browser_filter_manifest_sha256= +browser_filter_directory= # Only explicit scenario arguments may authorize the staged Code inputs. COOPERATIVE_CODE_BUNDLE= COOPERATIVE_CODE_MANIFEST_SHA256= @@ -70,6 +73,7 @@ usage() { ' --source DIRECTORY --bin DIRECTORY --output DIRECTORY' \ ' --scenario signal-backup also runs the exact pinned native Signal backup test' \ ' --scenario agent-cooperative-code requires --code-bundle DIRECTORY --code-manifest-sha256 HEX' \ + ' --scenario content-provider accepts --browser-filter-bundle ARCHIVE --browser-filter-manifest-sha256 HEX' \ ' --mpquic PATH --expected-commit SHA [--scenario alpha|reciprocity|reciprocity-private-dns|local-link|mixed-link|mpquic-growth|mptcp-growth|mptcp-refill|sharing|download-sharing|wifi-link|uplink-link|crash-recovery|content|content-message|content-https|content-provider|content-replication|content-repair|content-mailbox|content-custody|private-storage-peer|private-storage-replicas|signal-backup|private-storage-handoff|private-storage-fragments|agent-artifact|agent-train-cycle|agent-train-loop|agent-artifact-quarantine|agent-jobs|agent-jobs-loss|agent-jobs-follow|agent-jobs-peer-recovery|agent-jobs-ready-queue|agent-jobs-package-queue|agent-public-task|agent-public-document|agent-public-collection|agent-public-network-sources|agent-task-graph|agent-ready-dag|agent-model-planning|agent-model-task-graph|agent-successor-serving|agent-active-recovery|agent-adapter-aggregation|agent-autonomous-aggregation|agent-policy-assessment|dns-cache]' } @@ -865,6 +869,16 @@ while [ "$#" -gt 0 ]; do [ -n "$code_bundle" ] || { usage >&2; exit 64; } shift ;; + --browser-filter-bundle) + if [ "$#" -lt 2 ] || [ -n "$browser_filter_bundle" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + browser_filter_bundle=$2 + shift + ;; + --browser-filter-manifest-sha256) + if [ "$#" -lt 2 ] || [ -n "$browser_filter_manifest_sha256" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + browser_filter_manifest_sha256=$2 + shift + ;; --code-manifest-sha256) if [ "$#" -lt 2 ] || [ -n "$code_manifest_sha256" ]; then usage >&2; exit 64; fi code_manifest_sha256=$2 @@ -885,7 +899,7 @@ done if [ "$mode" = preview ]; then if [ "$approval" != no ] \ - || [ -n "$source_directory$binary_directory$mpquic_binary$output_directory$expected_commit$code_bundle$code_manifest_sha256" ]; then + || [ -n "$source_directory$binary_directory$mpquic_binary$output_directory$expected_commit$code_bundle$code_manifest_sha256$browser_filter_bundle$browser_filter_manifest_sha256" ]; then usage >&2 exit 64 fi @@ -922,6 +936,13 @@ elif [ -n "$code_bundle$code_manifest_sha256" ]; then exit 64 fi +if [ -n "$browser_filter_bundle$browser_filter_manifest_sha256" ]; then + [ "$scenario" = content-provider ] || exit 64 + case $browser_filter_bundle in /*) ;; *) exit 64 ;; esac + case $browser_filter_bundle in /|*/|*//*|*/../*|*/./*|*/..|*/.|*[[:cntrl:]]*) exit 64 ;; esac + case $browser_filter_manifest_sha256 in ''|*[!0-9a-f]*) exit 64 ;; esac + [ "${#browser_filter_manifest_sha256}" -eq 64 ] || exit 64 +fi [ "$(id -u)" -eq 0 ] || { printf '%s\n' 'execution requires root inside KVM' >&2; exit 77; } [ "$(sed -n '1{s/\..*$//;p;}' /etc/debian_version)" = 13 ] \ || { printf '%s\n' 'execution requires Debian 13' >&2; exit 77; } @@ -939,6 +960,21 @@ for command_name in awk busctl cat chmod chown cut date find getent grep install command -v "$command_name" >/dev/null 2>&1 \ || { printf 'required guest tool unavailable: %s\n' "$command_name" >&2; exit 69; } done +if [ -n "$browser_filter_bundle" ]; then + [ -f "$browser_filter_bundle" ] && [ ! -L "$browser_filter_bundle" ] \ + && [ "$(readlink -e -- "$browser_filter_bundle")" = "$browser_filter_bundle" ] \ + && [ "$(stat -Lc '%a' "$browser_filter_bundle")" = 600 ] || exit 69 + browser_filter_input_uid=$(stat -Lc '%u' "$browser_filter_bundle") + if [ "$browser_filter_input_uid" -eq 0 ]; then + python3 -B "$source_directory/tests/integration/browser-filter-bundle.py" verify \ + "$browser_filter_bundle" "$browser_filter_manifest_sha256" || exit 69 + elif [ "$browser_filter_input_uid" -eq "$(id -u vpci)" ]; then + runuser -u vpci -- python3 -B "$source_directory/tests/integration/browser-filter-bundle.py" verify \ + "$browser_filter_bundle" "$browser_filter_manifest_sha256" || exit 69 + else + exit 69 + fi +fi if [ "$agent_cooperative_code" = yes ]; then if [ -d "$code_bundle" ] && [ ! -L "$code_bundle" ] \ && [ "$(readlink -e -- "$code_bundle")" = "$code_bundle" ] \ diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 25c2b50d8..6ab193055 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -21,6 +21,8 @@ expected_commit= scenario=alpha code_bundle= code_manifest_sha256= +browser_filter_bundle= +browser_filter_manifest_sha256= guest_memory_for_scenario() { # These fixtures co-locate two 360M providers in one guest. Production @@ -42,7 +44,8 @@ usage() { ' --package is required only for alpha; --mpquic is unnecessary for standalone Wi-Fi/model/private-browser/transaction-TEST proofs.' \ ' agent-cooperative-code also requires --code-bundle DIRECTORY --code-manifest-sha256 SHA256.' \ ' --native-runtime PATH is required for agent-native-coding or agent-native-editor (exact verified bundle).' \ - ' Native-editor and cooperative-code optionally accept --host-tools-directory PATH (verified workspace-only QEMU).' + ' content-provider optionally accepts --browser-filter-bundle ARCHIVE --browser-filter-manifest-sha256 SHA256.' \ + ' Native-editor, cooperative-code and content-provider accept --host-tools-directory PATH (verified workspace-only QEMU).' } print_plan() { @@ -506,6 +509,16 @@ while [ "$#" -gt 0 ]; do code_bundle=$2 shift ;; + --browser-filter-bundle) + if [ "$#" -lt 2 ] || [ -n "$browser_filter_bundle" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + browser_filter_bundle=$2 + shift + ;; + --browser-filter-manifest-sha256) + if [ "$#" -lt 2 ] || [ -n "$browser_filter_manifest_sha256" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + browser_filter_manifest_sha256=$2 + shift + ;; --code-manifest-sha256) if [ "$#" -lt 2 ] || [ -n "$code_manifest_sha256" ]; then usage >&2; exit 64; fi code_manifest_sha256=$2 @@ -560,7 +573,7 @@ done if [ "$mode" = preview ]; then if [ "$approval" != no ] \ - || [ -n "$image_path$mpquic_path$package_path$native_runtime_path$host_tools_directory$output_directory$expected_commit$code_bundle$code_manifest_sha256" ]; then + || [ -n "$image_path$mpquic_path$package_path$native_runtime_path$host_tools_directory$output_directory$expected_commit$code_bundle$code_manifest_sha256$browser_filter_bundle$browser_filter_manifest_sha256" ]; then usage >&2 exit 64 fi @@ -589,12 +602,18 @@ if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative elif [ -n "$code_bundle$code_manifest_sha256" ]; then exit 64 fi +if [ -n "$browser_filter_bundle$browser_filter_manifest_sha256" ]; then + [ "$scenario" = content-provider ] || exit 64 + case $browser_filter_bundle in /*) ;; *) exit 64 ;; esac + case $browser_filter_manifest_sha256 in ''|*[!0-9a-f]*) exit 64 ;; esac + [ "${#browser_filter_manifest_sha256}" -eq 64 ] || exit 64 +fi case $expected_commit in ''|*[!0-9a-f]*) exit 64 ;; esac case ${#expected_commit} in 40|64) ;; *) exit 64 ;; esac [ "$(id -u)" -ne 0 ] || { printf '%s\n' 'VM runner must remain unprivileged' >&2; exit 77; } if [ -n "$host_tools_directory" ]; then - case $scenario in agent-native-editor|agent-cooperative-code|agent-cooperative-code-proposal) ;; *) exit 64 ;; esac + case $scenario in agent-native-editor|agent-cooperative-code|agent-cooperative-code-proposal|content-provider) ;; *) exit 64 ;; esac case $host_tools_directory in /*) ;; *) exit 64 ;; esac python3 -B "$(dirname -- "$0")/browser-native-tools.py" --verify --output "$host_tools_directory" PATH=$host_tools_directory/bin:$PATH @@ -752,6 +771,17 @@ git -C "$REPOSITORY" archive --format=tar --prefix=source/ "$expected_commit" \ [ "$(stat -Lc '%s' "$SOURCE_ARCHIVE")" -gt 0 ] || exit 69 [ "$(stat -Lc '%s' "$SOURCE_ARCHIVE")" -le 536870912 ] || exit 69 SOURCE_SHA256=$(sha256sum "$SOURCE_ARCHIVE" | awk '{ print $1 }') +BROWSER_FILTER_ARCHIVE=$RUN_DIRECTORY/browser-filter.tar +BROWSER_FILTER_ARCHIVE_SHA256=none +BROWSER_FILTER_MANIFEST_SHA256=none +if [ -n "$browser_filter_bundle" ]; then + python3 -B "$HERE/browser-filter-bundle.py" verify "$browser_filter_bundle" "$browser_filter_manifest_sha256" + install -m 0600 "$browser_filter_bundle" "$BROWSER_FILTER_ARCHIVE" + python3 -B "$HERE/browser-filter-bundle.py" verify "$BROWSER_FILTER_ARCHIVE" "$browser_filter_manifest_sha256" + BROWSER_FILTER_ARCHIVE_SHA256=$(sha256sum "$BROWSER_FILTER_ARCHIVE" | awk '{ print $1 }') + BROWSER_FILTER_MANIFEST_SHA256=$browser_filter_manifest_sha256 + printf '%s\n' 'Additional proof: exact Browser/ESR/signed-addons bundle, real core filter broker, uBO engine probes and revocation.' +fi CODE_ARCHIVE=$RUN_DIRECTORY/cooperative-code.tar CODE_ARCHIVE_SHA256=none CODE_MANIFEST_SHA256=none @@ -1356,6 +1386,8 @@ scenario=$5 native_runtime_sha256=none code_archive_sha256=none code_manifest_sha256=none +browser_filter_archive_sha256=none +browser_filter_manifest_sha256=none # Native and cooperative bundles share positions only in disjoint scenarios. case $scenario in agent-native-coding|agent-native-editor) @@ -1367,6 +1399,16 @@ case $scenario in code_archive_sha256=${6:-none} code_manifest_sha256=${7:-none} ;; + content-provider) + if [ "$#" -eq 7 ]; then + browser_filter_archive_sha256=$6 + browser_filter_manifest_sha256=$7 + case $browser_filter_archive_sha256:$browser_filter_manifest_sha256 in *[!0-9a-f:]*) exit 64 ;; esac + [ "${#browser_filter_archive_sha256}" -eq 64 ] && [ "${#browser_filter_manifest_sha256}" -eq 64 ] || exit 64 + else + [ "$#" -eq 5 ] || exit 64 + fi + ;; *) [ "$#" -eq 5 ] || exit 64 ;; esac case $scenario in alpha|datapath|reciprocity|reciprocity-private-dns|local-link|mixed-link|mpquic-growth|mptcp-growth|mptcp-refill|sharing|download-sharing|wifi-mesh|transaction-abci|wifi-link|uplink-link|crash-recovery|content|content-message|content-https|content-provider|private-storage-peer|private-storage-replicas|signal-backup|signal-backup-fragments|private-storage-handoff|private-storage-fragments|image-snapshot|cloud-private-file|cloud-private-upload|content-custody|content-repair|content-replication|content-mailbox|dns-cache|agent-training|agent-owner-priority|agent-owner-cancel|agent-private-task|agent-private-code|agent-private-conversation|agent-native-coding|agent-native-editor|agent-private-browser|browser-network|agent-reasoning|agent-artifact|agent-train-cycle|agent-train-loop|agent-artifact-quarantine|agent-jobs|agent-jobs-loss|agent-jobs-follow|agent-jobs-peer-recovery|agent-jobs-ready-queue|agent-jobs-package-queue|agent-public-task|agent-public-document|agent-cooperative-browser|agent-cooperative-browser-discovered|agent-cooperative-code|agent-cooperative-code-proposal|agent-public-collection|agent-public-network-sources|agent-task-graph|agent-ready-dag|agent-model-planning|agent-model-task-graph|agent-successor-serving|agent-active-recovery|agent-adapter-aggregation|agent-autonomous-aggregation|agent-policy-assessment) ;; *) exit 64 ;; esac @@ -1494,7 +1536,7 @@ fi if [ "$scenario" = image-snapshot ] || [ "$scenario" = cloud-private-file ] || [ "$scenario" = cloud-private-upload ]; then sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends gpg gpg-agent gpgconf tar fi -if [ "$scenario" = agent-cooperative-browser ] || [ "$scenario" = agent-cooperative-browser-discovered ] || [ "$scenario" = browser-network ] || [ "$scenario" = cloud-private-file ] || [ "$scenario" = cloud-private-upload ]; then +if [ "$scenario" = agent-cooperative-browser ] || [ "$scenario" = agent-cooperative-browser-discovered ] || [ "$scenario" = browser-network ] || [ "$scenario" = cloud-private-file ] || [ "$scenario" = cloud-private-upload ] || [ "$browser_filter_manifest_sha256" != none ]; then # Exact pinned ESR is extracted separately, not installed; these are its # ordinary Debian dependencies plus the isolated browser/HTTPS fixture tools. sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ @@ -1523,6 +1565,11 @@ fi tar -xzf source.tar.gz cd source test -x tests/integration/kvm-alpha-topology.sh +if [ "$browser_filter_manifest_sha256" != none ]; then + guest_phase browser-filter-inputs + printf '%s /home/vpci/browser-filter.tar\n' "$browser_filter_archive_sha256" | sha256sum --check --strict - + python3 -B tests/integration/browser-filter-bundle.py verify /home/vpci/browser-filter.tar "$browser_filter_manifest_sha256" +fi if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then guest_phase code-inputs case $code_archive_sha256:$code_manifest_sha256 in *[!0-9a-f:]*) exit 64 ;; esac @@ -1660,6 +1707,9 @@ set +e if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then set -- ./tests/integration/kvm-alpha-topology.sh --code-bundle /home/vpci/cooperative-code-inputs \ --code-manifest-sha256 "$code_manifest_sha256" +elif [ "$browser_filter_manifest_sha256" != none ]; then + set -- ./tests/integration/kvm-alpha-topology.sh --browser-filter-bundle /home/vpci/browser-filter.tar \ + --browser-filter-manifest-sha256 "$browser_filter_manifest_sha256" else set -- env "$@" ./tests/integration/kvm-alpha-topology.sh fi @@ -1877,6 +1927,7 @@ done [ "$ssh_attempt" -lt 240 ] || { tail -c 131072 "$CONSOLE" >&2; exit 1; } ssh_base sudo -n cloud-init status --wait >/dev/null scp_to "$SOURCE_ARCHIVE" /home/vpci/source.tar.gz +if [ -n "$browser_filter_bundle" ]; then scp_to "$BROWSER_FILTER_ARCHIVE" /home/vpci/browser-filter.tar; fi if [ -n "$native_runtime_path" ]; then scp_to "$native_runtime_path" /home/vpci/native-runtime.tar.gz; fi if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then scp_to "$CODE_ARCHIVE" /home/vpci/cooperative-code.tar; fi if [ -n "$mpquic_path" ]; then scp_to "$mpquic_path" /home/vpci/volparossa-mpquic; fi @@ -1920,6 +1971,15 @@ driver_time_bound=2400s [ "$scenario" != signal-backup ] || driver_time_bound=7200s [ "$scenario" != signal-backup-fragments ] || driver_time_bound=7200s case $scenario in + content-provider) + if [ -n "$browser_filter_bundle" ]; then + ssh_bounded "$driver_time_bound" /home/vpci/guest-driver.sh "$expected_commit" "$SOURCE_SHA256" \ + "$MPQUIC_SHA256" "$PACKAGE_SHA256" "$scenario" "$BROWSER_FILTER_ARCHIVE_SHA256" "$BROWSER_FILTER_MANIFEST_SHA256" + else + ssh_bounded "$driver_time_bound" /home/vpci/guest-driver.sh "$expected_commit" "$SOURCE_SHA256" \ + "$MPQUIC_SHA256" "$PACKAGE_SHA256" "$scenario" + fi + ;; agent-native-coding|agent-native-editor) ssh_bounded "$driver_time_bound" /home/vpci/guest-driver.sh "$expected_commit" "$SOURCE_SHA256" \ "$MPQUIC_SHA256" "$PACKAGE_SHA256" "$scenario" "$NATIVE_RUNTIME_SHA256" diff --git a/tests/integration/test-browser-filter-bundle.py b/tests/integration/test-browser-filter-bundle.py new file mode 100644 index 000000000..8958e6b07 --- /dev/null +++ b/tests/integration/test-browser-filter-bundle.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert transfer tests: no Firefox, Browser imports, network or VM execution.""" +import hashlib +import json +import os +from pathlib import Path +import runpy +import stat +import tarfile +import tempfile +import unittest + +B = runpy.run_path(str(Path(__file__).with_name("browser-filter-bundle.py"))) + + +class BrowserFilterBundle(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="browser-filter-bundle-test-") + self.root = Path(self.temporary.name).resolve() + self.source, self.stage = self.root / "source", self.root / "stage" + self.source.mkdir(mode=0o700) + self.stage.mkdir(mode=0o700) + for name in B["SOURCE_FILES"]: + data = ("synthetic source " + name + "\n").encode() + if name.startswith("scripts/"): + data = b"# inert dependency\nraise RuntimeError('capture must never execute this input')\n" + self.put(self.source / name, data) + imports = "\n".join("import " + name for name in sorted(B["SCRIPTS"] - {"smoke_filter_delivery"})) + self.put(self.source / "scripts/smoke_filter_delivery.py", (imports + "\nraise RuntimeError('not executable input')\n").encode()) + for name in B["RUNTIME_FILES"]: + self.put(self.stage / name, ("synthetic retained runtime " + name + "\n").encode(), + 0o755 if B["STAGE"] + name in B["EXECUTABLES"] else 0o664) + self.archive = self.root / "inputs.tar" + + def tearDown(self): + self.temporary.cleanup() + + @staticmethod + def put(path, data, mode=0o600): + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(mode) + + def capture(self): + return B["capture"](self.source, self.stage, self.archive) + + def unpack(self, receipt, name="unpacked"): + target = self.root / name + B["checked_archive"](self.archive, receipt["manifest_sha256"], target) + return target + + def members(self): + with tarfile.open(self.archive, "r:") as archive: + return [(item.name, archive.extractfile(item).read(), item.mode) for item in archive] + + def rewrite(self, entries, suffix=b"\0" * 1024): + with self.archive.open("wb") as output: + for name, raw, mode in entries: + output.write(B["header"](name, len(raw), mode)) + output.write(raw) + output.write(b"\0" * (-len(raw) % 512)) + output.write(suffix) + + def test_roundtrip_is_deterministic_preserves_xpis_and_never_executes_input(self): + original = self.stage / "distribution/extensions/uBlock0@raymondhill.net.xpi" + before = original.read_bytes(), original.stat().st_mode + receipt = self.capture() + second = self.root / "second.tar" + again = B["capture"](self.source, self.stage, second) + self.assertEqual(receipt, again) + self.assertEqual(self.archive.read_bytes(), second.read_bytes()) + self.assertEqual(receipt["files"], len(B["FILES"])) + self.assertEqual(receipt, B["checked_archive"](self.archive, receipt["manifest_sha256"])) + target = self.unpack(receipt) + self.assertEqual(original.read_bytes(), (target / B["STAGE"] / original.relative_to(self.stage)).read_bytes()) + self.assertEqual(before, (original.read_bytes(), original.stat().st_mode)) + report = B["verify_tree"](target, receipt["manifest_sha256"]) + self.assertFalse(report["private_output_inspected"]) + for name in B["FILES"]: + self.assertEqual(stat.S_IMODE((target / name).stat().st_mode), B["mode"](name)) + for directory, subdirs, _files in os.walk(target): + self.assertEqual(stat.S_IMODE(Path(directory).stat().st_mode), 0o700) + + def test_wrong_independent_hash_refuses_before_destination_creation(self): + self.capture() + target = self.root / "no-output" + with self.assertRaises(ValueError): + B["checked_archive"](self.archive, "f" * 64, target) + self.assertFalse(target.exists()) + + def test_runtime_inventory_is_exact(self): + self.put(self.stage / "unexpected.so", b"extra") + with self.assertRaises(ValueError): + self.capture() + self.assertFalse(self.archive.exists()) + + def test_missing_runtime_file_is_not_silently_omitted(self): + (self.stage / "libxul.so").unlink() + with self.assertRaises(ValueError): + self.capture() + + def test_missing_or_expanded_python_import_closure_is_rejected(self): + driver = self.source / "scripts/smoke_filter_delivery.py" + original = driver.read_bytes() + for raw in (b"import os\n", original + b"import unpublished_dependency\n", original + b"from . import sibling\n"): + with self.subTest(raw=raw[-40:]): + driver.write_bytes(raw) + with self.assertRaises(ValueError): + self.capture() + self.assertFalse(self.archive.exists()) + + def test_unsafe_source_symlink_hardlink_fifo_and_modes_are_rejected(self): + source = self.source / "defaults/privacy.json" + original = source.read_bytes() + source.unlink() + replacement = self.root / "other.json" + self.put(replacement, original) + source.symlink_to(replacement) + with self.assertRaises(ValueError): self.capture() + source.unlink() + os.link(replacement, source) + with self.assertRaises(ValueError): self.capture() + source.unlink() + os.mkfifo(source, 0o600) + with self.assertRaises(ValueError): self.capture() + source.unlink() + self.put(source, original) + for mode in (0o606, 0o4600, 0o2600, 0o1600): + source.chmod(mode) + with self.subTest(mode=mode), self.assertRaises(ValueError): self.capture() + + def test_existing_archive_and_destination_are_never_overwritten(self): + receipt = self.capture() + before = self.archive.read_bytes() + with self.assertRaises(ValueError): self.capture() + self.assertEqual(before, self.archive.read_bytes()) + target = self.unpack(receipt) + with self.assertRaises(ValueError): + B["checked_archive"](self.archive, receipt["manifest_sha256"], target) + + def test_duplicate_missing_extra_and_trailing_members_are_rejected(self): + receipt = self.capture() + entries = self.members() + cases = (entries + [entries[-1]], entries[:-1], + [entries[0], entries[0], *entries[2:]], + [*entries, ("../escape", b"bad", 0o600)]) + for entries_ in cases: + with self.subTest(names=[x[0] for x in entries_[-2:]]): + self.rewrite(entries_) + with self.assertRaises(ValueError): + B["checked_archive"](self.archive, receipt["manifest_sha256"]) + self.rewrite(entries, suffix=b"\0" * 1024 + b"hidden" + b"\0" * 506) + with self.assertRaises(ValueError): + B["checked_archive"](self.archive, receipt["manifest_sha256"]) + + def test_payload_corruption_leaves_no_accepted_manifest(self): + receipt = self.capture() + entries = self.members() + name, raw, mode = entries[1] + entries[1] = name, bytes([raw[0] ^ 1]) + raw[1:], mode + self.rewrite(entries) + target = self.root / "partial" + with self.assertRaises(ValueError): + B["checked_archive"](self.archive, receipt["manifest_sha256"], target) + self.assertFalse((target / "INPUTS.json").exists()) + + def test_archive_header_modes_and_links_cannot_grant_execution_or_escape(self): + receipt = self.capture() + original = self.archive.read_bytes() + entries = self.members() + offset = 512 + len(entries[0][1]) + (-len(entries[0][1]) % 512) + name, raw, mode = entries[1] + for permissions, kind in ((0o777, tarfile.REGTYPE), (mode, tarfile.SYMTYPE), + (mode, tarfile.LNKTYPE), (mode, tarfile.XHDTYPE)): + entry = tarfile.TarInfo(name) + entry.size, entry.mode, entry.type = len(raw), permissions, kind + malformed = entry.tobuf(format=tarfile.USTAR_FORMAT) + self.archive.write_bytes(original[:offset] + malformed + original[offset + 512:]) + with self.subTest(permissions=permissions, kind=kind), self.assertRaises(ValueError): + B["checked_archive"](self.archive, receipt["manifest_sha256"]) + + def test_manifest_schema_duplicate_keys_paths_bounds_and_modes_are_closed(self): + receipt = self.capture() + entries = self.members() + original = json.loads(entries[0][1]) + for fault in ("version", "path", "negative", "huge", "bool_size", "mode", "extra"): + value = json.loads(entries[0][1]) + name = sorted(value["files"])[0] + if fault == "version": value["version"] = True + elif fault == "path": value["files"]["../escape"] = value["files"].pop(name) + elif fault == "negative": value["files"][name]["bytes"] = -1 + elif fault == "huge": value["files"][name]["bytes"] = B["MAX_FILE"] + 1 + elif fault == "bool_size": value["files"][name]["bytes"] = True + elif fault == "mode": value["files"][name]["mode"] = 0o777 + else: value["extra"] = "ignored?" + raw = B["encode"](value) + with self.subTest(fault=fault), self.assertRaises(ValueError): + B["manifest"](raw, hashlib.sha256(raw).hexdigest()) + raw = entries[0][1].replace(b'"version":1', b'"version":1,"version":1') + with self.assertRaises(ValueError): B["manifest"](raw, hashlib.sha256(raw).hexdigest()) + value = original + for name in value["files"]: + value["files"][name]["bytes"] = B["bound"](name) + raw = B["encode"](value) + with self.assertRaises(ValueError): B["manifest"](raw, hashlib.sha256(raw).hexdigest()) + + def test_tree_verification_allows_only_named_output_and_detects_input_changes(self): + receipt = self.capture() + target = self.unpack(receipt) + output = target / B["OUTPUT"] + output.mkdir(mode=0o700) + self.put(output / "private.log", b"not input evidence") + B["verify_tree"](target, receipt["manifest_sha256"]) + extra = target / "scripts/unreviewed.py" + self.put(extra, b"unreviewed") + with self.assertRaises(ValueError): B["verify_tree"](target, receipt["manifest_sha256"]) + extra.unlink() + source = target / "defaults/privacy.json" + source.chmod(0o644) + with self.assertRaises(ValueError): B["verify_tree"](target, receipt["manifest_sha256"]) + source.chmod(0o600) + source.write_bytes(b"changed input") + with self.assertRaises(ValueError): B["verify_tree"](target, receipt["manifest_sha256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-browser-filter-delivery.py b/tests/integration/test-browser-filter-delivery.py new file mode 100644 index 000000000..c056bec02 --- /dev/null +++ b/tests/integration/test-browser-filter-delivery.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert joined-proof/ordering tests; never starts Browser, broker or namespaces.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import runpy +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).parent +CHECK = runpy.run_path(str(HERE / "browser-filter-delivery.py")) +FILTER_TEST = runpy.run_path(str(HERE / "test-content-provider-filter-smoke.py")) + + +def fixture(): + selected = FILTER_TEST["fixture"]()["input"] + expected = {key: selected[key] for key in ("publisher_key", "name", "manifest_id")} + expected["authority_expires_unix_seconds"] = 2_000_000_000 + manifest = {"files": {"scripts/smoke_filter_delivery.py": {"sha256": "a" * 64}, + "integration/filters/Gecko.sys.mjs": {"sha256": "b" * 64}, + "tests/fixtures/filter_delivery.sys.mjs": {"sha256": "c" * 64}}} + stage = dict(version="140.16.0", source_stamp="d" * 40, runtime_sha256={"firefox-esr": "e" * 64}, + extensions={"packages": [dict(id="test-original", sha256="f" * 64)]}) + logs = {name: dict(bytes=0, sha256=hashlib.sha256(b"").hexdigest()) for name in ("browser", "broker")} + report = dict(schema=1, passed=True, scope=CHECK["SCOPE"], operation="complete", failure=None, + expected=expected, broker_sha256="7" * 64, host_read_only=True, interfaces=["lo"], browser_exit_status=0, + broker_cleanup=dict(started=True, reaped=True, graceful=True), broker_socket_removed=True, + temporary_profiles_removed=True, receipts={operation: CHECK["operation_receipt"](operation, expected) + for operation in CHECK["OPERATIONS"]}, + probes={case: {kind: 0 if case == "enrolled" and kind != "neutral" else 1 for kind in ("ads", "tracker", "neutral")} + for case in CHECK["CASES"]}, + driver_sha256="a" * 64, + module_sha256={"integration/filters/Gecko.sys.mjs": "b" * 64, "tests/fixtures/filter_delivery.sys.mjs": "c" * 64}, + runtime_version=stage["version"], runtime_source_stamp=stage["source_stamp"], runtime_sha256=stage["runtime_sha256"], + extension_packages=stage["extensions"]["packages"], logs=logs) + outer = dict(schema=1, temporary_profiles_removed=True, broker_copy_removed=True, logs=logs) + return report, outer, expected, manifest, stage, "7" * 64 + + +class JoinedFilterProof(unittest.TestCase): + def setUp(self): + guard = patch.object(os, "execv", side_effect=AssertionError("execution forbidden")) + guard.start(); self.addCleanup(guard.stop) + + def test_exact_real_broker_asset_engine_sequence_is_required(self): + CHECK["validate_browser"](*fixture()) + + def test_join_rejects_wrong_bytes_manifest_mutation_missing_reload_or_missing_cleanup(self): + changes = [(("passed",), False), (("browser_exit_status",), 1), (("schema",), True), + (("broker_sha256",), "0" * 64), (("driver_sha256",), "0" * 64), + (("module_sha256", "integration/filters/Gecko.sys.mjs"), "0" * 64), + (("receipts", "fetch", "delivery", "manifest_id"), "f" * 64), + (("receipts", "fetch", "delivery", "bytes"), 62), + (("receipts", "enroll", "asset", "completed"), 0), + (("receipts", "enroll", "asset", "accepted"), 2), + (("receipts", "enroll", "result", "receipt", "reloadEvents"), 1), + (("receipts", "suspend", "broker_revoked"), False), + (("receipts", "retry", "result", "attempted"), True), + (("probes", "enrolled", "ads"), 1), (("probes", "enrolled", "tracker"), 1), + (("probes", "baseline", "ads"), 0), (("probes", "suspended", "neutral"), 0), + (("broker_cleanup", "graceful"), False), (("temporary_profiles_removed",), False)] + for path, value in changes: + args = list(fixture()); args[0] = copy.deepcopy(args[0]); target = args[0] + for part in path[:-1]: target = target[part] + target[path[-1]] = value + with self.subTest(path=path), self.assertRaises(ValueError): CHECK["validate_browser"](*args) + + def test_extra_private_fields_and_scope_overclaims_are_refused(self): + for change in (lambda report: report.update(raw_output="PRIVATE"), + lambda report: report["logs"]["browser"].update(raw="PRIVATE"), + lambda report: report["scope"].update(cold_cache_proven=True)): + args = list(fixture()); args[0] = copy.deepcopy(args[0]); change(args[0]) + with self.assertRaises(ValueError) as error: CHECK["validate_browser"](*args) + self.assertNotIn("PRIVATE", str(error.exception)) + + def test_outer_cleanup_must_match_inner_actual_logs_and_remove_broker_copy(self): + args = list(fixture()); args[1] = copy.deepcopy(args[1]); args[1]["broker_copy_removed"] = False + with self.assertRaises(ValueError): CHECK["validate_browser"](*args) + args = list(fixture()); args[1] = copy.deepcopy(args[1]); args[1]["logs"]["broker"]["sha256"] = "0" * 64 + with self.assertRaises(ValueError): CHECK["validate_browser"](*args) + + def test_original_cold_packet_route_and_actual_peer_bytes_remain_mandatory(self): + original = FILTER_TEST["fixture"]() + CHECK["validate_cold"](original) + mutations = [("peer_bytes", 0), ("providers_used", 0), ("provider_peer_ids", [])] + for key, value in mutations: + changed = copy.deepcopy(original); changed["cold"]["application"]["receipt"][key] = value + with self.assertRaises(ValueError): CHECK["validate_cold"](changed) + changed = copy.deepcopy(original); changed["cold"]["privacy"]["client"]["direct_client_exit_packets"] = 1 + with self.assertRaises(ValueError): CHECK["validate_cold"](changed) + changed = copy.deepcopy(original); changed["selected_route"]["paths"][1]["relay_peer_id"] = changed["selected_route"]["paths"][0]["relay_peer_id"] + with self.assertRaises(ValueError): CHECK["validate_cold"](changed) + + def test_owner_authority_is_four_fields_same_publication_and_at_most_ten_minutes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "browser-filter-delivery"; root.mkdir(mode=0o700) + (root / "home").mkdir(mode=0o700) + CHECK["write_new"](root / "selection.json", json.dumps(FILTER_TEST["fixture"]()["input"]).encode()) + with patch("time.time", return_value=1000): result = CHECK["prepare"](root, 1500) + self.assertEqual(result["expected"]["authority_expires_unix_seconds"], 1500) + self.assertEqual(set(result["expected"]), {"publisher_key", "name", "manifest_id", "authority_expires_unix_seconds"}) + self.assertEqual(CHECK["read"](root / "expected.json"), result["expected"]) + self.assertTrue(all(CHECK["cleanup"](root).values())); self.assertFalse(root.exists()) + + def test_cleanup_refuses_unknown_entries_before_removing_selection(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "browser-filter-delivery"; root.mkdir(mode=0o700) + CHECK["write_new"](root / "selection.json", b"keep") + CHECK["write_new"](root / "unknown", b"keep too") + with self.assertRaises(ValueError): CHECK["cleanup"](root) + self.assertEqual((root / "selection.json").read_bytes(), b"keep") + + def test_launcher_observes_primary_control_gid_empty_groups_no_caps_and_exact_netns(self): + fields = dict(Uid="985 985 985 985", Gid="984 984 984 984", Groups="", NoNewPrivs="1", + CapInh="0", CapPrm="0", CapEff="0", CapBnd="0", CapAmb="0") + for key, bad in ((None, None), ("Gid", "985 985 985 985"), ("Groups", "984"), ("CapEff", "1"), ("NoNewPrivs", "0")): + candidate = dict(fields) + if key: candidate[key] = bad + with patch.object(Path, "read_text", return_value="\n".join(k+":"+v for k,v in candidate.items())), \ + patch.object(os, "readlink", return_value="net:[20]"): + if key: + with self.assertRaises(ValueError): CHECK["process_boundary"]("net:[10]", "net:[20]", 985, 984) + else: + self.assertTrue(CHECK["process_boundary"]("net:[10]", "net:[20]", 985, 984)["all_capabilities_dropped"]) + + def test_failed_original_receipt_retained_byte_exact_and_absence_not_success(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); output = work / "client-fixtures/browser-filter-source/build/filter-delivery" + output.mkdir(parents=True) + raw = b'{"schema":1,"passed":false,"failure":"fixture_failed"}' + (output / "report.json").write_bytes(raw) + (output / "report.json").chmod(0o600) + first = CHECK["retain"](work) + self.assertFalse(first["successful_native_execution_claimed"]) + self.assertEqual((work / "browser-filter-delivery-original-report.json").read_bytes(), raw) + self.assertEqual(CHECK["retain"](work), first) + (output / "report.json").write_bytes(b'{"schema":1,"passed":true}') + with self.assertRaises(ValueError): CHECK["retain"](work) + + def test_closed_file_reader_refuses_duplicate_json_keys_symlinks_and_oversize(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "receipt.json"; path.write_text('{"a":1,"a":2}') + path.chmod(0o600) + with self.assertRaises(ValueError): CHECK["read"](path) + link = path.parent / "link"; link.symlink_to(path) + with self.assertRaises(ValueError): CHECK["raw_file"](link) + path.write_bytes(b"a" * 65537) + with self.assertRaises(ValueError): CHECK["raw_file"](path) + + def test_opt_in_call_is_additive_after_original_cold_before_original_warm_baseline(self): + shell = (HERE / "content-provider-filter-smoke.sh").read_text() + cold = shell.index(" content_provider_filter_consume cold\n") + hook = shell.index('if [ -n "${browser_filter_bundle:-}" ]; then', cold) + baseline = shell.index(" filter_cache_before=", cold) + self.assertLess(cold, hook); self.assertLess(hook, baseline) + self.assertIn("browser_filter_delivery_run", shell[hook:baseline]) + source = (HERE / "browser-filter-delivery.sh").read_text() + self.assertIn('--regid="$site_control_gid" --clear-groups', source) + self.assertIn('ip netns exec "$CLIENT" setpriv', source) + self.assertIn('"$WORK/bin/browser-filter-bundle.py" verify-tree', source) + self.assertIn('env HOME="$browser_filter_input/home"', source) + self.assertLess(source.index(' retain "$WORK"'), source.index('"$browser_filter_status" -eq 0')) + self.assertNotIn('chmod "$filter_cache"', source) + self.assertNotIn('chown "$filter_cache"', source) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/integration/test-browser-filter-runner.py b/tests/integration/test-browser-filter-runner.py new file mode 100644 index 000000000..66d01c22a --- /dev/null +++ b/tests/integration/test-browser-filter-runner.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Only shell argument/syntax checks; never start a VM, browser or service.""" +from pathlib import Path +import subprocess +import unittest + +HERE = Path(__file__).resolve().parent +RUNNERS = ("run-alpha-topology-vm.sh", "kvm-alpha-topology.sh") + + +class BrowserFilterArguments(unittest.TestCase): + def call(self, runner, *args): + return subprocess.run(["/bin/sh", str(HERE / runner), *args], + capture_output=True, timeout=5) + + def test_shell_syntax(self): + for runner in RUNNERS: + subprocess.run(["/bin/sh", "-n", str(HERE / runner)], check=True, timeout=5) + + def test_original_provider_preview_is_inert_and_available(self): + for runner in RUNNERS: + result = self.call(runner, "--preview", "--scenario", "content-provider") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(b"PREVIEW ONLY", result.stdout) + + def test_bundle_flags_cannot_authorize_preview_execution(self): + for runner in RUNNERS: + for flag in ("--browser-filter-bundle", "--browser-filter-manifest-sha256"): + for value in (None, "", "/unread-fixture"): + args = ["--preview", "--scenario", "content-provider", flag] + if value is not None: + args.append(value) + self.assertEqual(self.call(runner, *args).returncode, 64) + + def test_duplicate_flags_are_refused_before_files_or_vm(self): + for runner in RUNNERS: + for flag in ("--browser-filter-bundle", "--browser-filter-manifest-sha256"): + self.assertEqual(self.call(runner, "--execute", "--scenario", "content-provider", + flag, "/unread-one", flag, "/unread-two").returncode, 64) + + def test_bundle_is_not_authorized_by_environment(self): + for runner in RUNNERS: + source = (HERE / runner).read_text() + self.assertIn("\nbrowser_filter_bundle=\n", source) + self.assertIn("\nbrowser_filter_manifest_sha256=\n", source) + self.assertIn('[ "$scenario" = content-provider ] || exit 64', source) + + +if __name__ == "__main__": + unittest.main() From 2e4f4527d669b104ed56d87184bb11ab73ccabe6 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:56:36 +0200 Subject: [PATCH 02/49] test: protect host memory and retain joined browser receipts --- docs/IMPLEMENTATION_STATUS.md | 22 ++++- tests/integration/browser-filter-delivery.py | 87 +++++++++++++++++++ tests/integration/browser-filter-delivery.sh | 8 ++ tests/integration/kvm-alpha-topology.sh | 6 ++ tests/integration/run-alpha-topology-vm.sh | 25 +++++- .../test-browser-filter-delivery.py | 59 +++++++++++++ .../integration/test-browser-filter-runner.py | 75 +++++++++++++++- 7 files changed, 276 insertions(+), 6 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index a696f3cc2..892d06159 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -118,12 +118,30 @@ can enable it. The browser and broker run as the disposable worker in nested namespaces; neither receives permission to read the agent cache directly. The Browser fixture passes 84 focused JavaScript checks and 24 Python checks; -12 core transfer checks and five runner argument checks also pass. These are -preparatory component checks, **not a passed combined runtime trial**. The hook +12 core transfer checks, nine runner argument/resource checks and 15 joined-proof +and receipt-export checks also pass. These are preparatory component checks, +**not a passed combined runtime trial**. The hook must still prove real control access after namespace mapping, end-to-end browser behavior and full cleanup. Automatic startup enrollment, native admission registration and expiry handling for already loaded rules remain open. +The first local combined attempt used core source +`82450eb84f96c66818c069c1b287f0633c2f46d5` and Browser bundle manifest +`ea6dd5425ffec9c73e226eecee6aa71076c78cc40f537e59501cc1a8698e6c9b`. +It was stopped during the guest Rust build: the disposable overlay under +RAM-backed `/tmp` competed with the 4-GiB guest for development-host memory. +No Browser runtime or combined acceptance was reached, and no Browser receipts +were collected. The VM and its owned temporary overlay were removed; source +trees, the base image and frozen inputs were preserved. This is an aborted +preparation attempt, not evidence that the joined data path passed or failed. +The corrected opt-in runner places this trial's scratch beside the workspace +evidence, rejects a tmpfs/ramfs parent and requires available host memory for the +unchanged 4-GiB guest plus a 2-GiB reserve before launch. This is a preflight +snapshot, not a memory reservation. A fixed JSON-only export retains original +receipts and their hashes, including failures, without exporting private logs, +profiles or authority inputs. The corrected combined runtime trial remains +pending. + ### Bank and Transaction layer implementation The 2026-10-04 user extension adds VOLPAROSSA Bank and a reusable Transaction-layer. diff --git a/tests/integration/browser-filter-delivery.py b/tests/integration/browser-filter-delivery.py index 232f925bd..9c98688db 100644 --- a/tests/integration/browser-filter-delivery.py +++ b/tests/integration/browser-filter-delivery.py @@ -31,6 +31,41 @@ "broker_sha256", "expected", "runtime_version", "runtime_source_stamp", "runtime_sha256", "extension_packages", "failure", "host_read_only", "interfaces", "browser_exit_status", "broker_cleanup", "broker_socket_removed", "temporary_profiles_removed", "logs", "failure_phase"} +EXPORT_KEYS = { + "original-report": REPORT_KEYS, + "original-outer-cleanup": {"schema", "temporary_profiles_removed", "broker_copy_removed", "logs"}, + "evidence": {"version", "kind", "success", "bundle_manifest_sha256", "core_broker_sha256", "originals_sha256", + "publication", "protected_cold", "browser", "scope", "cleanup"}, + "prepared": {"version", "expected", "created_unix_seconds", "maximum_authority_seconds"}, + "cache-before": {"device", "inode", "mode", "user_uid", "group_gid"}, + "cache-after": {"device", "inode", "mode", "user_uid", "group_gid"}, + "launch": {"version", "boundary"}, + "tree-before": {"version", "action", "transferred", "runtime_executed", "manifest_sha256", "files", "private_output_inspected"}, + "tree-after": {"version", "action", "transferred", "runtime_executed", "manifest_sha256", "files", "private_output_inspected"}, + "input-cleanup": {"directory_removed", "files_removed"}, + "retained": {"version", "originals_sha256", "successful_native_execution_claimed"}, +} +EXPORT_FIELDS = set().union(*EXPORT_KEYS.values(), SCOPE, OPERATIONS, CASES, { + "schema", "status", "choice", "state", "closed", "failed", "result", "attempted", "receipt", "selected", "imported", + "preserved", "freshReload", "reloadEvents", "delivery", "asset", "broker_revoked", "bytes", "sha256", "rules", + "manifest_id", "publisher_key", "name", "authority_expires_unix_seconds", "accepted", "completed", "denied", + "ok", "phase", "ads", "tracker", "neutral", "started", "reaped", "graceful", "broker", "id", "size", + "unpacked_bytes", "zip_entries", "report", "outer-cleanup", "content_type", "peer_bytes", "providers_used", + "privacy_captures_validated", "exit_status", "same_publication_joined", "broker_reuse_requested", "broker_cache_only_proven", + "new_broker_peer_transfer_proven", "two_subflow_bulk_aggregation", "browser_profile_removed", "broker_reaped", + "fixture_authority_removed", "separate_owner_input_removed", "agent_cache_identity_preserved", "agent_cache_private", + "staged_inputs_removed_by_enclosing_topology", "primary_control_gid", "supplementary_groups_empty", "client_namespace", + "outside_parent_namespace", "all_capabilities_dropped", "no_new_privileges", + *("integration/filters/" + name + ".sys.mjs" for name in BUNDLE["MODULES"]), + "tests/fixtures/filter_delivery.sys.mjs", "firefox-esr", "libxul.so", "omni.ja", "browser/omni.ja"}) +EXPORT_WORDS = {"complete", "prepare", "browser_start", "browser_quit", "broker_start", "command", "bootstrap", *OPERATIONS, + "eligible", "opted-out", "fresh", "pending", "active", "suspended", "lo", "0700", "verify-tree", NAME, + "text/plain", "volparossa-browser-filter-delivery", "uBlock0@raymondhill.net", "jid1-BoFifL9Vbdl2zQ@jetpack", + "ATBC@EasonWong", "gdpr@cavi.au.dk", "fixture_command", "fixture_failed", "fixture_cleanup", "fixture_clock", + "fixture_other", "invalid_request", "duplicate_request", "handshake_required", "busy", "unavailable", "revoked", + "clock_error", "expired", "invalid_snapshot", "invalid_response", "not_configured", "owner_config", "owner_context", + "owner_clock", "owner_busy", "owner_closed", "owner_failed", "owner_cleanup", "asset_config", "asset_package", + "asset_context", "asset_authority", "asset_expired", "asset_clock", "asset_busy", "asset_closed", "asset_failed", "asset_cleanup"} def require(condition): @@ -236,6 +271,56 @@ def retain(work): return dict(version=1, originals_sha256=retained, successful_native_execution_claimed=False) +def export_value(value, depth=0, field=None): + """Only the fixture's closed fields/scalars, never raw errors, paths or logs.""" + require(depth <= 12) + if type(value) is dict: + require(len(value) <= 64 and set(value) <= EXPORT_FIELDS) + for key, child in value.items(): export_value(child, depth + 1, key) + elif type(value) is list: + require(len(value) <= 16) + for child in value: export_value(child, depth + 1, field) + elif type(value) is str: + require(value in EXPORT_WORDS or hex64(value) or re.fullmatch(r"[0-9a-f]{40}", value) + or (field in {"version", "runtime_version"} + and re.fullmatch(r"[0-9]{1,3}(?:\.[0-9]{1,3}){1,3}", value))) + else: + require(value is None or type(value) is bool or (type(value) is int and 0 <= value < 2**64)) + + +def copy_artifacts(work, output, uid, gid): + require(work.resolve(strict=True) == work and output.resolve(strict=True) == output and work != output) + require(output.is_dir() and not output.lstat().st_mode & 0o002 and uid >= 0 and gid >= 0) + pending, missing = {}, [] + # Validate the complete available set before copying anything. Empty files + # can result from a command which failed before emitting JSON; never turn + # that absence into a successful-native-execution claim. + for suffix, keys in EXPORT_KEYS.items(): + name = "browser-filter-delivery-" + suffix + ".json" + source = work / name + if not source.exists() and not source.is_symlink(): missing.append(suffix); continue + raw = raw_file(source) + if not raw: missing.append(suffix); continue + value = json.loads(raw, object_pairs_hook=BUNDLE["unique_pairs"]) + require(type(value) is dict and set(value) <= keys) + if suffix == "original-report": + require({"schema", "passed", "failure"} <= set(value) + and type(value["schema"]) is int and value["schema"] == 1 and type(value["passed"]) is bool) + else: require(set(value) == keys) + export_value(value) + pending[name] = raw + result = dict(version=1, originals={name: dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + for name, raw in pending.items()}, missing=missing, successful_native_execution_claimed=False) + pending["browser-filter-delivery-artifacts.json"] = (json.dumps(result, sort_keys=True, separators=(",", ":")) + "\n").encode() + for name, raw in pending.items(): + destination = output / name + if destination.exists() or destination.is_symlink(): require(raw_file(destination) == raw) + else: write_new(destination, raw) + os.chmod(destination, 0o600, follow_symlinks=False) + os.chown(destination, uid, gid, follow_symlinks=False) + return result + + def evidence(work, manifest_sha, binary_sha, uid, original_gid, control_gid): source = work / "client-fixtures/browser-filter-source" output = source / "build/filter-delivery" @@ -308,6 +393,8 @@ def main(): elif action == "cache-state" and len(args) == 3: result = cache_state(Path(args[0]), int(args[1]), int(args[2])) elif action == "launch" and len(args) == 10: return launch(args) elif action == "retain" and len(args) == 1: result = retain(Path(args[0])) + elif action == "copy-artifacts" and len(args) == 4: + result = copy_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3])) elif action == "evidence" and len(args) == 6: result = evidence(Path(args[0]), args[1], args[2], *map(int, args[3:])) else: raise ValueError("browser_filter_delivery_rejected") diff --git a/tests/integration/browser-filter-delivery.sh b/tests/integration/browser-filter-delivery.sh index c02d3c583..0c73b7979 100644 --- a/tests/integration/browser-filter-delivery.sh +++ b/tests/integration/browser-filter-delivery.sh @@ -18,6 +18,14 @@ browser_filter_delivery_cleanup() { # owns final removal of the staged source/runtime/report tree under WORK. } +browser_filter_delivery_copy_artifacts() { + # Exact bounded JSON only, including partial failure receipts. Never export + # the Browser log, archive, temporary authority, profile or source tree. + [ -f "$WORK/bin/browser-filter-delivery.py" ] || return 0 + python3 -B "$WORK/bin/browser-filter-delivery.py" copy-artifacts "$WORK" \ + "$output_directory" "$OUTPUT_UID" "$OUTPUT_GID" >/dev/null +} + browser_filter_delivery_run() { if [ -z "${browser_filter_bundle:-}" ] || [ -z "${browser_filter_manifest_sha256:-}" ]; then fail BROWSER_FILTER_INPUTS_MISSING diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index f6cd4ada9..4e0a6f57a 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1584,6 +1584,9 @@ capture_host_state() { } copy_artifacts() { + if [ -n "$browser_filter_bundle" ] && command -v browser_filter_delivery_copy_artifacts >/dev/null 2>&1; then + browser_filter_delivery_copy_artifacts || return 1 + fi if [ "$scenario" = browser-network ]; then # The application proof has a closed JSON-only export; never copy grant/profile/log files. for artifact in host-state-before.json host-state-after.json a15-evidence.json; do @@ -2235,6 +2238,9 @@ cleanup() { if [ "$scenario" = content-provider ] && command -v content_provider_filter_cleanup >/dev/null 2>&1; then content_provider_filter_cleanup || original_status=1 fi + if [ -n "$browser_filter_bundle" ] && command -v browser_filter_delivery_cleanup >/dev/null 2>&1; then + browser_filter_delivery_cleanup || original_status=1 + fi if [ "$scenario" = mpquic-growth ] && command -v mpquic_growth_cleanup >/dev/null 2>&1; then mpquic_growth_cleanup || original_status=1 fi diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 6ab193055..4ece1e606 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -713,6 +713,19 @@ elif [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperati [ "$(readlink -f -- "$output_directory")" = "$output_directory" ] || exit 64 RUN_DIRECTORY_PARENT=$(dirname -- "$output_directory") RUN_DIRECTORY_PREFIX=cooperative-code-kvm +elif [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle" ]; then + # The real Browser join builds Rust in this 4 GiB guest. A growing qcow2 + # overlay on tmpfs otherwise competes directly with the guest's RAM. + # Keep scratch beside validated workspace evidence, never in its export. + [ "$(readlink -f -- "$output_directory")" = "$output_directory" ] || exit 64 + RUN_DIRECTORY_PARENT=$(dirname -- "$output_directory") + RUN_DIRECTORY_PREFIX=browser-filter-kvm + runner_scratch_filesystem=$(stat -f -c '%T' -- "$RUN_DIRECTORY_PARENT") || exit 69 + case $runner_scratch_filesystem in + ''|tmpfs|ramfs) + printf '%s\n' 'Browser filter VM scratch requires non-memory-backed workspace storage' >&2 + exit 69 ;; + esac fi RUN_DIRECTORY=$(mktemp -d "$RUN_DIRECTORY_PARENT/$RUN_DIRECTORY_PREFIX.XXXXXX") case $RUN_DIRECTORY in "$RUN_DIRECTORY_PARENT"/"$RUN_DIRECTORY_PREFIX".??????) ;; *) exit 69 ;; esac @@ -1804,12 +1817,18 @@ chmod 0700 "$GUEST_DRIVER" set -- -no-reboot case $scenario in wifi-mesh|wifi-link) set -- ;; esac guest_memory_mib=$(guest_memory_for_scenario) -if [ "$guest_memory_mib" -gt 4096 ]; then +runner_reserve_mib=1024 +if [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle" ]; then + runner_reserve_mib=2048 +fi +if [ "$guest_memory_mib" -gt 4096 ] || [ "$runner_reserve_mib" -eq 2048 ]; then # A read-only preflight snapshot, not a host memory reservation. Refuse - # before spawning QEMU if the larger model fixture cannot fit plus 1 GiB. + # before QEMU unless the guest plus its selected host reserve fits. The + # explicit Browser join requires 2 GiB reserve; larger model fixtures keep + # their existing 1 GiB reserve. No guest or model resource limit increases. runner_available_kib=$(awk '$1 == "MemAvailable:" && $3 == "kB" {print $2; found++} END {if (found != 1) exit 1}' /proc/meminfo) case $runner_available_kib in ''|*[!0-9]*) exit 69 ;; esac - if [ "$runner_available_kib" -lt "$(((guest_memory_mib + 1024) * 1024))" ]; then + if [ "$runner_available_kib" -lt "$(((guest_memory_mib + runner_reserve_mib) * 1024))" ]; then printf '%s\n' 'insufficient runner memory for the selected isolated fixture plus host reserve' >&2 exit 69 fi diff --git a/tests/integration/test-browser-filter-delivery.py b/tests/integration/test-browser-filter-delivery.py index c056bec02..5d0088b43 100644 --- a/tests/integration/test-browser-filter-delivery.py +++ b/tests/integration/test-browser-filter-delivery.py @@ -151,6 +151,62 @@ def test_closed_file_reader_refuses_duplicate_json_keys_symlinks_and_oversize(se path.write_bytes(b"a" * 65537) with self.assertRaises(ValueError): CHECK["raw_file"](path) + def test_artifact_export_preserves_bytes_hash_and_omits_private_files(self): + with tempfile.TemporaryDirectory() as temporary: + work, output = Path(temporary) / "work", Path(temporary) / "output" + work.mkdir(); output.mkdir() + report = fixture()[0] + report["extension_packages"][0]["id"] = "uBlock0@raymondhill.net" + raw = json.dumps(report, indent=2).encode() # Deliberately no trailing newline. + name = "browser-filter-delivery-original-report.json" + CHECK["write_new"](work / name, raw) + CHECK["write_new"](work / "browser-filter-delivery.err", b"PRIVATE") + CHECK["write_new"](work / "browser-filter-inputs.tar", b"PRIVATE") + result = CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual((output / name).read_bytes(), raw) + self.assertEqual(result["originals"][name], dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest())) + self.assertFalse(result["successful_native_execution_claimed"]) + self.assertIn("evidence", result["missing"]) + self.assertEqual({path.name for path in output.iterdir()}, {name, "browser-filter-delivery-artifacts.json"}) + self.assertEqual(CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()), result) + + def test_empty_failure_receipts_are_missing_not_success_and_original_false_is_preserved(self): + with tempfile.TemporaryDirectory() as temporary: + work, output = Path(temporary) / "work", Path(temporary) / "output" + work.mkdir(); output.mkdir() + CHECK["write_new"](work / "browser-filter-delivery-evidence.json", b"") + raw = b'{"schema":1,"passed":false,"failure":"fixture_failed"}' + CHECK["write_new"](work / "browser-filter-delivery-original-report.json", raw) + result = CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertIn("evidence", result["missing"]) + self.assertFalse(result["successful_native_execution_claimed"]) + self.assertEqual((output / "browser-filter-delivery-original-report.json").read_bytes(), raw) + + def test_artifact_export_refuses_private_nested_value_before_any_copy(self): + for mutation in (lambda value: value.update(raw="PRIVATE"), + lambda value: value.update(failure="PRIVATE /home/user"), + lambda value: value.update(failure="192.168.1.23"), + lambda value: value.update(logs={"browser": {"raw": "PRIVATE"}})): + with tempfile.TemporaryDirectory() as temporary: + work, output = Path(temporary) / "work", Path(temporary) / "output" + work.mkdir(); output.mkdir() + value = dict(schema=1, passed=False, failure="fixture_failed"); mutation(value) + CHECK["write_new"](work / "browser-filter-delivery-original-report.json", json.dumps(value).encode()) + with self.assertRaises(ValueError): CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + + def test_artifact_export_refuses_destination_overwrite_and_source_symlink(self): + with tempfile.TemporaryDirectory() as temporary: + work, output = Path(temporary) / "work", Path(temporary) / "output" + work.mkdir(); output.mkdir() + name = "browser-filter-delivery-input-cleanup.json" + CHECK["write_new"](work / name, b'{"directory_removed":true,"files_removed":true}') + CHECK["write_new"](output / name, b"keep-original") + with self.assertRaises(ValueError): CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual((output / name).read_bytes(), b"keep-original") + (work / name).unlink(); (work / name).symlink_to(output / name) + with self.assertRaises(ValueError): CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + def test_opt_in_call_is_additive_after_original_cold_before_original_warm_baseline(self): shell = (HERE / "content-provider-filter-smoke.sh").read_text() cold = shell.index(" content_provider_filter_consume cold\n") @@ -166,6 +222,9 @@ def test_opt_in_call_is_additive_after_original_cold_before_original_warm_baseli self.assertLess(source.index(' retain "$WORK"'), source.index('"$browser_filter_status" -eq 0')) self.assertNotIn('chmod "$filter_cache"', source) self.assertNotIn('chown "$filter_cache"', source) + self.assertIn('browser_filter_delivery_copy_artifacts()', source) + self.assertIn('copy-artifacts "$WORK"', source) + self.assertNotIn('cp -r', source) if __name__ == "__main__": unittest.main() diff --git a/tests/integration/test-browser-filter-runner.py b/tests/integration/test-browser-filter-runner.py index 66d01c22a..a5f77e5c4 100644 --- a/tests/integration/test-browser-filter-runner.py +++ b/tests/integration/test-browser-filter-runner.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-only -"""Only shell argument/syntax checks; never start a VM, browser or service.""" +"""Inert argument, scratch-path and memory-gate checks; no VM/browser/service.""" from pathlib import Path +import stat import subprocess +import tempfile import unittest HERE = Path(__file__).resolve().parent @@ -46,6 +48,77 @@ def test_bundle_is_not_authorized_by_environment(self): self.assertIn("\nbrowser_filter_manifest_sha256=\n", source) self.assertIn('[ "$scenario" = content-provider ] || exit 64', source) + def directory_fragment(self): + source = (HERE / "run-alpha-topology-vm.sh").read_text() + return source.split("RUN_DIRECTORY_PARENT=/tmp\n", 1)[1].split("QEMU_PID=\n", 1)[0] + + def test_joined_browser_scratch_is_created_next_to_workspace_output_not_inside_it(self): + # Run only the actual path-selection/mktemp/chmod fragment, not the + # VM driver. This temporary directory itself is on the workspace disk. + with tempfile.TemporaryDirectory(prefix="filter-runner-test-", dir=HERE) as temporary: + parent = Path(temporary).resolve(); output = parent / "evidence"; output.mkdir() + script = ('set -eu\nscenario=content-provider\nbrowser_filter_bundle=/selected/bundle\n' + 'output_directory=$1\nRUN_DIRECTORY_PARENT=/tmp\n' + self.directory_fragment() + + 'printf "%s\\n" "$RUN_DIRECTORY"\n') + result = subprocess.run(["/bin/sh", "-c", script, "fixture", str(output)], capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + scratch = Path(result.stdout.decode().strip()) + self.assertEqual(scratch.parent, parent) + self.assertTrue(scratch.name.startswith("browser-filter-kvm.")) + self.assertNotEqual(scratch, output) + self.assertEqual(stat.S_IMODE(scratch.stat().st_mode), 0o700) + self.assertEqual(list(output.iterdir()), []) + + def test_joined_browser_rejects_symlink_output_and_memory_backed_parent(self): + with tempfile.TemporaryDirectory(prefix="filter-runner-test-", dir=HERE) as temporary: + parent = Path(temporary).resolve(); output = parent / "evidence"; output.mkdir() + alias = parent / "alias"; alias.symlink_to(output) + prefix = 'set -eu\nscenario=content-provider\nbrowser_filter_bundle=/selected/bundle\noutput_directory=$1\nRUN_DIRECTORY_PARENT=/tmp\n' + for path, filesystem, expected in ((alias, None, 64), (output, "tmpfs", 69), (output, "ramfs", 69), (output, "failed", 69)): + stub = '' if filesystem is None else ('stat() { return 1; }\n' if filesystem == "failed" + else 'stat() { printf "%s\\n" "' + filesystem + '"; }\n') + result = subprocess.run(["/bin/sh", "-c", prefix + stub + self.directory_fragment(), "fixture", str(path)], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertEqual({entry.name for entry in parent.iterdir()}, {"evidence", "alias"}) + + def test_ordinary_provider_and_existing_scratch_selection_unchanged(self): + # Stub mktemp/chmod so an ordinary /tmp-path check creates no files. + script = ('set -eu\nscenario=$1\nbrowser_filter_bundle=$2\noutput_directory=$3\nRUN_DIRECTORY_PARENT=/tmp\n' + 'mktemp() { printf "%s\\n" "${2%XXXXXX}abcdef"; }\nchmod() { :; }\n' + + self.directory_fragment() + 'printf "%s\\n" "$RUN_DIRECTORY"\n') + with tempfile.TemporaryDirectory(prefix="filter-runner-test-", dir=HERE) as temporary: + output = Path(temporary).resolve() / "evidence"; output.mkdir() + for scenario, bundle, prefix in (("content-provider", "", "/tmp/volparossa-alpha-kvm."), + ("alpha", "", "/tmp/volparossa-alpha-kvm."), + ("agent-native-editor", "", str(output.parent / "native-editor-kvm.")), + ("agent-cooperative-code", "", str(output.parent / "cooperative-code-kvm."))): + result = subprocess.run(["/bin/sh", "-c", script, "fixture", scenario, bundle, str(output)], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.decode().strip(), prefix + "abcdef") + + def test_memory_preflight_requires_original_four_gib_guest_plus_two_gib_reserve(self): + source = (HERE / "run-alpha-topology-vm.sh").read_text() + memory_function = "guest_memory_for_scenario() {\n" + source.split("guest_memory_for_scenario() {\n", 1)[1].split("\n}\n", 1)[0] + "\n}\n" + gate = "guest_memory_mib=$(guest_memory_for_scenario)\n" + source.split("guest_memory_mib=$(guest_memory_for_scenario)\n", 1)[1].split("qemu_usernet=", 1)[0] + script = ('set -eu\nscenario=$1\nbrowser_filter_bundle=$2\nfixture_available=$3\n' + 'awk() { printf "%s\\n" "$fixture_available"; }\n' + memory_function + gate + + 'printf "%s:%s\\n" "$guest_memory_mib" "$runner_reserve_mib"\n') + cases = (("content-provider", "bundle", 6144 * 1024 - 1, 69, None), + ("content-provider", "bundle", 6144 * 1024, 0, "4096:2048"), + ("content-provider", "bundle", "unknown", 69, None), + ("content-provider", "", 0, 0, "4096:1024"), + ("alpha", "", 0, 0, "4096:1024"), + ("agent-native-editor", "", 9216 * 1024 - 1, 69, None), + ("agent-native-editor", "", 9216 * 1024, 0, "8192:1024")) + for scenario, bundle, available, code, expected in cases: + with self.subTest(scenario=scenario, available=available): + result = subprocess.run(["/bin/sh", "-c", script, "fixture", scenario, bundle, str(available)], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, code, result.stderr) + if expected is not None: self.assertEqual(result.stdout.decode().strip(), expected) + if __name__ == "__main__": unittest.main() From e75f9231227b5400836a2693b4a20aefab67f24a Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:38:45 +0200 Subject: [PATCH 03/49] test: diagnose joined browser failures and prepare namespace access --- docs/IMPLEMENTATION_STATUS.md | 36 ++++++- tests/integration/browser-filter-delivery.py | 35 +++++-- tests/integration/kvm-alpha-topology.sh | 23 +++++ tests/integration/run-alpha-topology-vm.sh | 55 +++++++++++ .../test-browser-filter-delivery.py | 30 ++++++ .../test-browser-filter-netns-parent.py | 93 +++++++++++++++++++ .../integration/test-browser-filter-runner.py | 90 ++++++++++++++++++ 7 files changed, 352 insertions(+), 10 deletions(-) create mode 100644 tests/integration/test-browser-filter-netns-parent.py diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 892d06159..b43f7c4d4 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -117,8 +117,8 @@ revocation and sticky opt-out. Only an explicitly supplied, hash-verified bundle can enable it. The browser and broker run as the disposable worker in nested namespaces; neither receives permission to read the agent cache directly. -The Browser fixture passes 84 focused JavaScript checks and 24 Python checks; -12 core transfer checks, nine runner argument/resource checks and 15 joined-proof +The Browser fixture passes 84 focused JavaScript checks and 34 Python checks; +12 core transfer checks, thirteen runner argument/resource checks and 17 joined-proof and receipt-export checks also pass. These are preparatory component checks, **not a passed combined runtime trial**. The hook must still prove real control access after namespace mapping, end-to-end browser @@ -139,8 +139,36 @@ evidence, rejects a tmpfs/ramfs parent and requires available host memory for th unchanged 4-GiB guest plus a 2-GiB reserve before launch. This is a preflight snapshot, not a memory reservation. A fixed JSON-only export retains original receipts and their hashes, including failures, without exporting private logs, -profiles or authority inputs. The corrected combined runtime trial remains -pending. +profiles or authority inputs. + +The second local attempt on `2e4f4527d669b104ed56d87184bb11ab73ccabe6`, +using the same Browser input manifest, completed the guest build and reached the +capability-free Client-namespace launcher, but failed with +`BROWSER_FILTER_NATIVE_FAILED`. The original launch, source-inventory and input +cleanup receipts were retained; neither the Browser report nor its outer-cleanup +report was produced. Those missing reports do not establish whether failure +occurred during preflight, execution or cleanup. This is a failed combined trial, +not proof of a working Firefox-to-broker path. Evidence remains in the local +`browser-filter-delivery-02.yaD4ng` collection. The VM and owned overlay were +removed. A15 verifies unchanged disposable-guest root networking outside the +owned namespaces, with matching before/after hash +`13e338046f6a7e418c119f377258cc98664eb005cf89d8a5b6ff8aa8d44f6824`. + +Failure diagnosis now retains only a closed phase code and cleanup-failure flag, +including failures before Browser output preparation. The emergency collector +uses the same root-staged JSON validator and preserves incomplete status; it +never exports raw Browser errors, logs or temporary authority. These additions +improve the next trial's evidence, not the outcome of the failed second attempt. + +The guest recipe also had a namespace-parent permission defect: `ip netns add` +could create `/run/netns` under `umask 077`, preventing the capability-free +Browser worker from checking its own namespace identity. The opt-in fixture now +creates a missing parent explicitly as root-owned `0755` and preserves an already +safe parent; unsafe existing paths are rejected rather than changed. Six inert +tests pass. This fixes the fresh-guest recipe, but the missing diagnostic from +attempt two cannot prove it was that attempt's first failure. Browser namespace +validation and private agent-cache permissions remain unchanged. A successful +combined runtime trial is still required. ### Bank and Transaction layer implementation diff --git a/tests/integration/browser-filter-delivery.py b/tests/integration/browser-filter-delivery.py index 9c98688db..99c3b9cde 100644 --- a/tests/integration/browser-filter-delivery.py +++ b/tests/integration/browser-filter-delivery.py @@ -27,6 +27,13 @@ cold_cache_proven=False, native_admission_registered=False, production_default_enrollment=False, startup_resume_stale_filter_barrier=False, real_os_expiry=False) MAX_REPORT = 65536 +DIAGNOSTIC_PHASES = { + "arguments", "stage", "packages", "sources", "expected", "binary", "control", + "topology_guest", "topology_init", "topology_root", "topology_unit", "topology_namespace", "topology_peer", + "output_path", "output_prepare", "binary_copy", "wrapper_inputs", "home", "wrapper", + "cleanup_profile", "cleanup_binary", "cleanup_receipt", +} +DIAGNOSTIC_KEYS = {"schema", "success", "phase", "cleanup_failed"} REPORT_KEYS = {"schema", "passed", "scope", "operation", "receipts", "probes", "module_sha256", "driver_sha256", "broker_sha256", "expected", "runtime_version", "runtime_source_stamp", "runtime_sha256", "extension_packages", "failure", "host_read_only", "interfaces", "browser_exit_status", "broker_cleanup", "broker_socket_removed", @@ -34,6 +41,7 @@ EXPORT_KEYS = { "original-report": REPORT_KEYS, "original-outer-cleanup": {"schema", "temporary_profiles_removed", "broker_copy_removed", "logs"}, + "original-diagnostic": DIAGNOSTIC_KEYS, "evidence": {"version", "kind", "success", "bundle_manifest_sha256", "core_broker_sha256", "originals_sha256", "publication", "protected_cold", "browser", "scope", "cleanup"}, "prepared": {"version", "expected", "created_unix_seconds", "maximum_authority_seconds"}, @@ -50,7 +58,7 @@ "preserved", "freshReload", "reloadEvents", "delivery", "asset", "broker_revoked", "bytes", "sha256", "rules", "manifest_id", "publisher_key", "name", "authority_expires_unix_seconds", "accepted", "completed", "denied", "ok", "phase", "ads", "tracker", "neutral", "started", "reaped", "graceful", "broker", "id", "size", - "unpacked_bytes", "zip_entries", "report", "outer-cleanup", "content_type", "peer_bytes", "providers_used", + "unpacked_bytes", "zip_entries", "report", "outer-cleanup", "diagnostic", "content_type", "peer_bytes", "providers_used", "privacy_captures_validated", "exit_status", "same_publication_joined", "broker_reuse_requested", "broker_cache_only_proven", "new_broker_peer_transfer_proven", "two_subflow_bulk_aggregation", "browser_profile_removed", "broker_reaped", "fixture_authority_removed", "separate_owner_input_removed", "agent_cache_identity_preserved", "agent_cache_private", @@ -65,7 +73,8 @@ "fixture_other", "invalid_request", "duplicate_request", "handshake_required", "busy", "unavailable", "revoked", "clock_error", "expired", "invalid_snapshot", "invalid_response", "not_configured", "owner_config", "owner_context", "owner_clock", "owner_busy", "owner_closed", "owner_failed", "owner_cleanup", "asset_config", "asset_package", - "asset_context", "asset_authority", "asset_expired", "asset_clock", "asset_busy", "asset_closed", "asset_failed", "asset_cleanup"} + "asset_context", "asset_authority", "asset_expired", "asset_clock", "asset_busy", "asset_closed", "asset_failed", "asset_cleanup", + *DIAGNOSTIC_PHASES} def require(condition): @@ -109,6 +118,13 @@ def write_new(path, raw): stream.write(raw); stream.flush(); os.fsync(stream.fileno()) +def validate_diagnostic(value): + require(type(value) is dict and set(value) == DIAGNOSTIC_KEYS + and type(value["schema"]) is int and value["schema"] == 1 + and value["success"] is False and type(value["cleanup_failed"]) is bool + and type(value["phase"]) is str and value["phase"] in DIAGNOSTIC_PHASES) + + def private_root(root): require(root.name == "browser-filter-delivery" and root.resolve(strict=True) == root) info = root.lstat() @@ -141,7 +157,7 @@ def cleanup(root): require(root.name == "browser-filter-delivery" and not root.is_symlink()) if not root.exists(): return dict(directory_removed=True, files_removed=True) private_root(root) - require({path.name for path in root.iterdir()} <= {"selection.json", "expected.json", "home"}) + require({path.name for path in root.iterdir()} <= {"selection.json", "expected.json", "diagnostic.json", "home"}) files = [] for path in root.iterdir(): info = path.lstat() @@ -151,6 +167,7 @@ def cleanup(root): else: require(stat.S_ISREG(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o400 and info.st_nlink == 1 and info.st_size <= 1024) + if path.name == "diagnostic.json": validate_diagnostic(read(path)) files.append(path) for path in files: path.unlink() if (root / "home").exists(): (root / "home").rmdir() @@ -183,7 +200,8 @@ def launch(arguments): os.execv(sys.executable, [sys.executable, "-B", str(source / "scripts/smoke_filter_delivery.py"), "--execute", "--stage", str(source / "build/firefox-filter-runtime"), "--output", str(source / "build/filter-delivery"), "--broker", binary, "--broker-sha256", binary_sha, - "--control-socket", control_socket, "--agent-cache", cache, "--expected", str(root / "expected.json")]) + "--control-socket", control_socket, "--agent-cache", cache, "--expected", str(root / "expected.json"), + "--diagnostic-output", str(root / "diagnostic.json")]) def cache_state(path, uid, gid): @@ -258,12 +276,14 @@ def retain(work): """Preserve original closed receipts on failure too; absence is not success.""" output = work / "client-fixtures/browser-filter-source/build/filter-delivery" retained = {} - for name, keys in (("report", REPORT_KEYS), ("outer-cleanup", {"schema", "temporary_profiles_removed", "broker_copy_removed", "logs"})): - path = output / (name + ".json") + for name, keys in (("report", REPORT_KEYS), ("outer-cleanup", {"schema", "temporary_profiles_removed", "broker_copy_removed", "logs"}), + ("diagnostic", DIAGNOSTIC_KEYS)): + path = (work / "client-fixtures/browser-filter-delivery" if name == "diagnostic" else output) / (name + ".json") if not path.exists() and not path.is_symlink(): continue raw = raw_file(path) value = json.loads(raw, object_pairs_hook=BUNDLE["unique_pairs"]) require(type(value) is dict and set(value) <= keys and type(value.get("schema")) is int and value["schema"] == 1) + if name == "diagnostic": validate_diagnostic(value) target = work / ("browser-filter-delivery-original-" + name + ".json") if target.exists(): require(raw_file(target) == raw) else: write_new(target, raw) @@ -307,6 +327,7 @@ def copy_artifacts(work, output, uid, gid): require({"schema", "passed", "failure"} <= set(value) and type(value["schema"]) is int and value["schema"] == 1 and type(value["passed"]) is bool) else: require(set(value) == keys) + if suffix == "original-diagnostic": validate_diagnostic(value) export_value(value) pending[name] = raw result = dict(version=1, originals={name: dict(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) @@ -322,6 +343,8 @@ def copy_artifacts(work, output, uid, gid): def evidence(work, manifest_sha, binary_sha, uid, original_gid, control_gid): + diagnostic = work / "browser-filter-delivery-original-diagnostic.json" + require(not diagnostic.exists() and not diagnostic.is_symlink()) source = work / "client-fixtures/browser-filter-source" output = source / "build/filter-delivery" manifest_raw = raw_file(source / "INPUTS.json") diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 4e0a6f57a..b1b18b8e9 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -3064,6 +3064,29 @@ mpquic_binary=$WORK/bin/volparossa-mpquic [ "$scenario" != agent-artifact ] || agent_artifact_prepare [ "$scenario" != agent-jobs ] || agent_jobs_prepare +# BEGIN browser-filter namespace-parent preflight +if [ -n "$browser_filter_bundle" ]; then + PHASE=browser-filter-netns-parent + printf '%s\n' 'Browser fixture: create /run/netns root:root mode 0755 only if absent; preserve a safe existing parent; never change private cache permissions.' + # This is after the root/Debian/systemd/KVM gates, inside the disposable + # guest only. ip netns add otherwise inherits umask 077 when making its + # runtime parent, preventing the capless worker from statting its namespace. + # An explicit mkdir mode applies only to our new directory. The runtime + # parent, like iproute2's own parent mount, lasts until this VM is discarded. + [ ! -L /run/netns ] || fail BROWSER_FILTER_NETNS_PARENT_UNSAFE + if [ ! -e /run/netns ]; then + mkdir -m 0755 -- /run/netns || fail BROWSER_FILTER_NETNS_PARENT_UNAVAILABLE + fi + if [ ! -d /run/netns ] || [ "$(readlink -e -- /run/netns)" != /run/netns ]; then + fail BROWSER_FILTER_NETNS_PARENT_UNSAFE + fi + case $(stat -Lc '%u:%g:%a' /run/netns) in + 0:0:755|0:0:711) ;; + *) fail BROWSER_FILTER_NETNS_PARENT_UNSAFE ;; + esac +fi +# END browser-filter namespace-parent preflight + PHASE=network-topology for namespace in "$CLIENT" "$B1" "$B2" "$R0" "$R1" "$R2" "$R3" "$R4" "$R5" \ "$EXIT_NODE" "$EXIT2_NODE" "$DEST"; do diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 4ece1e606..9d1b9833f 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1062,6 +1062,49 @@ COOPERATIVE_CODE_PROPOSAL_NAMES |= { "content-provider-custody-executor-discovery-control.json", } | {f"content-custody-executor-discovery-privacy-{role}.json" for role in ("client", "relay0", "relay1", "relay2", "exit")} +BROWSER_FILTER_NAMES = {f"browser-filter-delivery-{suffix}.json" for suffix in ( + "original-report", "original-outer-cleanup", "original-diagnostic", "evidence", "prepared", "cache-before", "cache-after", + "launch", "tree-before", "tree-after", "input-cleanup", "retained", "artifacts")} +BROWSER_FILTER_HELPERS = ("browser-filter-delivery.py", "browser-filter-bundle.py", + "content-provider-filter-smoke.py", "content-provider-site-smoke.py", + "content-network-smoke.py", "content-provider-https-smoke.py") + + +def browser_filter_helper(work): + """Trust only the topology's immutable root-staged validation closure.""" + try: + directory = work / "bin" + if directory.resolve(strict=True) != directory: + return None + for parent in (directory, *directory.parents): + info = parent.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: + return None + for name in BROWSER_FILTER_HELPERS: + path = directory / name + info = path.lstat() + if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_nlink != 1 + or info.st_mode & 0o022 or not 0 < info.st_size <= 262144): + return None + return directory / "browser-filter-delivery.py" + except OSError: + return None + + +def browser_filter_candidates(root, label, helper, target): + # Reuse the closed recursive JSON validator, never import the worker's + # Browser tree or copy its logs. A failed/missing validator yields no files. + validated = Path(tempfile.mkdtemp(prefix="browser-filter-validated.", dir=target)) + try: + subprocess.run(["/usr/bin/python3", "-I", "-B", str(helper), "copy-artifacts", str(root), + str(validated), str(os.getuid()), str(os.getgid())], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + cwd="/", env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, timeout=3, check=True) + except (OSError, subprocess.SubprocessError): + return [] + return [(validated / name, f"{label}/{name}") for name in sorted(BROWSER_FILTER_NAMES) + if (validated / name).is_file() and not (validated / name).is_symlink()] + def read_tail(path, limit=FILE_LIMIT): try: @@ -1163,6 +1206,16 @@ def collect(home, opt, revision, scenario, guest_status, roots.append((path, f"work-{len(roots)}")) if len(roots) == 5: break + if scenario == "content-provider": + # Prioritize already validated exact Browser receipts before generic + # diagnostics consume the existing file-count/byte limits. This is + # recovery evidence only, even when an inner receipt says passed=true. + helper = next((found for root, _ in roots[1:] + if (found := browser_filter_helper(root)) is not None), None) + if helper is not None: + for root, label in roots: + if root.is_dir() and not root.is_symlink(): + candidates.extend(browser_filter_candidates(root, label, helper, target)) for root, label in roots: if root.is_symlink() or not root.is_dir(): continue @@ -1329,6 +1382,8 @@ def collect(home, opt, revision, scenario, guest_status, if observed is None: continue data, original_size = observed + if path.name in BROWSER_FILTER_NAMES and original_size > len(data): + continue # Preserve complete validated original bytes, never a JSON tail. if scenario == "browser-network" and path.name in browser_names and original_size > len(data): # A truncated JSON tail is not reconstructible structured evidence. continue diff --git a/tests/integration/test-browser-filter-delivery.py b/tests/integration/test-browser-filter-delivery.py index 5d0088b43..78622ff47 100644 --- a/tests/integration/test-browser-filter-delivery.py +++ b/tests/integration/test-browser-filter-delivery.py @@ -151,6 +151,36 @@ def test_closed_file_reader_refuses_duplicate_json_keys_symlinks_and_oversize(se path.write_bytes(b"a" * 65537) with self.assertRaises(ValueError): CHECK["raw_file"](path) + def test_early_diagnostic_retained_without_driver_report_exported_and_cleaned(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve(); root = work / "client-fixtures/browser-filter-delivery" + root.mkdir(parents=True, mode=0o700) + raw = b'{"schema":1,"success":false,"phase":"topology_namespace","cleanup_failed":false}' + CHECK["write_new"](root / "diagnostic.json", raw) + result = CHECK["retain"](work) + self.assertEqual(result["originals_sha256"], {"diagnostic": hashlib.sha256(raw).hexdigest()}) + self.assertFalse(result["successful_native_execution_claimed"]) + self.assertEqual(CHECK["retain"](work), result) + output = work / "output"; output.mkdir() + exported = CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertIn("original-report", exported["missing"]) + self.assertEqual((output / "browser-filter-delivery-original-diagnostic.json").read_bytes(), raw) + CHECK["cleanup"](root) + self.assertFalse(root.exists()) + + def test_diagnostic_schema_is_closed_and_never_a_success_receipt(self): + good = dict(schema=1, success=False, phase="binary", cleanup_failed=True) + for phase in CHECK["DIAGNOSTIC_PHASES"]: + CHECK["validate_diagnostic"](dict(good, phase=phase)) + for change in (dict(schema=True), dict(success=True), dict(cleanup_failed=1), dict(phase="PRIVATE"), + dict(phase=["binary"]), dict(error="PRIVATE")): + with self.subTest(change=change), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "browser-filter-delivery"; root.mkdir(mode=0o700) + CHECK["write_new"](root / "selection.json", b"keep") + CHECK["write_new"](root / "diagnostic.json", json.dumps(dict(good, **change)).encode()) + with self.assertRaises(ValueError): CHECK["cleanup"](root) + self.assertTrue((root / "selection.json").exists()) + def test_artifact_export_preserves_bytes_hash_and_omits_private_files(self): with tempfile.TemporaryDirectory() as temporary: work, output = Path(temporary) / "work", Path(temporary) / "output" diff --git a/tests/integration/test-browser-filter-netns-parent.py b/tests/integration/test-browser-filter-netns-parent.py new file mode 100644 index 000000000..5c87cecf6 --- /dev/null +++ b/tests/integration/test-browser-filter-netns-parent.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert runtime-parent checks; no namespace, mount, service or VM operations.""" +from pathlib import Path +import stat +import subprocess +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +SOURCE = (HERE / "kvm-alpha-topology.sh").read_text() +BEGIN = "# BEGIN browser-filter namespace-parent preflight\n" +END = "# END browser-filter namespace-parent preflight\n" +FRAGMENT = SOURCE.split(BEGIN, 1)[1].split(END, 1)[0] + + +class BrowserFilterNamespaceParent(unittest.TestCase): + def run_fragment(self, path, enabled=True, owner="0:0"): + # Only ownership is represented by an inert stub: the test account need + # not be root. mkdir, permissions, links and canonical paths are real, + # under this private temporary fixture, never /run/netns. + script = ('set -eu\numask 077\nbrowser_filter_bundle=$1\nfixture_owner=$2\n' + 'fail() { printf "%s\\n" "$1" >&2; exit 69; }\n' + 'stat() { test "$1" = -Lc; test "$2" = "%u:%g:%a"; ' + 'printf "%s:%s\\n" "$fixture_owner" "$(command stat -Lc "%a" "$3")"; }\n' + + FRAGMENT.replace("/run/netns", str(path))) + return subprocess.run(["/bin/sh", "-c", script, "fixture", "selected" if enabled else "", owner], + capture_output=True, timeout=5) + + def test_new_parent_is_searchable_despite_private_umask(self): + with tempfile.TemporaryDirectory() as temporary: + parent = Path(temporary).resolve() / "netns" + result = self.run_fragment(parent) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(stat.S_IMODE(parent.stat().st_mode), 0o755) + self.assertEqual(list(parent.iterdir()), []) + + def test_safe_existing_parent_and_entries_are_unchanged(self): + for mode in (0o755, 0o711): + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as temporary: + parent = Path(temporary).resolve() / "netns"; parent.mkdir(); parent.chmod(mode) + entry = parent / "pre-existing"; entry.write_bytes(b"untouched fixture") + before = parent.stat() + result = self.run_fragment(parent) + after = parent.stat() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((after.st_dev, after.st_ino, after.st_mode, after.st_mtime_ns), + (before.st_dev, before.st_ino, before.st_mode, before.st_mtime_ns)) + self.assertEqual(entry.read_bytes(), b"untouched fixture") + + def test_private_writable_or_foreign_parent_is_rejected_without_chmod(self): + for mode, owner in ((0o700, "0:0"), (0o775, "0:0"), (0o777, "0:0"), (0o755, "1000:1000")): + with self.subTest(mode=mode, owner=owner), tempfile.TemporaryDirectory() as temporary: + parent = Path(temporary).resolve() / "netns"; parent.mkdir(); parent.chmod(mode) + before = parent.stat() + result = self.run_fragment(parent, owner=owner) + self.assertEqual(result.returncode, 69) + self.assertIn(b"BROWSER_FILTER_NETNS_PARENT_UNSAFE", result.stderr) + self.assertEqual(parent.stat().st_mode, before.st_mode) + self.assertEqual(parent.stat().st_ino, before.st_ino) + + def test_symlink_and_regular_file_are_rejected_without_changes(self): + for kind in ("symlink", "file"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve(); parent = root / "netns" + if kind == "symlink": + target = root / "target"; target.mkdir(); target.chmod(0o700); parent.symlink_to(target) + else: parent.write_bytes(b"untouched") + before = parent.lstat() + result = self.run_fragment(parent) + self.assertEqual(result.returncode, 69) + self.assertEqual(parent.lstat().st_mode, before.st_mode) + self.assertEqual(parent.lstat().st_ino, before.st_ino) + if kind == "symlink": self.assertEqual(stat.S_IMODE(target.stat().st_mode), 0o700) + else: self.assertEqual(parent.read_bytes(), b"untouched") + + def test_without_explicit_browser_bundle_nothing_is_created(self): + with tempfile.TemporaryDirectory() as temporary: + parent = Path(temporary).resolve() / "netns" + result = self.run_fragment(parent, enabled=False) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(parent.exists()) + + def test_preflight_is_after_disposable_guest_gates_and_before_namespace_creation(self): + position = SOURCE.index(BEGIN) + self.assertLess(SOURCE.index('[ "$(id -u)" -eq 0 ]'), position) + self.assertLess(SOURCE.index('[ "$(systemd-detect-virt)" = kvm ]'), position) + self.assertLess(position, SOURCE.index('ip netns add "$namespace"')) + self.assertNotIn("chmod", FRAGMENT) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-browser-filter-runner.py b/tests/integration/test-browser-filter-runner.py index a5f77e5c4..435b872af 100644 --- a/tests/integration/test-browser-filter-runner.py +++ b/tests/integration/test-browser-filter-runner.py @@ -2,10 +2,13 @@ # SPDX-License-Identifier: GPL-3.0-only """Inert argument, scratch-path and memory-gate checks; no VM/browser/service.""" from pathlib import Path +import json import stat import subprocess +import tarfile import tempfile import unittest +from unittest.mock import patch HERE = Path(__file__).resolve().parent RUNNERS = ("run-alpha-topology-vm.sh", "kvm-alpha-topology.sh") @@ -120,5 +123,92 @@ def test_memory_preflight_requires_original_four_gib_guest_plus_two_gib_reserve( if expected is not None: self.assertEqual(result.stdout.decode().strip(), expected) +class BrowserFilterEmergencyExport(unittest.TestCase): + @classmethod + def setUpClass(cls): + source = (HERE / "run-alpha-topology-vm.sh").read_text() + body = source.split("<<'GUEST_DIAGNOSTICS_PYTHON'\n", 1)[1].split("\nGUEST_DIAGNOSTICS_PYTHON\n", 1)[0] + cls.api = {"__name__": "inert_guest_diagnostics_fixture"} + exec(compile(body, "guest-diagnostics-fixture", "exec"), cls.api) + + def run_collector(self, temporary, raw, diagnostic=None): + base = Path(temporary).resolve() + home, opt, proc, cgroups = (base / name for name in ("home", "opt", "proc", "cgroups")) + for directory in (home, opt, proc, cgroups): directory.mkdir() + (home / "alpha-output").mkdir() + work = opt / ("va." + "a" * 32 + ".ABC123"); work.mkdir() + report = work / "browser-filter-delivery-original-report.json" + report.write_bytes(raw); report.chmod(0o600) + if diagnostic is not None: + path = work / "browser-filter-delivery-original-diagnostic.json" + path.write_bytes(diagnostic); path.chmod(0o400) + # These are never candidates, even though their basename is related. + (work / "browser-filter-delivery.err").write_text("private raw log") + (work / "browser-filter-delivery-private.json").write_text('{"private_code":"secret"}') + (work / "client-fixtures").mkdir() + (work / "client-fixtures/expected.json").write_text('{"private_key":"secret"}') + # Only the immutable-root staging gate is replaced for this unprivileged + # unit fixture. The actual closed production JSON validator still runs. + with patch.dict(self.api, {"browser_filter_helper": lambda root: HERE / "browser-filter-delivery.py"}): + archive = self.api["collect"](home, opt, "b" * 40, "content-provider", 124, cgroups, proc) + with tarfile.open(archive, "r:gz") as bundle: + return {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()} + + def test_validated_original_bytes_survive_without_claiming_acceptance(self): + raw = b'{ "schema": 1, "passed": true, "failure": null }\n' + with tempfile.TemporaryDirectory() as temporary: + files = self.run_collector(temporary, raw) + name = "work-1/browser-filter-delivery-original-report.json" + self.assertEqual(files[name], raw) + summary = json.loads(files["vm-incomplete.json"]) + self.assertFalse(summary["success"]) + self.assertEqual(summary["cleanup"], {"complete": False, "verified": False}) + self.assertEqual(summary["observed_blocker"], "GUEST_EXECUTION_TIMEOUT") + receipt = next(item for item in summary["diagnostics"]["files"] if item["file"] == name) + self.assertEqual(receipt["captured_bytes"], len(raw)) + self.assertFalse(receipt["truncated"]) + self.assertFalse(any("private" in name or name.endswith(".err") for name in files)) + manifest = json.loads(files["work-1/browser-filter-delivery-artifacts.json"]) + self.assertFalse(manifest["successful_native_execution_claimed"]) + + def test_private_unknown_nested_and_duplicate_fields_are_not_exported(self): + cases = ( + b'{"schema":1,"passed":true,"failure":null,"private_code":"secret"}', + b'{"schema":1,"passed":true,"failure":null,"logs":{"browser":{"private_code":"secret"}}}', + b'{"schema":1,"passed":true,"failure":null,"logs":{"browser":"/private/file"}}', + b'{"schema":1,"passed":true,"passed":false,"failure":null}', + b'{"schema":1,"passed":true,"failure":null} trailing', + b" " * 65537, + ) + for raw in cases: + with self.subTest(raw=raw[:90]), tempfile.TemporaryDirectory() as temporary: + files = self.run_collector(temporary, raw) + self.assertNotIn("work-1/browser-filter-delivery-original-report.json", files) + self.assertNotIn("work-1/browser-filter-delivery-artifacts.json", files) + self.assertFalse(json.loads(files["vm-incomplete.json"])["success"]) + + def test_untrusted_helper_cannot_be_executed(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve() + (work / "bin").mkdir() + for name in self.api["BROWSER_FILTER_HELPERS"]: + path = work / "bin" / name + path.write_text("raise RuntimeError('must not execute')\n") + path.chmod(0o555) + # A writable staging directory is rejected even when tests run as root. + (work / "bin").chmod(0o777) + self.assertIsNone(self.api["browser_filter_helper"](work)) + + def test_closed_failure_phase_survives_emergency_export_without_private_errors(self): + diagnostic = b'{"schema":1,"success":false,"phase":"topology_namespace","cleanup_failed":false}' + with tempfile.TemporaryDirectory() as temporary: + files = self.run_collector(temporary, b'', diagnostic) + self.assertEqual(files["work-1/browser-filter-delivery-original-diagnostic.json"], diagnostic) + self.assertFalse(json.loads(files["vm-incomplete.json"])["success"]) + with tempfile.TemporaryDirectory() as temporary: + files = self.run_collector(temporary, b'', diagnostic.replace(b'topology_namespace', b'private_value')) + self.assertNotIn("work-1/browser-filter-delivery-original-diagnostic.json", files) + + if __name__ == "__main__": unittest.main() From 7a039f38d018795507e19652811197e31f0d6d1c Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:46:03 +0200 Subject: [PATCH 04/49] docs: record first merged Citizen simulation and its limits --- README.md | 1 + docs/IMPLEMENTATION_STATUS.md | 19 +++++++++++++++++++ docs/applications/APPLICATIONS.md | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/README.md b/README.md index b444ebfb0..a375a584d 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,7 @@ and compute coordination belongs in the core. | [Weather](https://github.com/VOLPAROSSA/volparossa-weather) | Direct public model data | Shared weather processing and measured forecast improvement. | | [Cloud](https://github.com/VOLPAROSSA/volparossa-cloud) | OpenCloud | Private files, synchronization and server-independent access. | | [Bank](https://github.com/VOLPAROSSA/volparossa-bank) | Financial protocol research | Participant-owned portfolios, payments and auditable corrections; no real-money service. | +| [Citizen](https://github.com/VOLPAROSSA/volparossa-citizen) | Participation and voting research | Proposals and referenda alongside existing institutions; synthetic procedure simulation first. | [Application scope and current boundaries](docs/applications/APPLICATIONS.md) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index b43f7c4d4..2aa8b5b22 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -170,6 +170,25 @@ attempt two cannot prove it was that attempt's first failure. Browser namespace validation and private agent-cache permissions remain unchanged. A successful combined runtime trial is still required. +### Citizen procedure simulation + +Citizen [PR1](https://github.com/VOLPAROSSA/volparossa-citizen/pull/1) is merged +in its own `main` at `43ef0ce916dd7ba0542710bc90705acdd4d2dd16`. +The dependency-free executable uses synthetic identities and an immutable +all-eligible electorate, distinct from registered accounts and turnout. +One-third support starts voting; two-thirds yes adopts, two-thirds no rejects, +and other results are indeterminate. An indeterminate result permits an explicit +new procedure after three calendar years, with a fresh electorate snapshot and +renewed support. UTC, closing-time and leap-day conventions are documented +simulation assumptions, not new confirmed policy. + +Twenty-one independently rerun tests pass, including the actual deterministic +CLI demo, rounding boundaries, duplicates, ineligible participation, time +windows and repeat-state preservation. This is a single-process reference +workflow, not real-person authentication, secret voting, cryptographic tally +verification, distributed recovery, core integration or legal authority. +Those remain incomplete. See the [application overview](applications/APPLICATIONS.md#citizen). + ### Bank and Transaction layer implementation The 2026-10-04 user extension adds VOLPAROSSA Bank and a reusable Transaction-layer. diff --git a/docs/applications/APPLICATIONS.md b/docs/applications/APPLICATIONS.md index 7d80b7c59..e8e288ee7 100644 --- a/docs/applications/APPLICATIONS.md +++ b/docs/applications/APPLICATIONS.md @@ -116,6 +116,25 @@ financial authorization, consensus, gateway settlement and dispute governance remain required work. The immune system supplies bounded analysis, not unilateral financial authority or a guarantee that dispersed funds can be recovered. +## Citizen + +[volparossa-citizen](https://github.com/VOLPAROSSA/volparossa-citizen) targets +proposals, deliberation and referenda alongside existing democratic institutions. +The application owns its participation rules; transport, storage and compute +should reuse the core. + +The first executable simulation is merged in [PR1](https://github.com/VOLPAROSSA/volparossa-citizen/pull/1). +It freezes all eligible fictional people separately from registered accounts: +one-third support opens a referendum, two-thirds yes adopts it and two-thirds +no rejects it. Other outcomes remain indeterminate and permit an explicit repeat +procedure after three calendar years. Twenty-one tests pass. The documented UTC, +calendar and renewed-support details are simulation assumptions, not additional +confirmed governance rules. + +Real identity verification and uniqueness, secret verifiable voting, contestable +proposal review, core delivery and recovery remain open. A synthetic result is +neither a public-election security proof nor recognized legal authority. + ## Shared boundaries - Public cache, private retained storage and encrypted message custody have different From 93aeb30772522681b8c272a18ea45b24809361ef Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:54:30 +0200 Subject: [PATCH 05/49] docs: retain third joined trial route-selection failure --- docs/IMPLEMENTATION_STATUS.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 2aa8b5b22..cc80655dd 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -170,6 +170,19 @@ attempt two cannot prove it was that attempt's first failure. Browser namespace validation and private agent-cache permissions remain unchanged. A successful combined runtime trial is still required. +The third local attempt on `e75f9231227b5400836a2693b4a20aefab67f24a`, +with Browser input manifest +`26f0716f2b5e75cf3cec0ba145b8d6e9a0b4b6583051aee374c22e1d87d5b8d2`, +passed guest build and namespace-parent preparation but stopped earlier in the +provider scenario at `CONTENT_PROVIDER_MPTCP_SELECTION_UNAVAILABLE`. No content +transfer or Browser hook ran, so this attempt cannot verify the Browser fix or +its new failure diagnostics. The retained `browser-filter-delivery-03.gqbzoq` +collection records zero remaining owned objects and unchanged guest-root +networking, with matching hash +`abc03c0e46ecc703847d916133d7288ff6efdd22ee8dc88d62afcec308040f0e`. +The VM and disposable overlay were removed. Provider route selection must be +diagnosed before another joined run; the Browser integration remains incomplete. + ### Citizen procedure simulation Citizen [PR1](https://github.com/VOLPAROSSA/volparossa-citizen/pull/1) is merged From e2aa069052f8181fb74f54645507426248a632cb Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:37:34 +0200 Subject: [PATCH 06/49] test: await provider discovery inventory before route selection --- docs/IMPLEMENTATION_STATUS.md | 31 ++- .../integration/content-provider-readiness.py | 169 +++++++++++++++ tests/integration/content-provider-smoke.sh | 5 + tests/integration/kvm-alpha-topology.sh | 8 +- .../test-content-provider-readiness.py | 198 ++++++++++++++++++ 5 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 tests/integration/content-provider-readiness.py create mode 100644 tests/integration/test-content-provider-readiness.py diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index cc80655dd..684d498fd 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -180,8 +180,35 @@ its new failure diagnostics. The retained `browser-filter-delivery-03.gqbzoq` collection records zero remaining owned objects and unchanged guest-root networking, with matching hash `abc03c0e46ecc703847d916133d7288ff6efdd22ee8dc88d62afcec308040f0e`. -The VM and disposable overlay were removed. Provider route selection must be -diagnosed before another joined run; the Browser integration remains incomplete. +The VM and disposable overlay were removed. The retained Client inventory had +R0/R1/R2 and only Exit2, whose fixture capacity is intentionally below the required +route capacity; the primary Exit advertisement was missing. This explains the +terminal `NO_ELIGIBLE_PATHS` result, but does not yet distinguish delayed discovery +from a discovery defect. + +The provider scenario now waits at most 60 seconds for the exact R0/R1/R2 and +primary Exit identities and roles in the Client's read-only advertisement +inventory. Fourteen synthetic tests cover partial and malformed inventories, +role substitutions, deadline handling and the closed diagnostic receipt. The +original route selector, capacity requirements and terminal `NO_ELIGIBLE_PATHS` +handling are unchanged. Inventory convergence is not proof of route readiness; +another real combined runtime trial is required. Browser integration remains +incomplete. + +### Shared Weather application provider + +Weather [PR5](https://github.com/VOLPAROSSA/volparossa-weather/pull/5) is merged +in its own `main` at `c14042a75b77608fcc968f9eabb93e015f87c0aa`. +The executable provider contract exposes temperature and wind on the original +model time axis, retaining source evidence and the earliest forecast or signed +publication expiry. Its core-cache adapter reuses the existing authenticated +fetch path without an implicit origin or Open-Meteo fallback. All 33 focused +tests and the repository CI pass. + +This is the shared data interface, not yet rendering in Drizz.li or Overmorrow +or a live Weather-overlay proof. Application transport and UI adapters remain +required. Precipitation and air quality also remain required work; unsupported +fields are explicitly unavailable, never substituted with zero values. ### Citizen procedure simulation diff --git a/tests/integration/content-provider-readiness.py b/tests/integration/content-provider-readiness.py new file mode 100644 index 000000000..462f41114 --- /dev/null +++ b/tests/integration/content-provider-readiness.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Bounded Client advertisement inventory barrier, not route readiness evidence. + +Only the existing Client's read-only peers command is queried. No advertisements, +credentials, routes, role changes or discovery records are supplied to the agent. +""" +import json +import os +from pathlib import Path +import re +import socket +import stat +import subprocess +import sys +import time + +FIXTURE_ROLES = ("client", "bootstrap1", "bootstrap2", *(f"relay{i}" for i in range(6)), "exit", "exit2") +INVENTORY_ROLES = ("relay0", "relay1", "relay2", "exit") +SCOPE = "client_advertisement_inventory_not_route_readiness" +DEADLINE_SECONDS = 60 +MAX_ATTEMPTS = 600 +OUTCOMES = {"none", "timeout", "nonzero", "invalid_reply", "partial", "complete", "os_error"} +TERMINAL = {"ready", "deadline", "os_error", "invalid_fixture"} + + +def require(condition): + if not condition: + raise ValueError("content_provider_inventory_rejected") + + +def unique_pairs(pairs): + result = {} + for key, value in pairs: + require(key not in result) + result[key] = value + return result + + +def expected_inventory(peers): + require(type(peers) is dict and set(peers) == set(FIXTURE_ROLES)) + require(all(type(value) is str and re.fullmatch(r"[1-9A-HJ-NP-Za-km-z]{32,128}", value) + for value in peers.values()) and len(set(peers.values())) == len(FIXTURE_ROLES)) + return {peers[name]: "0b100" if name == "exit" else "0b010" for name in INVENTORY_ROLES} + + +def parse_inventory(raw): + # The pinned CLI validates its bounded RPC frame before printing. Do not + # accept duplicate identities, role substitutions or a truncated last line. + if type(raw) is not bytes or len(raw) > 1048576 or (raw and not raw.endswith(b"\n")): + return None + try: + lines = raw.decode("ascii").split("\n")[:-1] if raw else [] + except UnicodeError: + return None + if len(lines) > 4096: + return None + observed = {} + for line in lines: + match = re.fullmatch(r"([1-9A-HJ-NP-Za-km-z]{32,128})\troles=(0b[01]{3})\treachability=([0-3])", line) + if match is None or match[1] in observed: + return None + observed[match[1]] = match[2] + return observed + + +def new_record(): + return dict(version=1, scope=SCOPE, deadline_seconds=DEADLINE_SECONDS, attempts=0, + query_timeouts=0, query_nonzero=0, invalid_replies=0, last_query_outcome="none", + last_valid_presence=None, outcome="invalid_fixture", ready=False) + + +def closed_record(value): + require(type(value) is dict and set(value) == set(new_record())) + require(type(value["version"]) is int and value["version"] == 1 and value["scope"] == SCOPE + and type(value["deadline_seconds"]) is int and value["deadline_seconds"] == DEADLINE_SECONDS + and type(value["ready"]) is bool and value["outcome"] in TERMINAL + and value["last_query_outcome"] in OUTCOMES) + require(all(type(value[key]) is int and 0 <= value[key] <= MAX_ATTEMPTS + for key in ("attempts", "query_timeouts", "query_nonzero", "invalid_replies"))) + require(sum(value[key] for key in ("query_timeouts", "query_nonzero", "invalid_replies")) <= value["attempts"]) + presence = value["last_valid_presence"] + require(presence is None or (type(presence) is dict and set(presence) == set(INVENTORY_ROLES) + and all(type(item) is bool for item in presence.values()))) + require(value["ready"] == (value["outcome"] == "ready")) + if value["ready"]: + require(value["attempts"] > 0 and value["last_query_outcome"] == "complete" + and presence is not None and all(presence.values())) + return value + + +def load_fixture(work): + # Production invocation is additionally guarded by the enclosing disposable + # Debian/KVM topology. Keep this helper tied to its exact immutable staging. + require(os.geteuid() == 0 and socket.gethostname() == "volparossa-alpha" + and work.parent == Path("/opt") and re.fullmatch(r"va\.[0-9a-f]{32}\.[A-Za-z0-9]{6}", work.name) + and work.resolve(strict=True) == work) + info = work.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == 0 and not info.st_mode & 0o022) + binary = work / "bin/volparossa" + info = binary.lstat() + require(binary.resolve(strict=True) == binary and stat.S_ISREG(info.st_mode) + and info.st_uid == 0 and stat.S_IMODE(info.st_mode) == 0o555) + source = work / "a01-expected-peers.json" + with os.fdopen(os.open(source, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 and 0 < info.st_size <= 8192) + raw = stream.read(8193) + require(len(raw) == info.st_size and os.fstat(stream.fileno()).st_size == info.st_size) + peers = json.loads(raw, object_pairs_hook=unique_pairs) + expected_inventory(peers) + return peers + + +def await_inventory(work, peers): + expected = expected_inventory(peers) + record = new_record() + command = [str(work / "bin/volparossa"), "--control-socket", + str(work / "runtime-client/control/agent.sock"), "peers"] + deadline = time.monotonic() + DEADLINE_SECONDS + while record["attempts"] < MAX_ATTEMPTS and (remaining := deadline - time.monotonic()) > 0: + record["attempts"] += 1 + try: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + check=False, timeout=min(2, remaining)) + observed = parse_inventory(result.stdout) if result.returncode == 0 else None + if result.returncode != 0: + record["query_nonzero"] += 1 + record["last_query_outcome"] = "nonzero" + elif observed is None: + record["invalid_replies"] += 1 + record["last_query_outcome"] = "invalid_reply" + else: + record["last_valid_presence"] = { + name: observed.get(peers[name]) == expected[peers[name]] for name in INVENTORY_ROLES} + record["last_query_outcome"] = "complete" if all(record["last_valid_presence"].values()) else "partial" + if record["last_query_outcome"] == "complete" and time.monotonic() < deadline: + record.update(outcome="ready", ready=True) + return closed_record(record) + except subprocess.TimeoutExpired: + record["query_timeouts"] += 1 + record["last_query_outcome"] = "timeout" + except OSError: + record.update(last_query_outcome="os_error", outcome="os_error") + return closed_record(record) + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(0.1, remaining)) + record["outcome"] = "deadline" + return closed_record(record) + + +def main(arguments): + record = new_record() + try: + require(len(arguments) == 1) + work = Path(arguments[0]) + record = await_inventory(work, load_fixture(work)) + except (OSError, ValueError, TypeError, KeyboardInterrupt): + # Input/guard failures still yield a closed error record, never the + # exception text or any fixture IDs, addresses, credentials or paths. + pass + print(json.dumps(closed_record(record), sort_keys=True, separators=(",", ":"))) + return 0 if record["ready"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/integration/content-provider-smoke.sh b/tests/integration/content-provider-smoke.sh index 72f113dcb..979ff4234 100644 --- a/tests/integration/content-provider-smoke.sh +++ b/tests/integration/content-provider-smoke.sh @@ -159,6 +159,11 @@ content_provider_cancellation_run() { } content_provider_run() { + PHASE=content-provider-inventory + # Service activation alone does not mean the primary Exit advertisement has + # arrived. Query only Client inventory; keep all route checks/retries intact. + python3 -I -B "$WORK/bin/content-provider-readiness.py" "$WORK" \ + >"$WORK/content-provider-readiness.json" || fail CONTENT_PROVIDER_INVENTORY_UNAVAILABLE PHASE=content-provider-selection benchmark_select_route content-provider mptcp || fail CONTENT_PROVIDER_MPTCP_SELECTION_UNAVAILABLE benchmark_bind_slots "$WORK/content-provider-selection.json" || fail CONTENT_PROVIDER_SELECTION_INVALID diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index b1b18b8e9..744af9c67 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1316,7 +1316,7 @@ if [ "$scenario" = content-mailbox ]; then done fi if [ "$scenario" = content-provider ]; then - for provider_fixture in content-provider-smoke.sh content-provider-smoke.py content-network-smoke.py \ + for provider_fixture in content-provider-smoke.sh content-provider-smoke.py content-provider-readiness.py content-network-smoke.py \ content-provider-https-smoke.sh content-provider-https-smoke.py content-publication-smoke.sh content-named-smoke.sh \ content-provider-site-smoke.sh content-provider-site-smoke.py \ content-provider-filter-smoke.sh content-provider-filter-smoke.py \ @@ -1617,7 +1617,7 @@ copy_artifacts() { agent-relay0.log agent-relay1.log agent-relay2.log agent-relay3.log \ agent-relay4.log agent-relay5.log agent-exit.log agent-exit2.log \ helper-bootstrap1.log helper-bootstrap2.log \ - worker-network-diagnostics.txt content-replication-provider-network.txt \ + worker-network-diagnostics.txt content-replication-provider-network.txt content-provider-readiness.json \ status-bootstrap1.txt status-bootstrap2.txt \ roles-bootstrap1.txt roles-bootstrap2.txt \ a01-bootstrap1-blocked.json a01-bootstrap2-blocked.json \ @@ -2883,6 +2883,10 @@ if [ "$scenario" = content-https ] || [ "$scenario" = content-provider ]; then "$binary_directory/examples/https-content-acceptance-fixture" \ "$WORK/bin/examples/https-content-acceptance-fixture" fi +if [ "$scenario" = content-provider ]; then + install -o root -g root -m 0555 "$source_directory/tests/integration/content-provider-readiness.py" \ + "$WORK/bin/content-provider-readiness.py" +fi if [ "$scenario" = content-custody ] || [ "$scenario" = agent-artifact ] || [ "$scenario" = agent-jobs ]; then for custody_script in content-custody-smoke.py content-retain-smoke.py content-provider-smoke.py content-provider-https-smoke.py content-network-smoke.py; do install -o root -g root -m 0555 "$source_directory/tests/integration/$custody_script" "$WORK/bin/$custody_script" diff --git a/tests/integration/test-content-provider-readiness.py b/tests/integration/test-content-provider-readiness.py new file mode 100644 index 000000000..53fa4ea27 --- /dev/null +++ b/tests/integration/test-content-provider-readiness.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Synthetic inventory/clock contracts; no agent, route, namespace or VM runs.""" +import io +import json +from pathlib import Path +import re +import runpy +import subprocess +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +CHECK = runpy.run_path(str(HERE / "content-provider-readiness.py")) +WORK = Path("/opt/va." + "a" * 32 + ".ABC123") + + +def peers(): + return {name: "1" * 31 + "ABCDEFGHJKLMNPQRSTUVWXYZ"[index] + for index, name in enumerate(CHECK["FIXTURE_ROLES"])} + + +def inventory(names): + expected = peers() + return "".join(f"{expected[name]}\troles={'0b010' if name.startswith('relay') else '0b100'}\treachability=1\n" + for name in names).encode() + + +class ContentProviderReadiness(unittest.TestCase): + def run_wait(self, replies): + elapsed = [0.0] + calls = [] + sequence = iter(replies) + + def run(command, **options): + calls.append((command, options)) + reply, duration = next(sequence) + elapsed[0] += duration + if isinstance(reply, Exception): + raise reply + status, raw = reply + return SimpleNamespace(returncode=status, stdout=raw) + + def sleep(seconds): + elapsed[0] += seconds + + runtime = SimpleNamespace(run=run, PIPE=subprocess.PIPE, DEVNULL=subprocess.DEVNULL, + TimeoutExpired=subprocess.TimeoutExpired) + clock = SimpleNamespace(monotonic=lambda: elapsed[0], sleep=sleep) + function = CHECK["await_inventory"] + with patch.dict(function.__globals__, subprocess=runtime, time=clock): + value = function(WORK, peers()) + self.assertIs(CHECK["closed_record"](value), value) + for command, options in calls: + self.assertEqual(command, [str(WORK / "bin/volparossa"), "--control-socket", + str(WORK / "runtime-client/control/agent.sock"), "peers"]) + self.assertEqual(set(options), {"stdout", "stderr", "check", "timeout"}) + self.assertEqual(options["stdout"], subprocess.PIPE) + self.assertEqual(options["stderr"], subprocess.DEVNULL) + self.assertFalse(options["check"]) + self.assertGreater(options["timeout"], 0) + self.assertLessEqual(options["timeout"], 2) + return value, elapsed[0], calls + + def test_exact_vm3_partial_inventory_does_not_proceed(self): + # VM3 had R0/R1/R2 plus only the deliberately under-capacity Exit2. + raw = inventory(("relay0", "relay1", "relay2", "exit2")) + value, elapsed, calls = self.run_wait([((0, raw), 60)]) + self.assertEqual(elapsed, 60) + self.assertEqual(len(calls), 1) + self.assertFalse(value["ready"]) + self.assertEqual(value["outcome"], "deadline") + self.assertEqual(value["last_valid_presence"], dict(relay0=True, relay1=True, relay2=True, exit=False)) + + def test_primary_exit_arrives_then_inventory_barrier_proceeds(self): + value, elapsed, calls = self.run_wait([ + ((0, inventory(("relay0", "relay1", "relay2", "exit2"))), 0), + ((0, inventory(("relay0", "relay1", "relay2", "exit", "exit2"))), 0), + ]) + self.assertTrue(value["ready"]) + self.assertEqual(value["outcome"], "ready") + self.assertEqual((len(calls), elapsed), (2, 0.1)) + self.assertEqual(value["scope"], "client_advertisement_inventory_not_route_readiness") + + def test_only_three_route_relays_and_primary_exit_are_required(self): + self.assertEqual(CHECK["INVENTORY_ROLES"], ("relay0", "relay1", "relay2", "exit")) + value, _, _ = self.run_wait([((0, inventory(CHECK["INVENTORY_ROLES"])), 0)]) + self.assertTrue(value["ready"]) + + def test_other_relays_or_exit2_cannot_substitute_for_required_identity(self): + for missing in CHECK["INVENTORY_ROLES"]: + with self.subTest(missing=missing): + names = [name for name in (*CHECK["INVENTORY_ROLES"], "relay3", "relay4", "relay5", "exit2") + if name != missing] + value, _, _ = self.run_wait([((0, inventory(names)), 60)]) + self.assertFalse(value["ready"]) + self.assertFalse(value["last_valid_presence"][missing]) + + def test_roles_must_match_even_with_correct_identity_count(self): + raw = inventory(CHECK["INVENTORY_ROLES"]).replace(b"roles=0b010", b"roles=0b100", 1) + value, _, _ = self.run_wait([((0, raw), 60)]) + self.assertFalse(value["ready"]) + self.assertFalse(value["last_valid_presence"]["relay0"]) + + def test_expected_ids_are_complete_unique_and_immutable(self): + expected = peers() + before = dict(expected) + CHECK["expected_inventory"](expected) + self.assertEqual(expected, before) + for modified in (dict(expected, exit=expected["exit2"]), dict(expected, relay4=expected["relay0"]), + dict(expected, exit="private-invalid"), dict(expected, extra="2" * 32)): + with self.assertRaises(ValueError): CHECK["expected_inventory"](modified) + modified = dict(expected); del modified["bootstrap1"] + with self.assertRaises(ValueError): CHECK["expected_inventory"](modified) + + def test_malformed_duplicate_oversized_and_truncated_inventory_fail_closed(self): + valid = inventory(CHECK["INVENTORY_ROLES"]) + for raw in (b"PRIVATE_SENTINEL", b"\xff\n", valid + valid, valid[:-1], + b"x" * 1048577, valid.replace(b"reachability=1", b"reachability=9")): + with self.subTest(raw=raw[:60]): + self.assertIsNone(CHECK["parse_inventory"](raw)) + value, _, _ = self.run_wait([((0, raw), 60)]) + self.assertFalse(value["ready"]) + self.assertEqual(value["invalid_replies"], 1) + self.assertIsNone(value["last_valid_presence"]) + + def test_transient_query_failures_are_counted_without_exporting_raw_data(self): + complete = inventory(CHECK["INVENTORY_ROLES"]) + value, _, calls = self.run_wait([ + (subprocess.TimeoutExpired("PRIVATE_SENTINEL", 2), 2), + ((1, b"PRIVATE_SENTINEL"), 0), ((0, b"PRIVATE_SENTINEL"), 0), ((0, complete), 0), + ]) + self.assertTrue(value["ready"]) + self.assertEqual((value["query_timeouts"], value["query_nonzero"], value["invalid_replies"]), (1, 1, 1)) + self.assertEqual(len(calls), 4) + serialized = json.dumps(value) + self.assertNotIn("PRIVATE_SENTINEL", serialized) + self.assertTrue(all(peer not in serialized for peer in peers().values())) + self.assertNotIn(str(WORK), serialized) + + def test_complete_reply_at_deadline_is_not_success(self): + value, elapsed, _ = self.run_wait([((0, inventory(CHECK["INVENTORY_ROLES"])), 60)]) + self.assertEqual(elapsed, 60) + self.assertFalse(value["ready"]) + self.assertEqual(value["outcome"], "deadline") + + def test_query_deadlines_shrink_and_do_not_restart_budget(self): + replies = [(subprocess.TimeoutExpired("closed", 2), 2)] * 28 + replies.append((subprocess.TimeoutExpired("closed", 1.2), 1.2)) + value, elapsed, calls = self.run_wait(replies) + self.assertAlmostEqual(elapsed, 60) + self.assertFalse(value["ready"]) + self.assertEqual(value["outcome"], "deadline") + self.assertAlmostEqual(calls[-1][1]["timeout"], 1.2) + + def test_os_error_has_closed_terminal_receipt(self): + value, _, _ = self.run_wait([(OSError("PRIVATE_SENTINEL"), 0)]) + self.assertEqual(value["outcome"], "os_error") + self.assertFalse(value["ready"]) + self.assertNotIn("PRIVATE_SENTINEL", json.dumps(value)) + + def test_invalid_fixture_cli_still_emits_closed_error(self): + with patch("sys.stdout", new_callable=io.StringIO) as output: + status = CHECK["main"]([]) + value = json.loads(output.getvalue()) + self.assertEqual(status, 1) + self.assertEqual(value["outcome"], "invalid_fixture") + self.assertIs(CHECK["closed_record"](value), value) + + def test_receipt_rejects_private_fields_false_readiness_and_unbounded_counts(self): + record = CHECK["new_record"]() + for change in ({"raw_ids": list(peers().values())}, {"ready": True}, {"attempts": 601}, + {"last_valid_presence": {"unexpected": True}}, {"scope": "path_readiness_proven"}): + with self.assertRaises(ValueError): CHECK["closed_record"](dict(record, **change)) + + def test_hook_staging_and_exports_are_specific_without_selector_changes(self): + shell = (HERE / "content-provider-smoke.sh").read_text() + start = shell.index("content_provider_run() {\n") + barrier = shell[start:shell.index(" benchmark_bind_slots", start)] + self.assertLess(barrier.index("content-provider-readiness.py"), barrier.index("benchmark_select_route")) + self.assertIn('|| fail CONTENT_PROVIDER_INVENTORY_UNAVAILABLE', barrier) + self.assertIn('benchmark_select_route content-provider mptcp || fail CONTENT_PROVIDER_MPTCP_SELECTION_UNAVAILABLE', barrier) + topology = (HERE / "kvm-alpha-topology.sh").read_text() + self.assertIn('if [ "$scenario" = content-provider ]; then\n' + ' install -o root -g root -m 0555 "$source_directory/tests/integration/content-provider-readiness.py"', topology) + self.assertIn("content-provider-readiness.json", topology.split("copy_artifacts() {", 1)[1].split("\n}", 1)[0]) + self.assertIn('"$WORK"/content-provider-*.json', shell) + selection = (HERE / "benchmark-selection.sh").read_text() + retry = selection.split("a01_transient_connect_unavailable() {", 1)[1].split("\n}", 1)[0] + self.assertNotIn("NO_ELIGIBLE_PATHS", retry) + self.assertEqual(re.findall(r"\b[A-Z_]+_UNAVAILABLE\b", retry), ["PRESELECTION_UNAVAILABLE", + "NATIVE_PERMIT_UNAVAILABLE", "NATIVE_RELAY_READY_UNAVAILABLE", "NATIVE_HELPER_COMMIT_UNAVAILABLE", + "NATIVE_PROBE_START_UNAVAILABLE", "NATIVE_PROBE_PROOF_UNAVAILABLE", "ROUTE_ADMISSION_UNAVAILABLE"]) + + +if __name__ == "__main__": + unittest.main() From fa18d11a5fd0a4139e90f06e35676333304e444e Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:54:26 +0200 Subject: [PATCH 07/49] docs: record browser startup failure after discovery convergence --- docs/IMPLEMENTATION_STATUS.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 684d498fd..496fe80b7 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -192,8 +192,23 @@ inventory. Fourteen synthetic tests cover partial and malformed inventories, role substitutions, deadline handling and the closed diagnostic receipt. The original route selector, capacity requirements and terminal `NO_ELIGIBLE_PATHS` handling are unchanged. Inventory convergence is not proof of route readiness; -another real combined runtime trial is required. Browser integration remains -incomplete. +the combined runtime must still pass before Browser integration is complete. + +The fourth local attempt on `e2aa069052f8181fb74f54645507426248a632cb`, +using the same frozen Browser input manifest, observed all four required +identities after 210 inventory queries and reached the Browser hook. It failed +at `BROWSER_FILTER_NATIVE_FAILED`. This time the original report and outer +cleanup survived: the failure operation is `browser_start`, with no completed +browser probes or delivery receipts. The closed outer diagnostic records +`wrapper` and `cleanup_failed: false`. A started broker alone does not establish +successful Firefox startup or filter delivery. + +The `browser-filter-delivery-04.G3DJR5` collection retains successful broker +reaping, temporary-profile and input removal, zero remaining owned topology +objects, and matching guest-root network-state hashes +`558aace199b57512d8f96b624de8f70adab8c03a8712cec4070199c4810a4273`. +The VM and its temporary disk were removed. Browser startup diagnosis and a +successful joined trial remain required; raw browser logs are not exported. ### Shared Weather application provider From 397b5b84aaaf6cdf7494829bdc0f8376ac404006 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:06:33 +0200 Subject: [PATCH 08/49] test: retain closed browser initialization diagnostics --- docs/IMPLEMENTATION_STATUS.md | 15 ++++- tests/integration/browser-filter-delivery.py | 28 ++++++++- .../test-browser-filter-delivery.py | 58 +++++++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 496fe80b7..438277e14 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -200,16 +200,27 @@ identities after 210 inventory queries and reached the Browser hook. It failed at `BROWSER_FILTER_NATIVE_FAILED`. This time the original report and outer cleanup survived: the failure operation is `browser_start`, with no completed browser probes or delivery receipts. The closed outer diagnostic records -`wrapper` and `cleanup_failed: false`. A started broker alone does not establish +`wrapper` and `cleanup_failed: false`. The `browser_start` operation covers +Firefox initialization and baseline extension/probe checks, so this report does +not identify which of those failed. A started broker alone does not establish successful Firefox startup or filter delivery. The `browser-filter-delivery-04.G3DJR5` collection retains successful broker reaping, temporary-profile and input removal, zero remaining owned topology objects, and matching guest-root network-state hashes `558aace199b57512d8f96b624de8f70adab8c03a8712cec4070199c4810a4273`. -The VM and its temporary disk were removed. Browser startup diagnosis and a +The VM and its temporary disk were removed. Browser initialization/baseline diagnosis and a successful joined trial remain required; raw browser logs are not exported. +The next-trial Browser driver records closed initialization/check subphases and +error categories, while retaining the original failure through cleanup. The +core validates those optional failure-only fields before exporting them; 20 +joined-proof/export tests and 13 runner tests pass. The driver also corrects its +fresh-profile check to treat an absent uBO `importedLists` key as the upstream +default empty list. Explicit malformed values and unexpected lists remain +rejected. This is a verified fixture defect, not proof of VM4's precise cause; +no signed extension package or product filtering rule was changed. + ### Shared Weather application provider Weather [PR5](https://github.com/VOLPAROSSA/volparossa-weather/pull/5) is merged diff --git a/tests/integration/browser-filter-delivery.py b/tests/integration/browser-filter-delivery.py index 99c3b9cde..3b318cc45 100644 --- a/tests/integration/browser-filter-delivery.py +++ b/tests/integration/browser-filter-delivery.py @@ -34,10 +34,17 @@ "cleanup_profile", "cleanup_binary", "cleanup_receipt", } DIAGNOSTIC_KEYS = {"schema", "success", "phase", "cleanup_failed"} +BROWSER_DIAGNOSTIC_KEYS = {"schema", "phase", "error_kind", "cleanup_failed"} +BROWSER_PHASES = { + "log_open", "spawn", "log_drain", "connect", "greeting", "session", "context", "runtime_check", + "preference_check", "addon_check", "fixture_server", "baseline_selection", "baseline_probe", "exercise", + "selection", "probe", "final_addon_check", "final_preference_check", "quit", "join", "cleanup", +} +BROWSER_ERROR_KINDS = {"check_failed", "timeout", "io_error", "command_error", "interrupted", "other"} REPORT_KEYS = {"schema", "passed", "scope", "operation", "receipts", "probes", "module_sha256", "driver_sha256", "broker_sha256", "expected", "runtime_version", "runtime_source_stamp", "runtime_sha256", "extension_packages", "failure", "host_read_only", "interfaces", "browser_exit_status", "broker_cleanup", "broker_socket_removed", - "temporary_profiles_removed", "logs", "failure_phase"} + "temporary_profiles_removed", "logs", "failure_phase", "browser_diagnostic"} EXPORT_KEYS = { "original-report": REPORT_KEYS, "original-outer-cleanup": {"schema", "temporary_profiles_removed", "broker_copy_removed", "logs"}, @@ -57,7 +64,7 @@ "schema", "status", "choice", "state", "closed", "failed", "result", "attempted", "receipt", "selected", "imported", "preserved", "freshReload", "reloadEvents", "delivery", "asset", "broker_revoked", "bytes", "sha256", "rules", "manifest_id", "publisher_key", "name", "authority_expires_unix_seconds", "accepted", "completed", "denied", - "ok", "phase", "ads", "tracker", "neutral", "started", "reaped", "graceful", "broker", "id", "size", + "ok", "phase", "error_kind", "ads", "tracker", "neutral", "started", "reaped", "graceful", "broker", "id", "size", "unpacked_bytes", "zip_entries", "report", "outer-cleanup", "diagnostic", "content_type", "peer_bytes", "providers_used", "privacy_captures_validated", "exit_status", "same_publication_joined", "broker_reuse_requested", "broker_cache_only_proven", "new_broker_peer_transfer_proven", "two_subflow_bulk_aggregation", "browser_profile_removed", "broker_reaped", @@ -74,7 +81,7 @@ "clock_error", "expired", "invalid_snapshot", "invalid_response", "not_configured", "owner_config", "owner_context", "owner_clock", "owner_busy", "owner_closed", "owner_failed", "owner_cleanup", "asset_config", "asset_package", "asset_context", "asset_authority", "asset_expired", "asset_clock", "asset_busy", "asset_closed", "asset_failed", "asset_cleanup", - *DIAGNOSTIC_PHASES} + *DIAGNOSTIC_PHASES, *BROWSER_PHASES, *BROWSER_ERROR_KINDS} def require(condition): @@ -125,6 +132,19 @@ def validate_diagnostic(value): and type(value["phase"]) is str and value["phase"] in DIAGNOSTIC_PHASES) +def validate_report_diagnostic(report): + # Older reports have no browser subphase. A new diagnostic is failure-only, + # and must never turn baseline checks into a claim that Firefox did not start. + if "browser_diagnostic" not in report: + return + value = report["browser_diagnostic"] + require(report.get("passed") is False and type(value) is dict and set(value) == BROWSER_DIAGNOSTIC_KEYS + and type(value["schema"]) is int and value["schema"] == 1 + and type(value["phase"]) is str and value["phase"] in BROWSER_PHASES + and type(value["error_kind"]) is str and value["error_kind"] in BROWSER_ERROR_KINDS + and type(value["cleanup_failed"]) is bool) + + def private_root(root): require(root.name == "browser-filter-delivery" and root.resolve(strict=True) == root) info = root.lstat() @@ -283,6 +303,7 @@ def retain(work): raw = raw_file(path) value = json.loads(raw, object_pairs_hook=BUNDLE["unique_pairs"]) require(type(value) is dict and set(value) <= keys and type(value.get("schema")) is int and value["schema"] == 1) + if name == "report": validate_report_diagnostic(value) if name == "diagnostic": validate_diagnostic(value) target = work / ("browser-filter-delivery-original-" + name + ".json") if target.exists(): require(raw_file(target) == raw) @@ -326,6 +347,7 @@ def copy_artifacts(work, output, uid, gid): if suffix == "original-report": require({"schema", "passed", "failure"} <= set(value) and type(value["schema"]) is int and value["schema"] == 1 and type(value["passed"]) is bool) + validate_report_diagnostic(value) else: require(set(value) == keys) if suffix == "original-diagnostic": validate_diagnostic(value) export_value(value) diff --git a/tests/integration/test-browser-filter-delivery.py b/tests/integration/test-browser-filter-delivery.py index 78622ff47..011b45e5b 100644 --- a/tests/integration/test-browser-filter-delivery.py +++ b/tests/integration/test-browser-filter-delivery.py @@ -212,6 +212,64 @@ def test_empty_failure_receipts_are_missing_not_success_and_original_false_is_pr self.assertFalse(result["successful_native_execution_claimed"]) self.assertEqual((output / "browser-filter-delivery-original-report.json").read_bytes(), raw) + def test_browser_subphase_diagnostic_retained_and_exported_byte_exact_on_failure(self): + for phase in CHECK["BROWSER_PHASES"]: + for kind in CHECK["BROWSER_ERROR_KINDS"]: + value = dict(schema=1, passed=False, failure="fixture_failed", operation="browser_start", + browser_diagnostic=dict(schema=1, phase=phase, error_kind=kind, cleanup_failed=False)) + CHECK["validate_report_diagnostic"](value) + CHECK["export_value"](value) + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve() + original = work / "client-fixtures/browser-filter-source/build/filter-delivery" + original.mkdir(parents=True) + # browser_start includes baseline checks: preserve the actual subphase. + value = dict(schema=1, passed=False, failure="fixture_failed", operation="browser_start", + browser_diagnostic=dict(schema=1, phase="baseline_probe", error_kind="check_failed", cleanup_failed=True)) + raw = json.dumps(value, indent=2).encode() + CHECK["write_new"](original / "report.json", raw) + retained = CHECK["retain"](work) + self.assertEqual(retained["originals_sha256"], {"report": hashlib.sha256(raw).hexdigest()}) + self.assertFalse(retained["successful_native_execution_claimed"]) + output = work / "export"; output.mkdir() + exported = CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual((output / "browser-filter-delivery-original-report.json").read_bytes(), raw) + self.assertFalse(exported["successful_native_execution_claimed"]) + + def test_browser_subphase_schema_rejects_raw_fields_wrong_types_and_unknown_enums_before_export(self): + diagnostic = dict(schema=1, phase="greeting", error_kind="timeout", cleanup_failed=False) + changes = [dict(schema=True), dict(schema=2), dict(phase="PRIVATE_SENTINEL"), dict(phase=["greeting"]), + dict(error_kind="none"), dict(error_kind=["timeout"]), dict(cleanup_failed=1), dict(raw="PRIVATE_SENTINEL")] + bad_values = [None, {}, {key: value for key, value in diagnostic.items() if key != "phase"}, + *(dict(diagnostic, **change) for change in changes)] + for bad in bad_values: + with self.subTest(bad=bad), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve(); output = work / "export"; output.mkdir() + value = dict(schema=1, passed=False, failure="fixture_failed", browser_diagnostic=bad) + CHECK["write_new"](work / "browser-filter-delivery-original-report.json", json.dumps(value).encode()) + with self.assertRaises(ValueError) as error: + CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertNotIn("PRIVATE_SENTINEL", str(error.exception)) + self.assertEqual(list(output.iterdir()), []) + original = work / "client-fixtures/browser-filter-source/build/filter-delivery" + original.mkdir(parents=True) + CHECK["write_new"](original / "report.json", json.dumps(value).encode()) + with self.assertRaises(ValueError): CHECK["retain"](work) + + def test_browser_subphase_is_never_allowed_on_success_and_old_reports_remain_compatible(self): + args = list(fixture()) + CHECK["validate_report_diagnostic"](args[0]) + CHECK["validate_browser"](*args) + CHECK["validate_report_diagnostic"](dict(schema=1, passed=False, failure="fixture_failed")) + args[0]["browser_diagnostic"] = dict(schema=1, phase="cleanup", error_kind="io_error", cleanup_failed=True) + with self.assertRaises(ValueError): CHECK["validate_browser"](*args) + with self.assertRaises(ValueError): CHECK["validate_report_diagnostic"](args[0]) + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve(); output = work / "export"; output.mkdir() + CHECK["write_new"](work / "browser-filter-delivery-original-report.json", json.dumps(args[0]).encode()) + with self.assertRaises(ValueError): CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + def test_artifact_export_refuses_private_nested_value_before_any_copy(self): for mutation in (lambda value: value.update(raw="PRIVATE"), lambda value: value.update(failure="PRIVATE /home/user"), From b84f339baf34547ad1156d15ed9f58f61aa1d202 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:48:10 +0200 Subject: [PATCH 09/49] docs: record merged Drizzli browser milestone and remaining network proof --- docs/IMPLEMENTATION_STATUS.md | 43 +++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 438277e14..33c27efd2 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -221,20 +221,35 @@ default empty list. Explicit malformed values and unexpected lists remain rejected. This is a verified fixture defect, not proof of VM4's precise cause; no signed extension package or product filtering rule was changed. -### Shared Weather application provider - -Weather [PR5](https://github.com/VOLPAROSSA/volparossa-weather/pull/5) is merged -in its own `main` at `c14042a75b77608fcc968f9eabb93e015f87c0aa`. -The executable provider contract exposes temperature and wind on the original -model time axis, retaining source evidence and the earliest forecast or signed -publication expiry. Its core-cache adapter reuses the existing authenticated -fetch path without an implicit origin or Open-Meteo fallback. All 33 focused -tests and the repository CI pass. - -This is the shared data interface, not yet rendering in Drizz.li or Overmorrow -or a live Weather-overlay proof. Application transport and UI adapters remain -required. Precipitation and air quality also remain required work; unsupported -fields are explicitly unavailable, never substituted with zero values. +### Weather provider and Drizz.li web integration + +The shared Weather provider from +[PR5](https://github.com/VOLPAROSSA/volparossa-weather/pull/5) exposes temperature +and wind on the original model time axis, retaining source evidence and the +earliest forecast or signed publication expiry. Its core-cache adapter reuses +the existing authenticated fetch path without an implicit origin or Open-Meteo +fallback. + +Weather [PR6](https://github.com/VOLPAROSSA/volparossa-weather/pull/6) is merged +in its own `main` at `ad9b788f2f284833e2ecdcaa1f4c60516c1bcef8`. It adds the +pinned Drizz.li week-route integration and a fixed-area, loopback-only HTTP +bridge to that adapter. The full app build and Svelte/TypeScript checks pass, +alongside 46 Weather tests, 11 client/render tests and four browser-harness tests. +An isolated real Firefox trial verifies initial display, refresh, missing/stale +values, expiry, route guards and cleanup through the actual HTTP bridge with +synthetic weather. Its +[retained receipt](https://github.com/VOLPAROSSA/volparossa-weather/blob/537db8860d2d0567fb87c84782d23e50423620de/integrations/drizzli/browser-smoke.json) +binds the final UI, bridge, fixture, build and runtime inputs. Static-file +authorization uses the opened descriptor; the final browser trial includes that +correction. The persistent bridge requires an explicitly initialized, owned cache +with reuse enabled, rather than trying to create it again on every refresh. +The final repository CI and CodeQL checks pass. + +A real core-cache publication reaching the browser remains unproven; the +synthetic trial does not establish live overlay or weather-origin delivery. +Overmorrow, additional location selection, precipitation and air quality remain +required work. Unsupported fields are explicitly unavailable, never substituted +with zero values. ### Citizen procedure simulation From 34dc5cf4798d944fa8327a68275a5898ce1ae343 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 01:50:10 +0200 Subject: [PATCH 10/49] docs: retain failed browser session proof and verified VM cleanup --- docs/IMPLEMENTATION_STATUS.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 33c27efd2..439414799 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -212,7 +212,7 @@ objects, and matching guest-root network-state hashes The VM and its temporary disk were removed. Browser initialization/baseline diagnosis and a successful joined trial remain required; raw browser logs are not exported. -The next-trial Browser driver records closed initialization/check subphases and +The Browser driver records closed initialization/check subphases and error categories, while retaining the original failure through cleanup. The core validates those optional failure-only fields before exporting them; 20 joined-proof/export tests and 13 runner tests pass. The driver also corrects its @@ -221,6 +221,24 @@ default empty list. Explicit malformed values and unexpected lists remain rejected. This is a verified fixture defect, not proof of VM4's precise cause; no signed extension package or product filtering rule was changed. +The fifth local attempt tested core `397b5b84aaaf6cdf7494829bdc0f8376ac404006` +with frozen Browser manifest +`8fd5239261e5a78501b9d8fb4a77e46e2fb5f275f20900cf7ffe954410e42329`. +All four required identities converged after 203 inventory queries. The trial +reached the Browser hook but failed at `BROWSER_FILTER_NATIVE_FAILED`; its new +diagnostic narrows the failure to `session` / `command_error`, before extension +baseline checks. The attempted `WebDriver:NewSession` command did not complete +successfully. This does not establish its precise cause, and no browser probes +or delivery receipts completed. + +The retained `browser-filter-delivery-05.KDUpgD` collection records broker reaping, +profile/input removal, zero remaining owned topology objects and matching +guest-root network hashes +`f1146383d82ba978c0546b4e0f02f8268aaf25cb85b727c217387740579510fd`. +The VM and temporary disk were removed. The next diagnostic step is an isolated +session-start reproduction, not another full network trial without a diagnosed +change. Joined filter delivery remains incomplete. + ### Weather provider and Drizz.li web integration The shared Weather provider from From 51f78671ef841ca0cca6d5687a7bd8b4fdd4745e Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:08:14 +0200 Subject: [PATCH 11/49] test: validate closed browser protocol failure exports --- tests/integration/browser-filter-delivery.py | 35 ++++++++++- .../test-browser-filter-delivery.py | 62 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/tests/integration/browser-filter-delivery.py b/tests/integration/browser-filter-delivery.py index 3b318cc45..70bdbdffb 100644 --- a/tests/integration/browser-filter-delivery.py +++ b/tests/integration/browser-filter-delivery.py @@ -41,6 +41,16 @@ "selection", "probe", "final_addon_check", "final_preference_check", "quit", "join", "cleanup", } BROWSER_ERROR_KINDS = {"check_failed", "timeout", "io_error", "command_error", "interrupted", "other"} +PROTOCOL_ERROR_KINDS = {"webdriver_error", "unexpected_envelope", "framing_error", "eof", "timeout", "transport_error"} +WEBDRIVER_CODES = {"session_not_created", "invalid_argument", "unknown_command", "unknown_error", + "unsupported_operation", "invalid_session_id", "no_such_window", "timeout", "script_timeout", "other"} +# Must match the Browser driver's closed source map for pinned ESR 140.16.0. +RUNTIME_SOURCE_LINES = { + "marionette/driver.sys.mjs": 3886, "marionette/server.sys.mjs": 460, + "shared/WindowManager.sys.mjs": 595, "shared/NavigationManager.sys.mjs": 860, + "shared/webdriver/Capabilities.sys.mjs": 1075, "shared/webdriver/Errors.sys.mjs": 978, + "shared/webdriver/Session.sys.mjs": 489, "webdriver-bidi/NewSessionHandler.sys.mjs": 55, +} REPORT_KEYS = {"schema", "passed", "scope", "operation", "receipts", "probes", "module_sha256", "driver_sha256", "broker_sha256", "expected", "runtime_version", "runtime_source_stamp", "runtime_sha256", "extension_packages", "failure", "host_read_only", "interfaces", "browser_exit_status", "broker_cleanup", "broker_socket_removed", @@ -71,6 +81,7 @@ "fixture_authority_removed", "separate_owner_input_removed", "agent_cache_identity_preserved", "agent_cache_private", "staged_inputs_removed_by_enclosing_topology", "primary_control_gid", "supplementary_groups_empty", "client_namespace", "outside_parent_namespace", "all_capabilities_dropped", "no_new_privileges", + "protocol_error", "webdriver_code", "source_frames", "module", "line", *("integration/filters/" + name + ".sys.mjs" for name in BUNDLE["MODULES"]), "tests/fixtures/filter_delivery.sys.mjs", "firefox-esr", "libxul.so", "omni.ja", "browser/omni.ja"}) EXPORT_WORDS = {"complete", "prepare", "browser_start", "browser_quit", "broker_start", "command", "bootstrap", *OPERATIONS, @@ -81,7 +92,8 @@ "clock_error", "expired", "invalid_snapshot", "invalid_response", "not_configured", "owner_config", "owner_context", "owner_clock", "owner_busy", "owner_closed", "owner_failed", "owner_cleanup", "asset_config", "asset_package", "asset_context", "asset_authority", "asset_expired", "asset_clock", "asset_busy", "asset_closed", "asset_failed", "asset_cleanup", - *DIAGNOSTIC_PHASES, *BROWSER_PHASES, *BROWSER_ERROR_KINDS} + *DIAGNOSTIC_PHASES, *BROWSER_PHASES, *BROWSER_ERROR_KINDS, + *PROTOCOL_ERROR_KINDS, *WEBDRIVER_CODES, *RUNTIME_SOURCE_LINES} def require(condition): @@ -132,17 +144,36 @@ def validate_diagnostic(value): and type(value["phase"]) is str and value["phase"] in DIAGNOSTIC_PHASES) +def validate_protocol_error(value): + require(type(value) is dict and set(value) == {"kind", "webdriver_code", "source_frames"} + and type(value["kind"]) is str and value["kind"] in PROTOCOL_ERROR_KINDS + and type(value["source_frames"]) is list and len(value["source_frames"]) <= 8) + if value["kind"] != "webdriver_error": + require(value["webdriver_code"] is None and value["source_frames"] == []) + return + require(type(value["webdriver_code"]) is str and value["webdriver_code"] in WEBDRIVER_CODES) + seen = set() + for frame in value["source_frames"]: + require(type(frame) is dict and set(frame) == {"module", "line"} + and type(frame["module"]) is str and frame["module"] in RUNTIME_SOURCE_LINES + and type(frame["line"]) is int and 1 <= frame["line"] <= RUNTIME_SOURCE_LINES[frame["module"]]) + pair = (frame["module"], frame["line"]) + require(pair not in seen); seen.add(pair) + + def validate_report_diagnostic(report): # Older reports have no browser subphase. A new diagnostic is failure-only, # and must never turn baseline checks into a claim that Firefox did not start. if "browser_diagnostic" not in report: return value = report["browser_diagnostic"] - require(report.get("passed") is False and type(value) is dict and set(value) == BROWSER_DIAGNOSTIC_KEYS + require(report.get("passed") is False and type(value) is dict + and set(value) in (BROWSER_DIAGNOSTIC_KEYS, BROWSER_DIAGNOSTIC_KEYS | {"protocol_error"}) and type(value["schema"]) is int and value["schema"] == 1 and type(value["phase"]) is str and value["phase"] in BROWSER_PHASES and type(value["error_kind"]) is str and value["error_kind"] in BROWSER_ERROR_KINDS and type(value["cleanup_failed"]) is bool) + if "protocol_error" in value: validate_protocol_error(value["protocol_error"]) def private_root(root): diff --git a/tests/integration/test-browser-filter-delivery.py b/tests/integration/test-browser-filter-delivery.py index 011b45e5b..0a170880e 100644 --- a/tests/integration/test-browser-filter-delivery.py +++ b/tests/integration/test-browser-filter-delivery.py @@ -256,6 +256,68 @@ def test_browser_subphase_schema_rejects_raw_fields_wrong_types_and_unknown_enum CHECK["write_new"](original / "report.json", json.dumps(value).encode()) with self.assertRaises(ValueError): CHECK["retain"](work) + def test_closed_protocol_diagnostics_are_retained_and_exported_byte_exact(self): + values = [dict(kind=kind, webdriver_code=None, source_frames=[]) for kind in CHECK["PROTOCOL_ERROR_KINDS"] + if kind != "webdriver_error"] + values += [dict(kind="webdriver_error", webdriver_code=code, + source_frames=[dict(module=module, line=maximum)]) + for code in CHECK["WEBDRIVER_CODES"] for module, maximum in CHECK["RUNTIME_SOURCE_LINES"].items()] + for protocol in values: + with self.subTest(protocol=protocol), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve() + original = work / "client-fixtures/browser-filter-source/build/filter-delivery" + original.mkdir(parents=True) + diagnostic = dict(schema=1, phase="session", error_kind="command_error", cleanup_failed=False, + protocol_error=protocol) + value = dict(schema=1, passed=False, failure="fixture_failed", browser_diagnostic=diagnostic) + raw = json.dumps(value, indent=2).encode() + CHECK["write_new"](original / "report.json", raw) + retained = CHECK["retain"](work) + self.assertEqual(retained["originals_sha256"], {"report": hashlib.sha256(raw).hexdigest()}) + self.assertFalse(retained["successful_native_execution_claimed"]) + output = work / "export"; output.mkdir() + exported = CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual((output / "browser-filter-delivery-original-report.json").read_bytes(), raw) + self.assertFalse(exported["successful_native_execution_claimed"]) + + def test_protocol_diagnostic_rejects_private_fields_wrong_shapes_bounds_and_categories(self): + valid = dict(kind="webdriver_error", webdriver_code="session_not_created", + source_frames=[dict(module="marionette/driver.sys.mjs", line=717)]) + changed = [dict(kind="PRIVATE"), dict(kind=["webdriver_error"]), dict(webdriver_code="PRIVATE"), + dict(webdriver_code=None), dict(webdriver_code=["unknown_error"]), dict(source_frames=None), + dict(source_frames={}), dict(source_frames=[None]), dict(source_frames=[{}]), + dict(source_frames=[dict(module="PRIVATE", line=1)]), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line=True)]), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line=0)]), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line=3887)]), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line="717")]), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line=717, column=1)]), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line=717)] * 2), + dict(source_frames=[dict(module="marionette/driver.sys.mjs", line=i) for i in range(1, 10)]), + dict(message="PRIVATE"), dict(stacktrace="PRIVATE"), dict(response="PRIVATE"), dict(data="PRIVATE")] + bad_values = [None, {}, *(dict(valid, **change) for change in changed)] + for kind in CHECK["PROTOCOL_ERROR_KINDS"] - {"webdriver_error"}: + bad_values += [dict(kind=kind, webdriver_code="unknown_error", source_frames=[]), + dict(kind=kind, webdriver_code=None, source_frames=valid["source_frames"])] + for protocol in bad_values: + with self.subTest(protocol=protocol), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve(); output = work / "export"; output.mkdir() + value = dict(schema=1, passed=False, failure="fixture_failed", browser_diagnostic=dict( + schema=1, phase="session", error_kind="command_error", cleanup_failed=False, protocol_error=protocol)) + raw = json.dumps(value).encode() + CHECK["write_new"](work / "browser-filter-delivery-original-report.json", raw) + with self.assertRaises(ValueError) as error: + CHECK["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertNotIn("PRIVATE", str(error.exception)); self.assertEqual(list(output.iterdir()), []) + original = work / "client-fixtures/browser-filter-source/build/filter-delivery" + original.mkdir(parents=True); CHECK["write_new"](original / "report.json", raw) + with self.assertRaises(ValueError): CHECK["retain"](work) + + def test_protocol_diagnostic_is_never_accepted_on_success(self): + value = dict(passed=True, browser_diagnostic=dict(schema=1, phase="session", error_kind="command_error", + cleanup_failed=False, protocol_error=dict(kind="eof", webdriver_code=None, source_frames=[]))) + with self.assertRaises(ValueError): CHECK["validate_report_diagnostic"](value) + def test_browser_subphase_is_never_allowed_on_success_and_old_reports_remain_compatible(self): args = list(fixture()) CHECK["validate_report_diagnostic"](args[0]) From a5dc03bacb1f586394adb7d1de6936c085ad2528 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:23:52 +0200 Subject: [PATCH 12/49] test: freeze verified Weather and Drizzli runtime inputs --- .../integration/test-weather-client-bundle.py | 190 +++++++++++ tests/integration/weather-client-bundle.py | 304 ++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 tests/integration/test-weather-client-bundle.py create mode 100644 tests/integration/weather-client-bundle.py diff --git a/tests/integration/test-weather-client-bundle.py b/tests/integration/test-weather-client-bundle.py new file mode 100644 index 000000000..ba8d10f3a --- /dev/null +++ b/tests/integration/test-weather-client-bundle.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert archive/parser tests. No real binaries, acquisition, browser or VM.""" +import copy +import hashlib +import io +import json +import os +from pathlib import Path +import runpy +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +W = runpy.run_path(str(Path(__file__).with_name("weather-client-bundle.py"))) + + +class WeatherBundle(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="weather-bundle-test-") + self.root = Path(self.temp.name).resolve() + self.raw = b"inert input; must not execute\n" + self.digest = hashlib.sha256(self.raw).hexdigest() + self.pins = patch.dict(W["manifest"].__globals__, dict(NODE_SHA=self.digest, DECODER_SHA=self.digest, + PIN_SHA=self.digest, BUILD_SHA=self.digest)) + self.pins.start() + self.addCleanup(self.pins.stop) + self.addCleanup(self.temp.cleanup) + self.files = {name: dict(bytes=len(self.raw), mode=W["mode"](name), sha256=self.digest) + for name in W["FIXED"] | {W["STATIC"] + "index.html"}} + + def manifest(self, value=None): + raw = W["encode"](value or dict(version=1, kind=W["KIND"], files=self.files)) + return raw, hashlib.sha256(raw).hexdigest() + + def archive(self, suffix=b"\0" * 1024): + raw, expected = self.manifest() + archive = self.root / "inputs.tar" + with archive.open("wb") as target: + for name, data, mode in [("INPUTS.json", raw, 0o400), + *((name, self.raw, item["mode"]) for name, item in sorted(self.files.items()))]: + target.write(W["header"](name, len(data), mode)); target.write(data) + target.write(b"\0" * (-len(data) % 512)) + target.write(suffix) + archive.chmod(0o600) + return archive, expected + + def test_exact_roundtrip_readonly_tree_no_execution(self): + archive, expected = self.archive() + before = archive.read_bytes() + destination = self.root / "unpacked" + result = W["checked_archive"](archive, expected, destination) + self.assertEqual(result["files"], len(self.files)) + checked = W["verify_tree"](destination, expected) + self.assertFalse(checked["runtime_executed"]) + self.assertEqual(before, archive.read_bytes()) + for name, item in self.files.items(): + self.assertEqual((destination / name).read_bytes(), self.raw) + self.assertEqual((destination / name).stat().st_mode & 0o777, item["mode"]) + self.assertEqual(destination.stat().st_mode & 0o777, 0o500) + + def test_manifest_rejects_missing_unknown_escape_and_mode(self): + base = dict(version=1, kind=W["KIND"], files=self.files) + for fault in ("missing", "unknown", "escape", "mode", "size_bool", "extra", "version_bool", "zero"): + value = json.loads(json.dumps(base)) + if fault == "missing": del value["files"]["runtime/node"] + elif fault == "unknown": value["files"]["runtime/extra.so"] = self.files["runtime/node"] + elif fault == "escape": value["files"][W["STATIC"] + "../private"] = self.files["runtime/node"] + elif fault == "mode": value["files"]["runtime/node"]["mode"] = 0o777 + elif fault == "size_bool": value["files"]["runtime/node"]["bytes"] = True + elif fault == "extra": value["secret"] = "no" + elif fault == "version_bool": value["version"] = True + else: value["files"]["runtime/node"]["bytes"] = 0 + raw, expected = self.manifest(value) + with self.subTest(fault=fault), self.assertRaises(ValueError): W["manifest"](raw, expected) + + def test_runtime_pins_and_static_budget_are_enforced(self): + self.files["runtime/node"]["sha256"] = "f" * 64 + raw, expected = self.manifest() + with self.assertRaises(ValueError): W["manifest"](raw, expected) + self.files["runtime/node"]["sha256"] = self.digest + for i in range(5): self.files[W["STATIC"] + f"large{i}.js"] = dict(bytes=8*1024**2, mode=0o400, sha256=self.digest) + raw, expected = self.manifest() + with self.assertRaises(ValueError): W["manifest"](raw, expected) + + def test_bad_hash_refuses_before_creation_and_existing_tree_is_preserved(self): + archive, expected = self.archive(); destination = self.root / "target" + with self.assertRaises(ValueError): W["checked_archive"](archive, "f"*64, destination) + self.assertFalse(destination.exists()) + destination.mkdir(mode=0o700) + with self.assertRaises(ValueError): W["checked_archive"](archive, expected, destination) + self.assertEqual(list(destination.iterdir()), []) + + def test_trailing_payload_and_corruption_never_gain_accepted_marker(self): + archive, expected = self.archive(suffix=b"\0"*1024 + b"hidden") + with self.assertRaises(ValueError): W["checked_archive"](archive, expected) + archive, expected = self.archive() + raw = bytearray(archive.read_bytes()); offset = raw.index(self.raw); raw[offset] ^= 1 + archive.write_bytes(raw) + destination = self.root / "partial" + with self.assertRaises(ValueError): W["checked_archive"](archive, expected, destination) + self.assertFalse((destination / "INPUTS.json").exists()) + + def test_changed_or_extra_tree_inputs_rejected(self): + archive, expected = self.archive(); destination = self.root / "tree" + W["checked_archive"](archive, expected, destination) + file = destination / "runtime/node"; file.chmod(0o700); file.write_bytes(b"changed") + with self.assertRaises(ValueError): W["verify_tree"](destination, expected) + file.write_bytes(self.raw); file.chmod(0o500) + directory = destination / "runtime"; directory.chmod(0o700) + (directory / "shadow.so").write_bytes(b"unexpected") + directory.chmod(0o500) + with self.assertRaises(ValueError): W["verify_tree"](destination, expected) + + def proof_fixture(self): + selected = copy.deepcopy(self.files) + module_hashes = {name: self.digest for name in W["PROVEN_WEATHER_MODULES"]} + runtime = {name: self.digest for name in W["PROVEN_RUNTIME_FILES"]} + static = {"index.html": self.digest} + proof = dict(passed=True, scope=dict(synthetic_weather=True, core_executed=False), runtime=dict(sha256=runtime), + source=dict(built_tree_sha256=hashlib.sha256(json.dumps(static, sort_keys=True, separators=(",", ":")).encode()).hexdigest(), + built_files=1, built_bytes=len(self.raw), upstream="b" * 40, tree="c" * 40, + patch_sha256=self.digest, driver_sha256=self.digest, + overlay_sha256={"src/lib/services/volparossa/client.mjs": self.digest}, + weather_modules=module_hashes, + bridge_modules={name: self.digest for name in W["PROVEN_BRIDGE_MODULES"]}, + browser_helpers={name: self.digest for name in W["PROVEN_BROWSER_HELPERS"]})) + upstream = dict(commit="b" * 40, tree="c" * 40, license_sha256=self.digest, + upstream_patch_sha256=self.digest, weather_modules=copy.deepcopy(module_hashes)) + metadata = dict(runtime_sha256=copy.deepcopy(runtime), defaults_sha256=self.digest, autoconfig_sha256=self.digest) + build = dict(compiled=True, embedded_definitions=True, required_environment={}, source_pin_sha256=self.digest, + build_script_sha256=self.digest, original_sources_unchanged=True, artifacts=[dict( + path="build/eccodes/eccodes-build/bin/grib_ls", bytes=31328216, sha256=self.digest)]) + return selected, {"browser-smoke.json": proof, "upstream.json": upstream, + "volparossa-staging.json": metadata, "build-receipt.json": build} + + def check_proof(self, selected, records): + # No real source/binary capture or import. Only inert JSON-shaped values. + with patch.dict(W["source_proof"].__globals__, small_json=lambda path: records[path.name]): + return W["source_proof"](Path("/inert/weather"), Path("/inert/browser"), + Path("/inert/ui"), Path("/inert/runtime"), selected) + + def test_complete_provenance_binds_all_known_module_groups_without_source_execution(self): + self.check_proof(*self.proof_fixture()) + for name in (W["WEATHER"] + "src/provider-http.mjs", W["BROWSER"] + "scripts/smoke_privacy.py", + W["STAGE"] + "firefox-esr", W["WEATHER"] + "integrations/drizzli/smoke_browser.py", + W["STAGE"] + "volparossa.cfg", W["WEATHER"] + "integrations/drizzli/upstream.patch"): + selected, records = self.proof_fixture(); selected[name]["sha256"] = "f" * 64 + with self.subTest(name=name), self.assertRaises(ValueError): self.check_proof(selected, records) + + def test_empty_truncated_extra_or_malformed_provenance_maps_cannot_skip_bindings(self): + paths = [("browser-smoke.json", "source", "bridge_modules"), + ("browser-smoke.json", "source", "browser_helpers"), ("browser-smoke.json", "source", "weather_modules"), + ("browser-smoke.json", "runtime", "sha256"), ("upstream.json", "weather_modules"), + ("volparossa-staging.json", "runtime_sha256")] + for path in paths: + for fault in ("empty", "missing", "extra", "hash", "none", "list"): + selected, records = self.proof_fixture(); parent = records + for key in path[:-1]: parent = parent[key] + original = parent[path[-1]] + changed = dict(original) + if fault == "empty": changed = {} + elif fault == "missing": changed.pop(next(iter(changed))) + elif fault == "extra": changed["PRIVATE"] = self.digest + elif fault == "hash": changed[next(iter(changed))] = "PRIVATE" + elif fault == "none": changed = None + elif fault == "list": changed = list(changed) + parent[path[-1]] = changed + with self.subTest(path=path, fault=fault), self.assertRaises(ValueError): self.check_proof(selected, records) + + def test_missing_malformed_duplicate_or_unbounded_artifacts_are_closed_rejections(self): + selected, records = self.proof_fixture() + good = records["build-receipt.json"]["artifacts"][0] + bad = [[], None, {}, [None], [{}], [dict(good, bytes=True)], [dict(good, bytes=0)], + [dict(good, bytes=W["MAX_FILE"] + 1)], [dict(good, sha256="PRIVATE")], + [dict(good, path="different")], [dict(good, path="x" * 257)], [dict(good, extra="PRIVATE")], + [good, good], [dict(good, path=str(i)) for i in range(17)]] + for artifacts in bad: + selected, records = self.proof_fixture(); records["build-receipt.json"]["artifacts"] = artifacts + with self.subTest(artifacts=artifacts), self.assertRaises(ValueError): self.check_proof(selected, records) + + def test_invalid_archive_length_closes_descriptor_before_rejection(self): + source = io.BytesIO(b"x") + with patch.dict(W["checked_archive"].__globals__, open_checked=lambda *_: (source, SimpleNamespace(st_size=1))): + with self.assertRaises(ValueError): W["checked_archive"](Path("/inert/archive"), self.digest) + self.assertTrue(source.closed) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/integration/weather-client-bundle.py b/tests/integration/weather-client-bundle.py new file mode 100644 index 000000000..87be2d48e --- /dev/null +++ b/tests/integration/weather-client-bundle.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Freeze exact reviewed Weather/UI/runtime inputs; never execute captured files. + +The separately retained manifest hash authorizes transfer, not runtime execution. +Outputs live outside this immutable tree. The enclosing VM supplies the separately +verified core binary; no mutable package install or data download occurs here. +""" +import hashlib +import io +import json +import os +from pathlib import Path +import re +import runpy +import stat +import sys +import tarfile + +B = runpy.run_path(str(Path(__file__).with_name("browser-filter-bundle.py"))) +KIND = "volparossa-weather-client-inputs" +WEATHER = "volparossa-weather/" +BROWSER = "volparossa-browser/" +STAGE = BROWSER + "build/firefox-esr-smoke-v3/" +UI_STAGE = WEATHER + "build/drizzli-volparossa/" +STATIC = UI_STAGE + "build/" +NODE_SHA = "bc17c508ffeed0ec622934f9b7fa72f8e78da65350e63c3eceb56fa688aa5e12" +DECODER_SHA = "3a3b64581e6922db4287445f217f993f6967dea5fcc45ff66944b0bc883408f4" +PIN_SHA = "f136020562a6ff0d2a83b948189c31f839afb8a00ea87565469f38640fb0dc86" +BUILD_SHA = "5bcb6d58b34e5b45647feae836bfd086ff0adce0e30487430f5c50941f579827" +PROVEN_BRIDGE_MODULES = frozenset(("src/provider-http.mjs", "src/provider-core.mjs", "src/core-cache.mjs")) +PROVEN_BROWSER_HELPERS = frozenset(("scripts/smoke_compute_model.py", "scripts/smoke_privacy.py", "scripts/stage_firefox.py")) +PROVEN_RUNTIME_FILES = frozenset(("firefox-esr", "libxul.so", "omni.ja", "browser/omni.ja")) +PROVEN_WEATHER_MODULES = frozenset(("src/forecast.mjs", "src/provider.mjs")) +RUNTIME_FILES = frozenset(name for name in B["RUNTIME_FILES"] if not name.startswith("distribution/")) +WEATHER_FILES = frozenset(("LICENSE", "src/forecast.mjs", "src/ecmwf.mjs", "src/core-cache.mjs", + "src/provider.mjs", "src/provider-core.mjs", "src/provider-http.mjs", "scripts/forecast.mjs", + "scripts/serve-provider.mjs", "scripts/build_eccodes.py", "third_party/eccodes-sources.json", + "build/eccodes/build-receipt.json", "integrations/drizzli/upstream.json", + "integrations/drizzli/upstream.patch", "integrations/drizzli/browser-smoke.json", + "integrations/drizzli/smoke_browser.py", "integrations/drizzli/smoke_core_browser.py", + "integrations/drizzli/prepare-core-browser.mjs", + "third_party/eccodes/licenses/eccodes/LICENSE", "third_party/eccodes/licenses/eccodes/NOTICE", + "third_party/eccodes/licenses/ecbuild/LICENSE", "third_party/eccodes/licenses/ecbuild/NOTICE", + "third_party/eccodes/licenses/ecbuild/COPYING", "third_party/eccodes/licenses/libaec/LICENSE.txt", + "third_party/eccodes/licenses/expected/COPYING")) +BROWSER_FILES = frozenset(("defaults/privacy.json", "scripts/smoke_compute_model.py", + "scripts/smoke_privacy.py", "scripts/stage_firefox.py")) +CLIENT_FILES = frozenset(("LICENSE", *("src/lib/services/volparossa/" + name + ".mjs" + for name in ("client", "provider", "forecast")))) +FIXED = frozenset({*(WEATHER + name for name in WEATHER_FILES), + *(BROWSER + name for name in BROWSER_FILES), *(STAGE + name for name in RUNTIME_FILES), + *(UI_STAGE + name for name in CLIENT_FILES), "runtime/node", "decoder/grib_ls"}) +EXECUTABLES = frozenset({"runtime/node", "decoder/grib_ls", *(STAGE + name for name in + ("firefox-esr", "firefox-bin", "crashhelper", "crashreporter", "glxtest", "vaapitest", "pingsender"))}) +MAX_FILES = 512 +MAX_MANIFEST = 128 * 1024 +MAX_FILE = 256 * 1024 * 1024 +MAX_TOTAL = 512 * 1024 * 1024 +MAX_ARCHIVE = MAX_TOTAL + 2 * 1024 * 1024 +MAX_STATIC = 32 * 1024 * 1024 +require, hex64, encode = B["require"], B["hex64"], B["encode"] +fingerprint, open_checked, identity = B["fingerprint"], B["open_checked"], B["identity"] +header, read_exact, new_path = B["header"], B["read_exact"], B["new_path"] + + +def mode(name): + return 0o500 if name in EXECUTABLES else 0o400 + + +def bound(name): + if name.startswith(STAGE) or name in ("runtime/node", "decoder/grib_ls"): + return MAX_FILE + return 8 * 1024 * 1024 if name.startswith(STATIC) else 1024 * 1024 + + +def static_name(name): + return name.startswith(STATIC) and all(re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.@+-]{0,254}", part) + and part not in (".", "..") for part in name[len(STATIC):].split("/")) + + +def manifest(raw, expected): + require(hex64(expected) and 0 < len(raw) <= MAX_MANIFEST and hashlib.sha256(raw).hexdigest() == expected) + value = json.loads(raw, object_pairs_hook=B["unique_pairs"]) + require(type(value) is dict and set(value) == {"version", "kind", "files"} + and type(value["version"]) is int and value["version"] == 1 and value["kind"] == KIND + and type(value["files"]) is dict and FIXED <= set(value["files"]) + and 1 <= len(value["files"]) <= MAX_FILES and encode(value) == raw) + require(all(name in FIXED or static_name(name) for name in value["files"]) + and STATIC + "index.html" in value["files"]) + total = static_bytes = 0 + for name, item in value["files"].items(): + require(type(item) is dict and set(item) == {"bytes", "mode", "sha256"} + and type(item["bytes"]) is int and 0 < item["bytes"] <= bound(name) + and type(item["mode"]) is int and item["mode"] == mode(name) and hex64(item["sha256"])) + total += item["bytes"] + static_bytes += item["bytes"] if name.startswith(STATIC) else 0 + require(total <= MAX_TOTAL and static_bytes <= MAX_STATIC) + for name, expected_sha in (("runtime/node", NODE_SHA), ("decoder/grib_ls", DECODER_SHA), + (WEATHER + "third_party/eccodes-sources.json", PIN_SHA), (WEATHER + "scripts/build_eccodes.py", BUILD_SHA)): + require(value["files"][name]["sha256"] == expected_sha) + return value + + +def inventory(root, *, exact=None, normalized=False): + B["canonical"](root) + files, pending, count = {}, [root], 0 + while pending: + directory = pending.pop() + info = directory.lstat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid in (0, os.geteuid()) and not info.st_mode & 0o7022) + if normalized: + require(stat.S_IMODE(info.st_mode) == 0o500) + with os.scandir(directory) as children: + for child in children: + count += 1 + require(count <= MAX_FILES * 4) + path = Path(child.path) + relative = str(path.relative_to(root)) + require(len(path.relative_to(root).parts) <= 20) + if child.is_dir(follow_symlinks=False): + pending.append(path) + else: + require(child.is_file(follow_symlinks=False)) + files[relative] = path + require(len(files) <= MAX_FILES) + if exact is not None: + require(set(files) == set(exact)) + require(B["directories"](files) == {str(path.relative_to(root)) for path in root.rglob("*") if path.is_dir()}) + return files + + +def small_json(path): + source, info = open_checked(path, 1024 * 1024) + with source: + raw = source.read(1024 * 1024 + 1) + require(len(raw) == info.st_size and identity(os.fstat(source.fileno())) == identity(info)) + return json.loads(raw, object_pairs_hook=B["unique_pairs"]) + + +def proven_hashes(value, names): + # A truncated retained map must not silently skip a source-binding check. + require(type(value) is dict and set(value) == set(names) and all(hex64(digest) for digest in value.values())) + return value + + +def source_proof(weather, browser, ui, stage, selected): + """Check retained build provenance; do not run either project's validators.""" + proof = small_json(weather / "integrations/drizzli/browser-smoke.json") + upstream = small_json(weather / "integrations/drizzli/upstream.json") + require(proof["passed"] is True and proof["scope"]["synthetic_weather"] is True + and proof["scope"]["core_executed"] is False) + hashes = {name[len(STATIC):]: item["sha256"] for name, item in selected.items() if name.startswith(STATIC)} + digest = hashlib.sha256(json.dumps(hashes, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + require(digest == proof["source"]["built_tree_sha256"] and len(hashes) == proof["source"]["built_files"] + and sum(item["bytes"] for name, item in selected.items() if name.startswith(STATIC)) == proof["source"]["built_bytes"]) + require(proof["source"]["upstream"] == upstream["commit"] and proof["source"]["tree"] == upstream["tree"] + and selected[UI_STAGE + "LICENSE"]["sha256"] == upstream["license_sha256"] + and selected[WEATHER + "integrations/drizzli/upstream.patch"]["sha256"] + == upstream["upstream_patch_sha256"] == proof["source"]["patch_sha256"] + and selected[WEATHER + "integrations/drizzli/smoke_browser.py"]["sha256"] == proof["source"]["driver_sha256"]) + weather_modules = proven_hashes(upstream["weather_modules"], PROVEN_WEATHER_MODULES) + require(weather_modules == proven_hashes(proof["source"]["weather_modules"], PROVEN_WEATHER_MODULES)) + for module in ("forecast", "provider"): + digest = selected[WEATHER + "src/" + module + ".mjs"]["sha256"] + require(digest == selected[UI_STAGE + "src/lib/services/volparossa/" + module + ".mjs"]["sha256"] + == weather_modules["src/" + module + ".mjs"]) + require(selected[UI_STAGE + "src/lib/services/volparossa/client.mjs"]["sha256"] + == proof["source"]["overlay_sha256"]["src/lib/services/volparossa/client.mjs"]) + for relative, digest in proven_hashes(proof["source"]["bridge_modules"], PROVEN_BRIDGE_MODULES).items(): + require(selected[WEATHER + relative]["sha256"] == digest) + for relative, digest in proven_hashes(proof["source"]["browser_helpers"], PROVEN_BROWSER_HELPERS).items(): + require(selected[BROWSER + relative]["sha256"] == digest) + metadata = small_json(stage / "volparossa-staging.json") + runtime = proven_hashes(metadata["runtime_sha256"], PROVEN_RUNTIME_FILES) + require(runtime == proven_hashes(proof["runtime"]["sha256"], PROVEN_RUNTIME_FILES) + and metadata["defaults_sha256"] == selected[BROWSER + "defaults/privacy.json"]["sha256"] + and metadata["autoconfig_sha256"] == selected[STAGE + "volparossa.cfg"]["sha256"]) + for relative, digest in runtime.items(): + require(selected[STAGE + relative]["sha256"] == digest) + build = small_json(weather / "build/eccodes/build-receipt.json") + require(build["compiled"] is True and build["embedded_definitions"] is True + and build["required_environment"] == {} and build["source_pin_sha256"] == PIN_SHA + and build["build_script_sha256"] == BUILD_SHA and build["original_sources_unchanged"] is True) + require(type(build["artifacts"]) is list and 1 <= len(build["artifacts"]) <= 16) + artifacts = {} + for item in build["artifacts"]: + require(type(item) is dict and set(item) == {"path", "bytes", "sha256"} + and type(item["path"]) is str and len(item["path"]) <= 256 and item["path"] not in artifacts + and type(item["bytes"]) is int and 0 < item["bytes"] <= MAX_FILE and hex64(item["sha256"])) + artifacts[item["path"]] = item + require(artifacts.get("build/eccodes/eccodes-build/bin/grib_ls") + == dict(path="build/eccodes/eccodes-build/bin/grib_ls", bytes=31328216, sha256=DECODER_SHA)) + + +def capture(weather, ui, browser, stage, node, archive): + for root in (weather, ui, browser, stage): + B["canonical"](root) + inventory(stage, exact=RUNTIME_FILES) + sources = {**{WEATHER + name: weather / name for name in WEATHER_FILES}, + **{BROWSER + name: browser / name for name in BROWSER_FILES}, + **{STAGE + name: stage / name for name in RUNTIME_FILES}, + **{UI_STAGE + name: ui / name for name in CLIENT_FILES}, + **{STATIC + name: path for name, path in inventory(ui / "build").items()}, + "runtime/node": node, "decoder/grib_ls": weather / "build/eccodes/eccodes-build/bin/grib_ls"} + selected = {name: dict(**fingerprint(path, bound(name)), mode=mode(name)) for name, path in sorted(sources.items())} + for name in EXECUTABLES: + require(sources[name].stat().st_mode & 0o100) + raw = encode(dict(version=1, kind=KIND, files=selected)); expected = hashlib.sha256(raw).hexdigest() + manifest(raw, expected) + source_proof(weather, browser, ui, stage, selected) + new_path(archive) + with os.fdopen(os.open(archive, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "wb") as output: + for name in ("INPUTS.json", *sorted(selected)): + item = selected.get(name, dict(bytes=len(raw), mode=0o400, sha256=expected)) + output.write(header(name, item["bytes"], item["mode"])) + source, before = (io.BytesIO(raw), None) if name == "INPUTS.json" else open_checked(sources[name], item["bytes"]) + with source: + digest, remaining = hashlib.sha256(), item["bytes"] + while remaining: + chunk = source.read(min(65536, remaining)); require(bool(chunk)) + output.write(chunk); digest.update(chunk); remaining -= len(chunk) + require(not source.read(1) and digest.hexdigest() == item["sha256"]) + if before is not None: + require(identity(os.fstat(source.fileno())) == identity(before)) + output.write(b"\0" * (-item["bytes"] % 512)) + output.write(b"\0" * 1024); output.flush(); os.fsync(output.fileno()) + return checked_archive(archive, expected) + + +def checked_archive(archive, expected, root=None): + source, before = open_checked(archive, MAX_ARCHIVE) + with source: + require(before.st_size % 512 == 0) + first = read_exact(source, 512) + item = tarfile.TarInfo.frombuf(first, "utf-8", "strict") + require(item.name == "INPUTS.json" and item.type == tarfile.REGTYPE and 0 < item.size <= MAX_MANIFEST + and first == header("INPUTS.json", item.size, 0o400)) + raw = read_exact(source, item.size) + require(read_exact(source, -item.size % 512) == b"\0" * (-item.size % 512)) + selected = manifest(raw, expected) + if root is not None: + new_path(root); root.mkdir(mode=0o700) + for name, record in sorted(selected["files"].items()): + require(read_exact(source, 512) == header(name, record["bytes"], record["mode"])) + target = None + try: + if root is not None: + path = root / name; path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + target = os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "wb") + digest, remaining = hashlib.sha256(), record["bytes"] + while remaining: + chunk = read_exact(source, min(65536, remaining)); digest.update(chunk); remaining -= len(chunk) + if target is not None: target.write(chunk) + require(digest.hexdigest() == record["sha256"]) + require(read_exact(source, -record["bytes"] % 512) == b"\0" * (-record["bytes"] % 512)) + if target is not None: + target.flush(); os.fchmod(target.fileno(), record["mode"]) + finally: + if target is not None: target.close() + require(read_exact(source, 1024) == b"\0" * 1024 and not source.read(1)) + require(identity(os.fstat(source.fileno())) == identity(before)) + source.seek(0); archive_sha = hashlib.file_digest(source, "sha256").hexdigest() + require(identity(os.fstat(source.fileno())) == identity(before)) + if root is not None: + with os.fdopen(os.open(root / "INPUTS.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o400), "wb") as output: + output.write(raw) + for directory in sorted([root, *(p for p in root.rglob("*") if p.is_dir())], key=lambda p: len(p.parts), reverse=True): + directory.chmod(0o500) + return dict(manifest_sha256=expected, archive_sha256=archive_sha, files=len(selected["files"]), + bytes=sum(item["bytes"] for item in selected["files"].values())) + + +def verify_tree(root, expected): + source, info = open_checked(root / "INPUTS.json", MAX_MANIFEST, normalized=0o400) + with source: + raw = source.read(MAX_MANIFEST + 1); require(identity(os.fstat(source.fileno())) == identity(info)) + selected = manifest(raw, expected) + inventory(root, exact=set(selected["files"]) | {"INPUTS.json"}, normalized=True) + for name, item in selected["files"].items(): + require(fingerprint(root / name, item["bytes"], normalized=item["mode"]) + == {key: item[key] for key in ("bytes", "sha256")}) + return dict(manifest_sha256=expected, files=len(selected["files"]), runtime_executed=False) + + +def main(): + action, *args = sys.argv[1:] + if action == "capture" and len(args) == 6: + result = capture(*map(Path, args)) + elif action == "unpack" and len(args) == 3: + result = checked_archive(Path(args[0]), args[2], Path(args[1])) + elif action in ("verify", "verify-tree") and len(args) == 2: + result = (checked_archive if action == "verify" else verify_tree)(Path(args[0]), args[1]) + else: + raise ValueError("weather_client_bundle_rejected") + print(json.dumps(dict(version=1, action=action, **(result | {"runtime_executed": False})), sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, TypeError, SyntaxError, tarfile.TarError, UnicodeError): + print('{"runtime_executed":false,"failure":"weather_client_bundle_rejected"}') + raise SystemExit(1) From c1418de71b0bb3d4d73600d84277d87df49189ac Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 02:31:12 +0200 Subject: [PATCH 13/49] test: join protected Weather delivery to the real Drizzli UI --- docs/IMPLEMENTATION_STATUS.md | 36 +- .../content-provider-site-smoke.sh | 5 + tests/integration/content-provider-smoke.sh | 2 +- tests/integration/kvm-alpha-topology.sh | 52 +- tests/integration/run-alpha-topology-vm.sh | 107 +++- .../test-weather-client-delivery-shell.py | 187 ++++++ .../test-weather-client-delivery.py | 270 +++++++++ .../integration/test-weather-client-runner.py | 202 +++++++ tests/integration/weather-client-delivery.py | 570 ++++++++++++++++++ tests/integration/weather-client-delivery.sh | 255 ++++++++ 10 files changed, 1666 insertions(+), 20 deletions(-) create mode 100644 tests/integration/test-weather-client-delivery-shell.py create mode 100644 tests/integration/test-weather-client-delivery.py create mode 100644 tests/integration/test-weather-client-runner.py create mode 100644 tests/integration/weather-client-delivery.py create mode 100644 tests/integration/weather-client-delivery.sh diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 439414799..a3bdf1864 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -214,8 +214,12 @@ successful joined trial remain required; raw browser logs are not exported. The Browser driver records closed initialization/check subphases and error categories, while retaining the original failure through cleanup. The -core validates those optional failure-only fields before exporting them; 20 -joined-proof/export tests and 13 runner tests pass. The driver also corrects its +core validates those optional failure-only fields before exporting them. The +latest local diagnostic changes distinguish WebDriver errors from malformed +responses, framing failures, EOF, timeout and transport failures. Only fixed +error codes and bounded locations in the pinned runtime may be retained; no +raw response, error message or browser log is exported. The 49 Browser driver, +23 joined-proof/export and 13 runner tests pass. The driver also corrects its fresh-profile check to treat an absent uBO `importedLists` key as the upstream default empty list. Explicit malformed values and unexpected lists remain rejected. This is a verified fixture defect, not proof of VM4's precise cause; @@ -235,9 +239,14 @@ The retained `browser-filter-delivery-05.KDUpgD` collection records broker reapi profile/input removal, zero remaining owned topology objects and matching guest-root network hashes `f1146383d82ba978c0546b4e0f02f8268aaf25cb85b727c217387740579510fd`. -The VM and temporary disk were removed. The next diagnostic step is an isolated -session-start reproduction, not another full network trial without a diagnosed -change. Joined filter delivery remains incomplete. +The VM and temporary disk were removed. A subsequent loopback-only session-start +probe using the same frozen runtime, original preferences and session parameters +succeeded and shut Firefox down normally. Its receipt is retained in +`browser-session-start-01.0EIlLL` with SHA-256 +`685683e29d023ac2c0cc5e30778164700feb875e8b5acc96ba2ebc8af11c3559`. +It did not run a core, broker or full VM. The VM-specific failure remains +unresolved; the more precise diagnostics have not yet been exercised in a new +joined run. Joined filter delivery remains incomplete. ### Weather provider and Drizz.li web integration @@ -265,6 +274,23 @@ The final repository CI and CodeQL checks pass. A real core-cache publication reaching the browser remains unproven; the synthetic trial does not establish live overlay or weather-origin delivery. +The next local proof fixture is prepared: acquire fresh public ECMWF data, +publish it to peers, require a cold protected core transfer, and display and +refresh the same publication through the production bridge in Drizz.li. Origin +HTTPS acquisition is distinct from the subsequent overlay transfer. Explicit +bundle selection, immutable input checks, private disk scratch and the existing +4 GiB guest plus 2 GiB host-memory preflight are wired into the disposable runner; +the nine runner and closed-export tests pass. The separate real-core browser +driver has four Node preparation tests and eight inert Python tests, including +exact display/refresh bindings to the cold publication. Ten bundle tests, ten +joined-evidence tests and eleven inert shell/cleanup tests pass. The immutable +168-file input bundle was captured and +independently verified without executing a runtime; its manifest is +`e77f3240ab93a836f3ef19393909e4d0809a2d1a75fd143613e6a476f858c905` +and archive SHA-256 is +`4843f20f8f47db5f82f5902217514c184dadc31d3e5fe7c19ca01238b65add68`. +This is preparation, not a successful end-to-end Weather run. + Overmorrow, additional location selection, precipitation and air quality remain required work. Unsupported fields are explicitly unavailable, never substituted with zero values. diff --git a/tests/integration/content-provider-site-smoke.sh b/tests/integration/content-provider-site-smoke.sh index 99f79d93f..92a911560 100644 --- a/tests/integration/content-provider-site-smoke.sh +++ b/tests/integration/content-provider-site-smoke.sh @@ -161,6 +161,11 @@ content_provider_site_run() { -- test -r "$site_client_cache"; then fail CONTENT_SITE_CONSUMER_CACHE_EXPOSED; fi # Preserve the two-publication site proof above before adding a third tiny publication. content_provider_filter_cold_run + if [ -n "${weather_client_bundle:-}" ]; then + # Preserve the existing publication-count proof. Acquire/publish the + # separate public forecast only now, before this route is disconnected. + weather_client_delivery_run + fi site_cache_before=$(stat -Lc '%d:%i' "$site_client_cache") || fail CONTENT_SITE_CACHE_IDENTITY_UNAVAILABLE content_provider_site_cache_only_run site_cache_after=$(stat -Lc '%d:%i' "$site_client_cache") || fail CONTENT_SITE_CACHE_IDENTITY_UNAVAILABLE diff --git a/tests/integration/content-provider-smoke.sh b/tests/integration/content-provider-smoke.sh index 979ff4234..64fa89834 100644 --- a/tests/integration/content-provider-smoke.sh +++ b/tests/integration/content-provider-smoke.sh @@ -82,7 +82,7 @@ content_provider_control_underlay() { content_provider_start_control_observer() { provider_control_prefix=$1 - case $provider_control_prefix in content-provider-*) ;; *) return 1 ;; esac + case $provider_control_prefix in content-provider-*|weather-client-delivery-control) ;; *) return 1 ;; esac [ -z "$PROVIDER_CONTROL_PID" ] || return 1 ip netns exec "$provider_control_ns" python3 "$WORK/bin/privacy-observer.py" \ content-control "$WORK/$provider_control_prefix.json" \ diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 744af9c67..ac9982f82 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -61,6 +61,9 @@ code_manifest_sha256= browser_filter_bundle= browser_filter_manifest_sha256= browser_filter_directory= +weather_client_bundle= +weather_client_manifest_sha256= +weather_client_directory= # Only explicit scenario arguments may authorize the staged Code inputs. COOPERATIVE_CODE_BUNDLE= COOPERATIVE_CODE_MANIFEST_SHA256= @@ -74,6 +77,7 @@ usage() { ' --scenario signal-backup also runs the exact pinned native Signal backup test' \ ' --scenario agent-cooperative-code requires --code-bundle DIRECTORY --code-manifest-sha256 HEX' \ ' --scenario content-provider accepts --browser-filter-bundle ARCHIVE --browser-filter-manifest-sha256 HEX' \ + ' or --weather-client-bundle ARCHIVE --weather-client-manifest-sha256 HEX (not both)' \ ' --mpquic PATH --expected-commit SHA [--scenario alpha|reciprocity|reciprocity-private-dns|local-link|mixed-link|mpquic-growth|mptcp-growth|mptcp-refill|sharing|download-sharing|wifi-link|uplink-link|crash-recovery|content|content-message|content-https|content-provider|content-replication|content-repair|content-mailbox|content-custody|private-storage-peer|private-storage-replicas|signal-backup|private-storage-handoff|private-storage-fragments|agent-artifact|agent-train-cycle|agent-train-loop|agent-artifact-quarantine|agent-jobs|agent-jobs-loss|agent-jobs-follow|agent-jobs-peer-recovery|agent-jobs-ready-queue|agent-jobs-package-queue|agent-public-task|agent-public-document|agent-public-collection|agent-public-network-sources|agent-task-graph|agent-ready-dag|agent-model-planning|agent-model-task-graph|agent-successor-serving|agent-active-recovery|agent-adapter-aggregation|agent-autonomous-aggregation|agent-policy-assessment|dns-cache]' } @@ -879,6 +883,16 @@ while [ "$#" -gt 0 ]; do browser_filter_manifest_sha256=$2 shift ;; + --weather-client-bundle) + if [ "$#" -lt 2 ] || [ -n "$weather_client_bundle" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + weather_client_bundle=$2 + shift + ;; + --weather-client-manifest-sha256) + if [ "$#" -lt 2 ] || [ -n "$weather_client_manifest_sha256" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + weather_client_manifest_sha256=$2 + shift + ;; --code-manifest-sha256) if [ "$#" -lt 2 ] || [ -n "$code_manifest_sha256" ]; then usage >&2; exit 64; fi code_manifest_sha256=$2 @@ -899,7 +913,7 @@ done if [ "$mode" = preview ]; then if [ "$approval" != no ] \ - || [ -n "$source_directory$binary_directory$mpquic_binary$output_directory$expected_commit$code_bundle$code_manifest_sha256$browser_filter_bundle$browser_filter_manifest_sha256" ]; then + || [ -n "$source_directory$binary_directory$mpquic_binary$output_directory$expected_commit$code_bundle$code_manifest_sha256$browser_filter_bundle$browser_filter_manifest_sha256$weather_client_bundle$weather_client_manifest_sha256" ]; then usage >&2 exit 64 fi @@ -943,6 +957,13 @@ if [ -n "$browser_filter_bundle$browser_filter_manifest_sha256" ]; then case $browser_filter_manifest_sha256 in ''|*[!0-9a-f]*) exit 64 ;; esac [ "${#browser_filter_manifest_sha256}" -eq 64 ] || exit 64 fi +if [ -n "$weather_client_bundle$weather_client_manifest_sha256" ]; then + [ "$scenario" = content-provider ] && [ -z "$browser_filter_bundle$browser_filter_manifest_sha256" ] || exit 64 + case $weather_client_bundle in /*) ;; *) exit 64 ;; esac + case $weather_client_bundle in /|*/|*//*|*/../*|*/./*|*/..|*/.|*[[:cntrl:]]*) exit 64 ;; esac + case $weather_client_manifest_sha256 in ''|*[!0-9a-f]*) exit 64 ;; esac + [ "${#weather_client_manifest_sha256}" -eq 64 ] || exit 64 +fi [ "$(id -u)" -eq 0 ] || { printf '%s\n' 'execution requires root inside KVM' >&2; exit 77; } [ "$(sed -n '1{s/\..*$//;p;}' /etc/debian_version)" = 13 ] \ || { printf '%s\n' 'execution requires Debian 13' >&2; exit 77; } @@ -975,6 +996,21 @@ if [ -n "$browser_filter_bundle" ]; then exit 69 fi fi +if [ -n "$weather_client_bundle" ]; then + [ -f "$weather_client_bundle" ] && [ ! -L "$weather_client_bundle" ] \ + && [ "$(readlink -e -- "$weather_client_bundle")" = "$weather_client_bundle" ] \ + && [ "$(stat -Lc '%a' "$weather_client_bundle")" = 600 ] || exit 69 + weather_client_input_uid=$(stat -Lc '%u' "$weather_client_bundle") + if [ "$weather_client_input_uid" -eq 0 ]; then + python3 -B "$source_directory/tests/integration/weather-client-bundle.py" verify \ + "$weather_client_bundle" "$weather_client_manifest_sha256" || exit 69 + elif [ "$weather_client_input_uid" -eq "$(id -u vpci)" ]; then + runuser -u vpci -- python3 -B "$source_directory/tests/integration/weather-client-bundle.py" verify \ + "$weather_client_bundle" "$weather_client_manifest_sha256" || exit 69 + else + exit 69 + fi +fi if [ "$agent_cooperative_code" = yes ]; then if [ -d "$code_bundle" ] && [ ! -L "$code_bundle" ] \ && [ "$(readlink -e -- "$code_bundle")" = "$code_bundle" ] \ @@ -1584,6 +1620,9 @@ capture_host_state() { } copy_artifacts() { + if [ -n "$weather_client_bundle" ] && command -v weather_client_delivery_copy_artifacts >/dev/null 2>&1; then + weather_client_delivery_copy_artifacts || return 1 + fi if [ -n "$browser_filter_bundle" ] && command -v browser_filter_delivery_copy_artifacts >/dev/null 2>&1; then browser_filter_delivery_copy_artifacts || return 1 fi @@ -2241,6 +2280,9 @@ cleanup() { if [ -n "$browser_filter_bundle" ] && command -v browser_filter_delivery_cleanup >/dev/null 2>&1; then browser_filter_delivery_cleanup || original_status=1 fi + if [ -n "$weather_client_bundle" ] && command -v weather_client_delivery_cleanup >/dev/null 2>&1; then + weather_client_delivery_cleanup || original_status=1 + fi if [ "$scenario" = mpquic-growth ] && command -v mpquic_growth_cleanup >/dev/null 2>&1; then mpquic_growth_cleanup || original_status=1 fi @@ -2643,6 +2685,10 @@ if [ "$scenario" = content-provider ]; then . "$source_directory/tests/integration/content-provider-filter-smoke.sh" # shellcheck source=tests/integration/content-provider-adaptive-smoke.sh . "$source_directory/tests/integration/content-provider-adaptive-smoke.sh" + if [ -n "$weather_client_bundle" ]; then + # shellcheck source=tests/integration/weather-client-delivery.sh + . "$source_directory/tests/integration/weather-client-delivery.sh" + fi fi if [ "$scenario" = content-replication ]; then # shellcheck source=tests/integration/content-replication-smoke.sh @@ -3069,7 +3115,7 @@ mpquic_binary=$WORK/bin/volparossa-mpquic [ "$scenario" != agent-jobs ] || agent_jobs_prepare # BEGIN browser-filter namespace-parent preflight -if [ -n "$browser_filter_bundle" ]; then +if [ -n "$browser_filter_bundle${weather_client_bundle:-}" ]; then PHASE=browser-filter-netns-parent printf '%s\n' 'Browser fixture: create /run/netns root:root mode 0755 only if absent; preserve a safe existing parent; never change private cache permissions.' # This is after the root/Debian/systemd/KVM gates, inside the disposable @@ -6141,7 +6187,7 @@ start_privacy_observers() { [ "$scenario" = content ] || [ "$scenario" = content-message ] || return 1 ;; content-https-complete-privacy|content-https-missing-privacy) [ "$scenario" = content-https ] || return 1 ;; - content-provider-privacy|content-provider-adaptive-privacy|content-provider-adaptive-https-privacy|content-provider-https-complete-privacy|content-provider-https-missing-privacy|content-provider-https-baseline-privacy|content-provider-https-origin-only-privacy|content-provider-https-auto-privacy|content-provider-https-digest-origin-only-privacy|content-provider-https-digest-peers-first-privacy|content-provider-https-limited-origin-only-privacy|content-provider-https-limited-peers-first-privacy|content-provider-https-limited-auto-privacy|content-provider-https-checksum-origin-only-privacy|content-provider-https-checksum-peers-first-privacy|content-provider-https-checksum-mismatch-privacy|content-provider-user-privacy|content-provider-named-privacy|content-provider-site-privacy|content-provider-site-cache-only-privacy|content-provider-filter-cold-privacy|content-provider-filter-warm-privacy) + content-provider-privacy|content-provider-adaptive-privacy|content-provider-adaptive-https-privacy|content-provider-https-complete-privacy|content-provider-https-missing-privacy|content-provider-https-baseline-privacy|content-provider-https-origin-only-privacy|content-provider-https-auto-privacy|content-provider-https-digest-origin-only-privacy|content-provider-https-digest-peers-first-privacy|content-provider-https-limited-origin-only-privacy|content-provider-https-limited-peers-first-privacy|content-provider-https-limited-auto-privacy|content-provider-https-checksum-origin-only-privacy|content-provider-https-checksum-peers-first-privacy|content-provider-https-checksum-mismatch-privacy|content-provider-user-privacy|content-provider-named-privacy|content-provider-site-privacy|content-provider-site-cache-only-privacy|content-provider-filter-cold-privacy|content-provider-filter-warm-privacy|weather-client-delivery-privacy) [ "$scenario" = content-provider ] || return 1 ;; content-message-publication-privacy) [ "$scenario" = content-message ] || return 1 ;; diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 9d1b9833f..533e0b7ef 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -23,6 +23,8 @@ code_bundle= code_manifest_sha256= browser_filter_bundle= browser_filter_manifest_sha256= +weather_client_bundle= +weather_client_manifest_sha256= guest_memory_for_scenario() { # These fixtures co-locate two 360M providers in one guest. Production @@ -45,6 +47,7 @@ usage() { ' agent-cooperative-code also requires --code-bundle DIRECTORY --code-manifest-sha256 SHA256.' \ ' --native-runtime PATH is required for agent-native-coding or agent-native-editor (exact verified bundle).' \ ' content-provider optionally accepts --browser-filter-bundle ARCHIVE --browser-filter-manifest-sha256 SHA256.' \ + ' Or --weather-client-bundle ARCHIVE --weather-client-manifest-sha256 SHA256 (one joined client per run).' \ ' Native-editor, cooperative-code and content-provider accept --host-tools-directory PATH (verified workspace-only QEMU).' } @@ -519,6 +522,16 @@ while [ "$#" -gt 0 ]; do browser_filter_manifest_sha256=$2 shift ;; + --weather-client-bundle) + if [ "$#" -lt 2 ] || [ -n "$weather_client_bundle" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + weather_client_bundle=$2 + shift + ;; + --weather-client-manifest-sha256) + if [ "$#" -lt 2 ] || [ -n "$weather_client_manifest_sha256" ] || [ -z "$2" ]; then usage >&2; exit 64; fi + weather_client_manifest_sha256=$2 + shift + ;; --code-manifest-sha256) if [ "$#" -lt 2 ] || [ -n "$code_manifest_sha256" ]; then usage >&2; exit 64; fi code_manifest_sha256=$2 @@ -573,7 +586,7 @@ done if [ "$mode" = preview ]; then if [ "$approval" != no ] \ - || [ -n "$image_path$mpquic_path$package_path$native_runtime_path$host_tools_directory$output_directory$expected_commit$code_bundle$code_manifest_sha256$browser_filter_bundle$browser_filter_manifest_sha256" ]; then + || [ -n "$image_path$mpquic_path$package_path$native_runtime_path$host_tools_directory$output_directory$expected_commit$code_bundle$code_manifest_sha256$browser_filter_bundle$browser_filter_manifest_sha256$weather_client_bundle$weather_client_manifest_sha256" ]; then usage >&2 exit 64 fi @@ -608,6 +621,13 @@ if [ -n "$browser_filter_bundle$browser_filter_manifest_sha256" ]; then case $browser_filter_manifest_sha256 in ''|*[!0-9a-f]*) exit 64 ;; esac [ "${#browser_filter_manifest_sha256}" -eq 64 ] || exit 64 fi +if [ -n "$weather_client_bundle$weather_client_manifest_sha256" ]; then + [ "$scenario" = content-provider ] && [ -z "$browser_filter_bundle$browser_filter_manifest_sha256" ] || exit 64 + case $weather_client_bundle in /*) ;; *) exit 64 ;; esac + case $weather_client_bundle in /|*/|*//*|*/../*|*/./*|*/..|*/.|*[[:cntrl:]]*) exit 64 ;; esac + case $weather_client_manifest_sha256 in ''|*[!0-9a-f]*) exit 64 ;; esac + [ "${#weather_client_manifest_sha256}" -eq 64 ] || exit 64 +fi case $expected_commit in ''|*[!0-9a-f]*) exit 64 ;; esac case ${#expected_commit} in 40|64) ;; *) exit 64 ;; esac [ "$(id -u)" -ne 0 ] || { printf '%s\n' 'VM runner must remain unprivileged' >&2; exit 77; } @@ -713,17 +733,18 @@ elif [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperati [ "$(readlink -f -- "$output_directory")" = "$output_directory" ] || exit 64 RUN_DIRECTORY_PARENT=$(dirname -- "$output_directory") RUN_DIRECTORY_PREFIX=cooperative-code-kvm -elif [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle" ]; then +elif [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle${weather_client_bundle:-}" ]; then # The real Browser join builds Rust in this 4 GiB guest. A growing qcow2 # overlay on tmpfs otherwise competes directly with the guest's RAM. # Keep scratch beside validated workspace evidence, never in its export. [ "$(readlink -f -- "$output_directory")" = "$output_directory" ] || exit 64 RUN_DIRECTORY_PARENT=$(dirname -- "$output_directory") RUN_DIRECTORY_PREFIX=browser-filter-kvm + [ -z "${weather_client_bundle:-}" ] || RUN_DIRECTORY_PREFIX=weather-client-kvm runner_scratch_filesystem=$(stat -f -c '%T' -- "$RUN_DIRECTORY_PARENT") || exit 69 case $runner_scratch_filesystem in ''|tmpfs|ramfs) - printf '%s\n' 'Browser filter VM scratch requires non-memory-backed workspace storage' >&2 + printf '%s\n' 'Joined client VM scratch requires non-memory-backed workspace storage' >&2 exit 69 ;; esac fi @@ -796,6 +817,17 @@ if [ -n "$browser_filter_bundle" ]; then printf '%s\n' 'Additional proof: exact Browser/ESR/signed-addons bundle, real core filter broker, uBO engine probes and revocation.' fi CODE_ARCHIVE=$RUN_DIRECTORY/cooperative-code.tar +WEATHER_CLIENT_ARCHIVE=$RUN_DIRECTORY/weather-client.tar +WEATHER_CLIENT_ARCHIVE_SHA256=none +WEATHER_CLIENT_MANIFEST_SHA256=none +if [ -n "$weather_client_bundle" ]; then + python3 -B "$HERE/weather-client-bundle.py" verify "$weather_client_bundle" "$weather_client_manifest_sha256" + install -m 0600 "$weather_client_bundle" "$WEATHER_CLIENT_ARCHIVE" + python3 -B "$HERE/weather-client-bundle.py" verify "$WEATHER_CLIENT_ARCHIVE" "$weather_client_manifest_sha256" + WEATHER_CLIENT_ARCHIVE_SHA256=$(sha256sum "$WEATHER_CLIENT_ARCHIVE" | awk '{ print $1 }') + WEATHER_CLIENT_MANIFEST_SHA256=$weather_client_manifest_sha256 + printf '%s\n' 'Additional proof: fresh public ECMWF HTTPS acquisition, protected peer transfer and real Drizz.li UI; origin acquisition is not an overlay claim.' +fi CODE_ARCHIVE_SHA256=none CODE_MANIFEST_SHA256=none if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then @@ -1068,9 +1100,14 @@ BROWSER_FILTER_NAMES = {f"browser-filter-delivery-{suffix}.json" for suffix in ( BROWSER_FILTER_HELPERS = ("browser-filter-delivery.py", "browser-filter-bundle.py", "content-provider-filter-smoke.py", "content-provider-site-smoke.py", "content-network-smoke.py", "content-provider-https-smoke.py") +WEATHER_CLIENT_NAMES = {f"weather-client-delivery-{suffix}.json" for suffix in ( + "evidence", "acquisition", "original-report", "tree-before", "tree-after", + "cache-before", "cache-after", "input-cleanup")} +WEATHER_CLIENT_HELPERS = ("weather-client-delivery.py", "weather-client-bundle.py", + "browser-filter-bundle.py", "content-provider-https-smoke.py", "content-network-smoke.py") -def browser_filter_helper(work): +def joined_client_helper(work, names, entrypoint): """Trust only the topology's immutable root-staged validation closure.""" try: directory = work / "bin" @@ -1080,17 +1117,39 @@ def browser_filter_helper(work): info = parent.lstat() if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022: return None - for name in BROWSER_FILTER_HELPERS: + for name in names: path = directory / name info = path.lstat() if (not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_nlink != 1 or info.st_mode & 0o022 or not 0 < info.st_size <= 262144): return None - return directory / "browser-filter-delivery.py" + return directory / entrypoint except OSError: return None +def browser_filter_helper(work): + return joined_client_helper(work, BROWSER_FILTER_HELPERS, "browser-filter-delivery.py") + + +def weather_client_helper(work): + return joined_client_helper(work, WEATHER_CLIENT_HELPERS, "weather-client-delivery.py") + + +def weather_client_candidates(root, label, helper, target): + # No raw forecast, request/config, CLI receipt or process log is exported. + validated = Path(tempfile.mkdtemp(prefix="weather-client-validated.", dir=target)) + try: + subprocess.run(["/usr/bin/python3", "-I", "-B", str(helper), "copy-artifacts", str(root), + str(validated), str(os.getuid()), str(os.getgid())], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + cwd="/", env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, timeout=3, check=True) + except (OSError, subprocess.SubprocessError): + return [] + return [(validated / name, f"{label}/{name}") for name in sorted(WEATHER_CLIENT_NAMES) + if (validated / name).is_file() and not (validated / name).is_symlink()] + + def browser_filter_candidates(root, label, helper, target): # Reuse the closed recursive JSON validator, never import the worker's # Browser tree or copy its logs. A failed/missing validator yields no files. @@ -1216,6 +1275,12 @@ def collect(home, opt, revision, scenario, guest_status, for root, label in roots: if root.is_dir() and not root.is_symlink(): candidates.extend(browser_filter_candidates(root, label, helper, target)) + helper = next((found for root, _ in roots[1:] + if (found := weather_client_helper(root)) is not None), None) + if helper is not None: + for root, label in roots: + if root.is_dir() and not root.is_symlink(): + candidates.extend(weather_client_candidates(root, label, helper, target)) for root, label in roots: if root.is_symlink() or not root.is_dir(): continue @@ -1382,7 +1447,7 @@ def collect(home, opt, revision, scenario, guest_status, if observed is None: continue data, original_size = observed - if path.name in BROWSER_FILTER_NAMES and original_size > len(data): + if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES and original_size > len(data): continue # Preserve complete validated original bytes, never a JSON tail. if scenario == "browser-network" and path.name in browser_names and original_size > len(data): # A truncated JSON tail is not reconstructible structured evidence. @@ -1456,6 +1521,8 @@ code_archive_sha256=none code_manifest_sha256=none browser_filter_archive_sha256=none browser_filter_manifest_sha256=none +weather_client_archive_sha256=none +weather_client_manifest_sha256=none # Native and cooperative bundles share positions only in disjoint scenarios. case $scenario in agent-native-coding|agent-native-editor) @@ -1468,7 +1535,13 @@ case $scenario in code_manifest_sha256=${7:-none} ;; content-provider) - if [ "$#" -eq 7 ]; then + if [ "$#" -eq 8 ]; then + [ "$6" = weather-client ] || exit 64 + weather_client_archive_sha256=$7 + weather_client_manifest_sha256=$8 + case $weather_client_archive_sha256:$weather_client_manifest_sha256 in *[!0-9a-f:]*) exit 64 ;; esac + [ "${#weather_client_archive_sha256}" -eq 64 ] && [ "${#weather_client_manifest_sha256}" -eq 64 ] || exit 64 + elif [ "$#" -eq 7 ]; then browser_filter_archive_sha256=$6 browser_filter_manifest_sha256=$7 case $browser_filter_archive_sha256:$browser_filter_manifest_sha256 in *[!0-9a-f:]*) exit 64 ;; esac @@ -1604,7 +1677,7 @@ fi if [ "$scenario" = image-snapshot ] || [ "$scenario" = cloud-private-file ] || [ "$scenario" = cloud-private-upload ]; then sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends gpg gpg-agent gpgconf tar fi -if [ "$scenario" = agent-cooperative-browser ] || [ "$scenario" = agent-cooperative-browser-discovered ] || [ "$scenario" = browser-network ] || [ "$scenario" = cloud-private-file ] || [ "$scenario" = cloud-private-upload ] || [ "$browser_filter_manifest_sha256" != none ]; then +if [ "$scenario" = agent-cooperative-browser ] || [ "$scenario" = agent-cooperative-browser-discovered ] || [ "$scenario" = browser-network ] || [ "$scenario" = cloud-private-file ] || [ "$scenario" = cloud-private-upload ] || [ "$browser_filter_manifest_sha256" != none ] || [ "$weather_client_manifest_sha256" != none ]; then # Exact pinned ESR is extracted separately, not installed; these are its # ordinary Debian dependencies plus the isolated browser/HTTPS fixture tools. sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ @@ -1638,6 +1711,11 @@ if [ "$browser_filter_manifest_sha256" != none ]; then printf '%s /home/vpci/browser-filter.tar\n' "$browser_filter_archive_sha256" | sha256sum --check --strict - python3 -B tests/integration/browser-filter-bundle.py verify /home/vpci/browser-filter.tar "$browser_filter_manifest_sha256" fi +if [ "$weather_client_manifest_sha256" != none ]; then + guest_phase weather-client-inputs + printf '%s /home/vpci/weather-client.tar\n' "$weather_client_archive_sha256" | sha256sum --check --strict - + python3 -B tests/integration/weather-client-bundle.py verify /home/vpci/weather-client.tar "$weather_client_manifest_sha256" +fi if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then guest_phase code-inputs case $code_archive_sha256:$code_manifest_sha256 in *[!0-9a-f:]*) exit 64 ;; esac @@ -1778,6 +1856,9 @@ if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative elif [ "$browser_filter_manifest_sha256" != none ]; then set -- ./tests/integration/kvm-alpha-topology.sh --browser-filter-bundle /home/vpci/browser-filter.tar \ --browser-filter-manifest-sha256 "$browser_filter_manifest_sha256" +elif [ "$weather_client_manifest_sha256" != none ]; then + set -- ./tests/integration/kvm-alpha-topology.sh --weather-client-bundle /home/vpci/weather-client.tar \ + --weather-client-manifest-sha256 "$weather_client_manifest_sha256" else set -- env "$@" ./tests/integration/kvm-alpha-topology.sh fi @@ -1873,7 +1954,7 @@ set -- -no-reboot case $scenario in wifi-mesh|wifi-link) set -- ;; esac guest_memory_mib=$(guest_memory_for_scenario) runner_reserve_mib=1024 -if [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle" ]; then +if [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle${weather_client_bundle:-}" ]; then runner_reserve_mib=2048 fi if [ "$guest_memory_mib" -gt 4096 ] || [ "$runner_reserve_mib" -eq 2048 ]; then @@ -2002,6 +2083,7 @@ done ssh_base sudo -n cloud-init status --wait >/dev/null scp_to "$SOURCE_ARCHIVE" /home/vpci/source.tar.gz if [ -n "$browser_filter_bundle" ]; then scp_to "$BROWSER_FILTER_ARCHIVE" /home/vpci/browser-filter.tar; fi +if [ -n "$weather_client_bundle" ]; then scp_to "$WEATHER_CLIENT_ARCHIVE" /home/vpci/weather-client.tar; fi if [ -n "$native_runtime_path" ]; then scp_to "$native_runtime_path" /home/vpci/native-runtime.tar.gz; fi if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then scp_to "$CODE_ARCHIVE" /home/vpci/cooperative-code.tar; fi if [ -n "$mpquic_path" ]; then scp_to "$mpquic_path" /home/vpci/volparossa-mpquic; fi @@ -2046,7 +2128,10 @@ driver_time_bound=2400s [ "$scenario" != signal-backup-fragments ] || driver_time_bound=7200s case $scenario in content-provider) - if [ -n "$browser_filter_bundle" ]; then + if [ -n "$weather_client_bundle" ]; then + ssh_bounded "$driver_time_bound" /home/vpci/guest-driver.sh "$expected_commit" "$SOURCE_SHA256" \ + "$MPQUIC_SHA256" "$PACKAGE_SHA256" "$scenario" weather-client "$WEATHER_CLIENT_ARCHIVE_SHA256" "$WEATHER_CLIENT_MANIFEST_SHA256" + elif [ -n "$browser_filter_bundle" ]; then ssh_bounded "$driver_time_bound" /home/vpci/guest-driver.sh "$expected_commit" "$SOURCE_SHA256" \ "$MPQUIC_SHA256" "$PACKAGE_SHA256" "$scenario" "$BROWSER_FILTER_ARCHIVE_SHA256" "$BROWSER_FILTER_MANIFEST_SHA256" else diff --git a/tests/integration/test-weather-client-delivery-shell.py b/tests/integration/test-weather-client-delivery-shell.py new file mode 100644 index 000000000..b731f1a3a --- /dev/null +++ b/tests/integration/test-weather-client-delivery-shell.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert shell contracts: no real /nonexistent, VM, browser or network use.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +SOURCE = HERE / "weather-client-delivery.sh" + + +class WeatherShell(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="weather-shell-test-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.home = self.root / "account-home" + # Rebind every occurrence in memory before sourcing. All shell file + # mutations are then restricted to this test's temporary directory. + self.source = SOURCE.read_text().replace("/nonexistent", str(self.home)).replace("systemd-detect-virt", "test_virt") + self.assertNotIn("/nonexistent", self.source) + + def call(self, body, **extra): + environment = dict(PATH="/usr/bin:/bin", TEST_HOME_PATH=str(self.home), + TEST_VIRT="kvm", TEST_HOME_MODE="755", WORK=str(self.root), + WORKER_UID="985", site_control_gid="987") + environment.update(extra) + mocks = r''' +id() { printf '0\n'; } +test_virt() { printf '%s\n' "$TEST_VIRT"; } +sed() { printf 'systemd\n'; } +getent() { printf 'fixture:x:985:987::%s:/usr/sbin/nologin\n' "$TEST_HOME_PATH"; } +stat() { + if [ "$2" = '%u:%g:%a' ]; then printf '0:0:%s\n' "$TEST_HOME_MODE"; + else /usr/bin/stat "$@"; fi +} +''' + return subprocess.run(["/bin/sh", "-s"], input="set -eu\n" + self.source + mocks + body, + text=True, capture_output=True, env=environment, timeout=5) + + def require_success(self, result): + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + + def test_syntax_and_no_source_side_effects(self): + subprocess.run(["/bin/sh", "-n", str(SOURCE)], check=True, timeout=5) + self.require_success(self.call(":")) + self.assertFalse(self.home.exists()) + + def test_created_home_is_removed_and_cleanup_idempotent(self): + self.require_success(self.call(''' +weather_client_home_prepare +[ "$WEATHER_CLIENT_HOME_CREATED" = yes ] +[ -n "$WEATHER_CLIENT_HOME_IDENTITY" ] +weather_client_home_cleanup +weather_client_home_cleanup +[ "$WEATHER_CLIENT_HOME_CREATED" = no ] +''')) + self.assertFalse(self.home.exists()) + + def test_existing_home_is_preserved_despite_inherited_ownership(self): + self.home.mkdir(mode=0o755) + identity = f"{self.home.stat().st_dev}:{self.home.stat().st_ino}" + self.require_success(self.call(''' +[ "$WEATHER_CLIENT_HOME_CREATED" = no ] +[ -z "$WEATHER_CLIENT_HOME_IDENTITY" ] +weather_client_home_prepare +weather_client_home_cleanup +''', WEATHER_CLIENT_HOME_CREATED="yes", WEATHER_CLIENT_HOME_IDENTITY=identity)) + self.assertTrue(self.home.is_dir()) + + def test_non_kvm_refuses_creation(self): + result = self.call("weather_client_home_prepare", TEST_VIRT="none") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.home.exists()) + + def test_unsafe_existing_homes_rejected_without_mutation(self): + for kind in ("file", "symlink", "nonempty", "mode"): + with self.subTest(kind=kind): + if kind == "file": self.home.touch() + elif kind == "symlink": self.home.symlink_to(self.root) + else: + self.home.mkdir(mode=0o755) + if kind == "nonempty": (self.home / "retain").touch() + result = self.call("weather_client_home_prepare", TEST_HOME_MODE="777" if kind == "mode" else "755") + self.assertNotEqual(result.returncode, 0) + self.assertTrue(self.home.exists()) + if kind in ("file", "symlink"): self.home.unlink() + else: + if kind == "nonempty": (self.home / "retain").unlink() + self.home.rmdir() + + def test_replaced_inode_is_not_removed(self): + self.require_success(self.call(''' +weather_client_home_prepare +mv -- "$TEST_HOME_PATH" "$WORK/original" +mkdir -m 0755 -- "$TEST_HOME_PATH" +if weather_client_home_cleanup; then exit 90; fi +[ "$WEATHER_CLIENT_HOME_CREATED" = yes ] +''')) + self.assertTrue(self.home.is_dir()) + self.assertTrue((self.root / "original").is_dir()) + + def test_nonempty_created_home_refuses_until_empty(self): + self.require_success(self.call(''' +weather_client_home_prepare +touch -- "$TEST_HOME_PATH/retained" +if weather_client_home_cleanup; then exit 90; fi +[ -f "$TEST_HOME_PATH/retained" ] +rm -- "$TEST_HOME_PATH/retained" +weather_client_home_cleanup +''')) + self.assertFalse(self.home.exists()) + + def test_cleanup_attempts_every_owned_resource_after_failure(self): + (self.root / "bin").mkdir() + (self.root / "bin/weather-client-delivery.py").touch() + self.require_success(self.call(''' +weather_client_home_prepare +weather_client_private() { + printf '%s\n' "$1" >> "$WORK/calls" + [ "$1" != publisher-cleanup ] +} +if weather_client_delivery_cleanup; then exit 90; fi +[ "$WEATHER_CLIENT_HOME_CREATED" = no ] +''')) + self.assertFalse(self.home.exists()) + self.assertEqual((self.root / "calls").read_text().splitlines(), ["publisher-cleanup", "user-cleanup"]) + + def test_runtime_arguments_boundaries_and_receipt_order(self): + source = SOURCE.read_text() + self.assertNotIn("env HOME=", source) + self.assertNotIn('--groups=', source) + self.assertIn('--lifetime-seconds 900', source) + self.assertIn('.publications == 4', source) + self.assertIn('-- env -i PATH=/usr/bin:/bin LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONDONTWRITEBYTECODE=1', source) + for limit in ("400s", "200s", "270s"): + self.assertIn(f'--kill-after=10s {limit}', source) + run = source.split("weather_client_delivery_run() {", 1)[1] + self.assertLess(run.index('"$WORK/weather-client-delivery-manifest.json"'), run.index('chown -R')) + self.assertLess(run.index('publisher-cleanup "$weather_publisher"'), run.index('PHASE=weather-client-delivery-cold')) + self.assertLess(run.index(' retain "$WORK"'), run.index('weather_client_delivery_cleanup')) + self.assertLess(run.index('weather_client_delivery_cleanup'), run.index(' evidence "$WORK"')) + self.assertIn('weather-client-delivery-tree-before.json', run) + self.assertIn('weather-client-delivery-tree-after.json', run) + + def test_interrupt_retains_before_cleanup_and_keeps_attempting_on_error(self): + (self.root / "bin").mkdir() + (self.root / "bin/weather-client-delivery.py").touch() + output = self.root / "client-fixtures/weather-viewer/browser-output" + output.mkdir(parents=True) + (output / "report.json").touch() + for result in (0, 1): + with self.subTest(retain_result=result): + self.require_success(self.call(''' +weather_client_home_prepare +python3() { printf 'retain\n' >> "$WORK/calls"; return "$TEST_RETAIN_STATUS"; } +weather_client_private() { printf '%s\n' "$1" >> "$WORK/calls"; } +status=0 +weather_client_delivery_cleanup || status=$? +[ "$status" -eq "$TEST_RETAIN_STATUS" ] +[ "$WEATHER_CLIENT_HOME_CREATED" = no ] +''', TEST_RETAIN_STATUS=str(result))) + self.assertFalse(self.home.exists()) + self.assertEqual((self.root / "calls").read_text().splitlines(), + ["retain", "publisher-cleanup", "user-cleanup"]) + (self.root / "calls").unlink() + + def test_already_retained_report_is_not_replaced(self): + (self.root / "bin").mkdir() + (self.root / "bin/weather-client-delivery.py").touch() + output = self.root / "client-fixtures/weather-viewer/browser-output" + output.mkdir(parents=True) + (output / "report.json").touch() + retained = self.root / "weather-client-delivery-original-report.json" + retained.write_bytes(b'original') + self.require_success(self.call(''' +python3() { exit 90; } +weather_client_private() { printf '%s\n' "$1" >> "$WORK/calls"; } +weather_client_delivery_cleanup +''')) + self.assertEqual(retained.read_bytes(), b'original') + self.assertEqual((self.root / "calls").read_text().splitlines(), ["publisher-cleanup", "user-cleanup"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-weather-client-delivery.py b/tests/integration/test-weather-client-delivery.py new file mode 100644 index 000000000..440c808d7 --- /dev/null +++ b/tests/integration/test-weather-client-delivery.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert receipt/cleanup tests. No origin requests, decoder, core or browser.""" +import hashlib +import json +import os +from pathlib import Path +import runpy +import signal +import sys +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +A = runpy.run_path(str(HERE / "weather-client-delivery.py")) +SHA = "a" * 64 +FILTER = runpy.run_path(str(HERE / "test-content-provider-filter-smoke.py")) + + +def publication(): + return dict(publisher_key=SHA, name=A["NAME"], revision=1, manifest_id="b" * 64, + sha256="c" * 64, bytes=6000, expires_unix_seconds=1900000000) + + +def selected(): + return dict(schema=1, publication=publication(), query=dict(latitude=52.52, longitude=13.41, + run="2026-10-10T12:00:00.000Z", model="ifs", steps=[3, 9, 24], communityArea=True, includeWind=True)) + + +def receipt(cache=Path("/fixture/state-client/weather-cache"), output=Path("/fixture/forecast.json")): + p = publication() + return dict(operation="named_content_download", **{k: v for k, v in p.items() if k != "expires_unix_seconds"}, + publication_expires_unix_seconds=p["expires_unix_seconds"], cache=str(cache), local_output=str(output), + local_delivery=True, output_mode="0600", peer_bytes=p["bytes"], providers_used=1, provider_peer_ids=["peer-a"], + origin_body_bytes=0, origin_range_requests=0, chunks=1, ownership_changed=False, origin_authenticated=False, + globally_latest=False, cache_only=False, control_relay_peer_id="control") + + +def acquisition(): + return dict(schema=1, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, forecast_sha256=publication()["sha256"], + forecast_bytes=publication()["bytes"], decoder_sha256=A["B"]["DECODER_SHA"], + fields=[dict(lead_hours=step, parameter=parameter, index_sha256=SHA, field_sha256=SHA) + for step in (3, 9, 24) for parameter in ("2t", "10u", "10v")]) + + +def report(): + return dict(schema=1, passed=True, phase="complete", source=dict(bundle_manifest_sha256=SHA, + driver_sha256=SHA, preparer_sha256=SHA, built_tree_sha256=SHA, node_sha256=A["B"]["NODE_SHA"], + bridge_modules={name: SHA for name in ("scripts/serve-provider.mjs", "src/provider-http.mjs", + "src/provider-core.mjs", "src/core-cache.mjs")}), + expected=dict(expected_sha256=SHA, cold_receipt_sha256=SHA, core_sha256=SHA, display_sha256=SHA, + forecast_sha256=publication()["sha256"], manifest_id=publication()["manifest_id"]), + scope=dict(actual_drizzli_ui=True, actual_http_bridge=True, real_core_adapter=True, + synthetic_weather=False, standalone_origin_proven=False, standalone_overlay_proven=False, + browser_triggered_cold_fetch=False), + checks=dict(api_response_sha256=SHA, initial_dom_display_sha256=SHA, refresh_dom_display_sha256=SHA, + initial_dom_matches_cold_publication=True, refresh_dom_matches_cold_publication=True, + browser_api_requests=2, provider_requests_checked=1, exact_native_times_and_values=True, + explicit_missing_capabilities=True, no_external_resources=True, source_attribution=True), + cleanup=dict(browser_exited=True, bridge_exited=True, private_state_removed=True, + normal_browser_stop=True, normal_bridge_stop=True), + outer_cleanup=dict(namespace_wrapper_exited=True, private_state_removed=True, client_namespace_unchanged=True)) + + +def save(path, value, mode=0o600): + A["write_new"](path, json.dumps(value).encode(), mode) + + +def joined_fixture(work): + """Synthetic closed records exercise the join, not real network transport.""" + f = FILTER["fixture"]() + p = publication() + r = receipt(work / "state-client/weather-cache", work / "client-fixtures/weather-viewer/forecast.json") + r.update(provider_peer_ids=f["cold"]["application"]["receipt"]["provider_peer_ids"], + control_relay_peer_id=f["layout"]["control_relay_peer_id"]) + files = { + "input": selected(), "acquisition": acquisition(), "publisher-cleanup": dict(directory_removed=True, files_removed=True), + "input-cleanup": dict(directory_removed=True, files_removed=True), "selection": f["selected_route"], + "control": f["cold"]["control_privacy"], "prepared": dict(prepared=True, expected_sha256=SHA, display_sha256=SHA), + "cold-consumer": dict(schema=1, receipt=r, publication=p, cold_cache_requested=True, core_sha256=SHA, + cold_receipt_sha256=SHA, observed_unix_seconds=p["expires_unix_seconds"]-60, + boundary=dict(user_uid=985, primary_control_gid=os.getgid(), supplementary_groups_empty=True, + all_capabilities_dropped=True, no_new_privileges=True, client_namespace=True, acquisition_parent_namespace=False)), + "publish": dict(operation="offline_content_publish", network_publication=False, + publisher_key_hex=p["publisher_key"], content_type="application/json", chunks=1, + **{k: p[k] for k in ("name", "revision", "bytes", "manifest_id", "expires_unix_seconds")})} + cache = dict(device=1, inode=10, mode="0700", user_uid=1234, group_gid=os.getgid()+1) + files.update({"cache-" + name: cache for name in ("before", "after")}) + for role, captured in f["cold"]["privacy"].items(): files["privacy-" + role] = captured + for node in f["layout"]["provider_nodes"]: + files[node + "-import"] = dict(f["imports"][node], manifest_id=p["manifest_id"], content_bytes=p["bytes"], + agent_cache=str(work / f"state-{node}/weather-cache")) + files[node + "-serve"] = dict(f["serves"][node], publications=4) + B = A["B"] + items = {name: dict(bytes=1, mode=B["mode"](name), sha256=SHA) for name in B["FIXED"] | {B["STATIC"]+"index.html"}} + for name, digest in (("runtime/node", B["NODE_SHA"]), ("decoder/grib_ls", B["DECODER_SHA"]), + (B["WEATHER"]+"third_party/eccodes-sources.json", B["PIN_SHA"]), + (B["WEATHER"]+"scripts/build_eccodes.py", B["BUILD_SHA"])): items[name]["sha256"] = digest + manifest = B["encode"](dict(version=1, kind=B["KIND"], files=items)) + manifest_sha = hashlib.sha256(manifest).hexdigest() + A["write_new"](work / "weather-client-delivery-manifest.json", manifest, 0o400) + for name in ("before", "after"): + files["tree-"+name] = dict(version=1, action="verify-tree", manifest_sha256=manifest_sha, files=len(items), runtime_executed=False) + browser = report(); browser["source"]["bundle_manifest_sha256"] = manifest_sha + browser["source"]["built_tree_sha256"] = hashlib.sha256(json.dumps({"index.html": SHA}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + files["original-report"] = browser + save(work / "a01-expected-peers.json", f["expected_peers"]) + save(work / "content-provider-layout.json", f["layout"]) + socket = work / "runtime-client/control/agent.sock"; socket.parent.mkdir(parents=True) + A["write_new"](socket, b"inert stat-only fixture") + for suffix, value in files.items(): save(work / f"weather-client-delivery-{suffix}.json", value) + return manifest_sha, cache + + +class Receipts(unittest.TestCase): + def test_join_requires_same_publication_dom_hash_and_closed_real_evidence(self): + mutations = [None, + ("original-report", ("checks", "refresh_dom_display_sha256"), "f"*64), + ("original-report", ("scope", "synthetic_weather"), True), + ("original-report", ("expected", "cold_receipt_sha256"), "f"*64), + ("original-report", ("expected", "expected_sha256"), "f"*64), + ("original-report", ("source", "node_sha256"), "f"*64), + ("original-report", ("cleanup", "normal_bridge_stop"), False), + ("acquisition", ("synthetic_weather",), True), + ("cold-consumer", ("boundary", "all_capabilities_dropped"), False), + ("cold-consumer", ("receipt", "origin_body_bytes"), 1), + ("publisher-cleanup", ("files_removed",), False), + ("tree-after", ("manifest_sha256",), "f"*64), + ("privacy-client", ("direct_client_exit_packets",), 1), + ("privacy-exit", ("packet_socket_drops",), 1), + ("control", ("unexpected_provider_application_packets",), 1)] + for mutation in mutations: + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, cache = joined_fixture(work) + if mutation: + suffix, keys, value = mutation + path = work / f"weather-client-delivery-{suffix}.json" + data = json.loads(path.read_bytes()); target = data + for key in keys[:-1]: target = target[key] + target[keys[-1]] = value + path.unlink(); save(path, data) + with patch.dict(A["evidence"].__globals__, cache_state=lambda *args: cache): + if mutation: + with self.assertRaises(ValueError): A["evidence"](work, digest, SHA) + else: + proof = A["evidence"](work, digest, SHA) + self.assertTrue(proof["success"]) + self.assertEqual(proof["protected_cold"]["peer_bytes"], publication()["bytes"]) + self.assertFalse(proof["scope"]["browser_triggered_cold_fetch"]) + self.assertFalse(proof["scope"]["browser_cache_only_proven"]) + self.assertFalse(proof["scope"]["two_subflow_bulk_aggregation"]) + A["validate_export"](proof) + + def test_selection_has_no_authority_or_query_fallback(self): + A["validate_selection"](selected()) + for key, value in (("latitude", 52), ("communityArea", False), ("includeWind", False), + ("steps", [3, 6, 9]), ("model", "aifs"), ("run", "2026-10-10T01:00:00.000Z")): + bad = selected(); bad["query"][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): A["validate_selection"](bad) + for key, value in (("revision", True), ("bytes", 0), ("bytes", 131073), ("publisher_key", "wrong")): + bad = selected(); bad["publication"][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): A["validate_selection"](bad) + + def test_cold_receipt_requires_real_peer_bytes_zero_origin_and_exact_object(self): + cache, output = Path("/fixture/state-client/weather-cache"), Path("/fixture/forecast.json") + A["validate_cold_receipt"](receipt(), publication(), cache, output) + changes = dict(peer_bytes=0, origin_body_bytes=1, origin_range_requests=1, cache_only=True, + providers_used=True, provider_peer_ids=["peer-a", "peer-a"], chunks=True, local_delivery=False, + publication_expires_unix_seconds=1, manifest_id=SHA, sha256=SHA, cache="/other", local_output="/other") + for key, value in changes.items(): + bad = receipt(); bad[key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + A["validate_cold_receipt"](bad, publication(), cache, output) + + def test_acquisition_requires_all_nine_actual_fields_and_decoder_pin(self): + A["validate_acquisition"](acquisition()) + changes = dict(synthetic_weather=True, upstream_signature_attested=True, acquisition_outside_overlay=False, + decoder_sha256=SHA, forecast_bytes=True, forecast_sha256="private", fields=[]) + for key, value in changes.items(): + bad = acquisition(); bad[key] = value + with self.subTest(key=key), self.assertRaises(ValueError): A["validate_acquisition"](bad) + bad = acquisition(); bad["fields"][-1] = bad["fields"][0] + with self.assertRaises(ValueError): A["validate_acquisition"](bad) + + def test_closed_browser_report_rejects_private_or_unbounded_fields(self): + A["closed_report"](report()) + variants = [] + bad = report(); bad["log"] = "PRIVATE"; variants.append(bad) + bad = report(); bad["checks"]["url"] = "PRIVATE"; variants.append(bad) + bad = report(); bad["checks"]["browser_api_requests"] = 500; variants.append(bad) + bad = report(); bad["checks"]["refresh_dom_display_sha256"] = "PRIVATE"; variants.append(bad) + bad = report(); bad["source"]["bridge_modules"]["raw.js"] = SHA; variants.append(bad) + bad = report(); bad["scope"]["actual_drizzli_ui"] = 1; variants.append(bad) + for bad in variants: + with self.assertRaises(ValueError): A["closed_report"](bad) + + def test_export_is_byte_exact_and_prevalidates_all_candidates(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); work = root / "work"; output = root / "output" + work.mkdir(); output.mkdir() + raw = json.dumps(acquisition(), indent=2).encode() + b"\n" + source = work / "weather-client-delivery-acquisition.json" + A["write_new"](source, raw) + late = work / "weather-client-delivery-input-cleanup.json" + save(late, dict(directory_removed=True, private="DO NOT EXPORT")) + with self.assertRaises(ValueError): A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + late.unlink() + result = A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(result, dict(files=1, raw_logs_exported=False)) + self.assertEqual((output / source.name).read_bytes(), raw) + before = (output / source.name).stat().st_ino + self.assertEqual(A["copy_artifacts"](work, output, os.getuid(), os.getgid()), result) + self.assertEqual((output / source.name).stat().st_ino, before) + source.unlink(); save(source, dict(schema=1, success=False)) + with self.assertRaises(ValueError): A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual((output / source.name).read_bytes(), raw) + + def test_duplicate_keys_rejected_before_raw_retention_or_export(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); output = root / "output"; output.mkdir() + raw = b'{"schema":"PRIVATE",' + json.dumps(acquisition()).encode()[1:] + A["write_new"](root / "weather-client-delivery-acquisition.json", raw) + with self.assertRaises(ValueError): A["copy_artifacts"](root, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + path = root / "client-fixtures/weather-viewer/browser-output"; path.mkdir(parents=True) + save(path / "report.json", report()) + duplicate = b'{"schema":"PRIVATE",' + json.dumps(report()).encode()[1:] + with patch.dict(A["retain"].__globals__, worker_report=lambda p: duplicate): + with self.assertRaises(ValueError): A["retain"](root) + self.assertFalse((root / "weather-client-delivery-original-report.json").exists()) + + def test_cleanup_refuses_unknown_input_before_removing_known_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "weather-publisher"; root.mkdir(mode=0o700) + A["write_new"](root / "passphrase", b"fixture") + A["write_new"](root / "foreign", b"preserve") + with self.assertRaises(ValueError): A["cleanup"](root, True) + self.assertEqual((root / "passphrase").read_bytes(), b"fixture") + (root / "foreign").unlink() + save(root / "publish-receipt.json", {"schema": 1}, 0o400) + self.assertEqual(A["cleanup"](root, True), dict(directory_removed=True, files_removed=True)) + self.assertFalse(root.exists()) + + +class ProcessBounds(unittest.TestCase): + def test_output_limit_timeout_environment_and_handler_restore(self): + previous = signal.getsignal(signal.SIGTERM) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + raw = A["run"]([sys.executable, "-c", "import os;print(sorted(os.environ))"], root, 1024, 3) + self.assertNotIn(b"TOKEN", raw) + for code, maximum, timeout in (("print('x'*200)", 32, 3), ("import time;time.sleep(60)", 32, .05)): + with self.assertRaises(ValueError): A["run"]([sys.executable, "-c", code], root, maximum, timeout) + self.assertEqual(signal.getsignal(signal.SIGTERM), previous) + self.assertEqual(Path(f"/proc/self/task/{os.getpid()}/children").read_text().strip(), "") + + def test_successful_parent_exit_reaps_detached_grandchild(self): + code = ("import subprocess,sys; p=subprocess.Popen([sys.executable,'-c','import time;time.sleep(60)']," + "start_new_session=True,stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL);print(p.pid)") + with tempfile.TemporaryDirectory() as temporary: + raw = A["run"]([sys.executable, "-c", code], Path(temporary), 64, 3) + pid = int(raw) + with self.assertRaises(ProcessLookupError): os.kill(pid, 0) + self.assertEqual(Path(f"/proc/self/task/{os.getpid()}/children").read_text().strip(), "") + + +if __name__ == "__main__": unittest.main() diff --git a/tests/integration/test-weather-client-runner.py b/tests/integration/test-weather-client-runner.py new file mode 100644 index 000000000..ca43da030 --- /dev/null +++ b/tests/integration/test-weather-client-runner.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Exercise runner argument/path/resource contracts without a VM or network.""" +import os +import json +from pathlib import Path +import stat +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +RUNNERS = ("run-alpha-topology-vm.sh", "kvm-alpha-topology.sh") +BUNDLE = "--weather-client-bundle" +DIGEST = "--weather-client-manifest-sha256" +SHA = "b" * 64 + + +class WeatherRunner(unittest.TestCase): + def call(self, runner, *args, env=None): + return subprocess.run(["/bin/sh", str(HERE / runner), *args], + capture_output=True, timeout=5, env=env) + + def required(self, runner, scenario="content-provider"): + args = ["--execute", "--yes", "--scenario", scenario, + "--output", "/unread/output", "--expected-commit", "a" * 40, + "--mpquic", "/unread/native"] + if runner.startswith("run-"): + args += ["--image", "/unread/image"] + else: + args += ["--source", "/unread/source", "--bin", "/unread/bin"] + return args + + def test_shells_and_unselected_preview_remain_inert(self): + for runner in RUNNERS: + subprocess.run(["/bin/sh", "-n", str(HERE / runner)], check=True, timeout=5) + env = dict(os.environ, weather_client_bundle="/unread/bundle", + weather_client_manifest_sha256=SHA) + result = self.call(runner, "--preview", "--scenario", "content-provider", env=env) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(b"PREVIEW ONLY", result.stdout) + self.assertNotIn(b"Additional proof:", result.stdout) + + def test_explicit_pair_and_no_preview_or_duplicate_authority(self): + for runner in RUNNERS: + for flag in (BUNDLE, DIGEST): + for value in (None, "", "/unread"): + args = [flag] + ([] if value is None else [value]) + self.assertEqual(self.call(runner, "--preview", *args).returncode, 64) + self.assertEqual(self.call(runner, *self.required(runner), + flag, "one", flag, "two").returncode, 64) + cases = ([BUNDLE, "/unread/bundle"], [DIGEST, SHA], + [BUNDLE, "relative", DIGEST, SHA], + [BUNDLE, "/unread/../bundle", DIGEST, SHA], + [BUNDLE, "/unread/bundle", DIGEST, "g" * 64], + [BUNDLE, "/unread/bundle", DIGEST, "b" * 63], + [BUNDLE, "/unread/bundle", DIGEST, SHA, + "--browser-filter-bundle", "/unread/browser", + "--browser-filter-manifest-sha256", SHA]) + for args in cases: + with self.subTest(runner=runner, args=args): + self.assertEqual(self.call(runner, *self.required(runner), *args).returncode, 64) + self.assertEqual(self.call(runner, *self.required(runner, "content"), + BUNDLE, "/unread/bundle", DIGEST, SHA).returncode, 64) + + def test_weather_scratch_is_private_disk_sibling_not_exported_evidence(self): + source = (HERE / RUNNERS[0]).read_text() + fragment = source.split("RUN_DIRECTORY_PARENT=/tmp\n", 1)[1].split("QEMU_PID=\n", 1)[0] + prefix = ("set -eu\nscenario=content-provider\nbrowser_filter_bundle=\n" + "weather_client_bundle=/selected/weather\noutput_directory=$1\nRUN_DIRECTORY_PARENT=/tmp\n") + with tempfile.TemporaryDirectory(prefix="weather-runner-test-", dir=HERE) as temporary: + parent = Path(temporary).resolve() + output = parent / "evidence" + output.mkdir() + result = subprocess.run(["/bin/sh", "-c", prefix + fragment + 'printf "%s\\n" "$RUN_DIRECTORY"\n', + "fixture", str(output)], capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + scratch = Path(result.stdout.decode().strip()) + self.assertEqual(scratch.parent, parent) + self.assertTrue(scratch.name.startswith("weather-client-kvm.")) + self.assertEqual(stat.S_IMODE(scratch.stat().st_mode), 0o700) + self.assertEqual(list(output.iterdir()), []) + for filesystem in ("tmpfs", "ramfs"): + stub = 'stat() { printf "%s\\n" "' + filesystem + '"; }\n' + result = subprocess.run(["/bin/sh", "-c", prefix + stub + fragment, + "fixture", str(output)], capture_output=True, timeout=5) + self.assertEqual(result.returncode, 69, result.stderr) + + def test_weather_uses_same_four_gib_guest_and_two_gib_host_reserve(self): + source = (HERE / RUNNERS[0]).read_text() + memory = "guest_memory_for_scenario() {\n" + source.split("guest_memory_for_scenario() {\n", 1)[1].split("\n}\n", 1)[0] + "\n}\n" + gate = "guest_memory_mib=$(guest_memory_for_scenario)\n" + source.split("guest_memory_mib=$(guest_memory_for_scenario)\n", 1)[1].split("qemu_usernet=", 1)[0] + script = ("set -eu\nscenario=content-provider\nbrowser_filter_bundle=\nweather_client_bundle=/selected\n" + 'fixture_available=$1\nawk() { printf "%s\\n" "$fixture_available"; }\n' + memory + gate + + 'printf "%s:%s\\n" "$guest_memory_mib" "$runner_reserve_mib"\n') + for available, expected in ((6144 * 1024 - 1, 69), (6144 * 1024, 0), ("unknown", 69)): + result = subprocess.run(["/bin/sh", "-c", script, "fixture", str(available)], capture_output=True, timeout=5) + self.assertEqual(result.returncode, expected, result.stderr) + if expected == 0: + self.assertEqual(result.stdout.decode().strip(), "4096:2048") + + def test_guest_discriminator_binds_only_weather_arguments(self): + source = (HERE / RUNNERS[0]).read_text() + guest = source.split("<<'GUEST_DRIVER_SCRIPT'\n", 1)[1].split("\nGUEST_DRIVER_SCRIPT\n", 1)[0] + fragment = guest[guest.index("expected_commit=$1"):guest.index("cd /home/vpci")] + script = "set -eu\n" + fragment + 'printf "%s:%s:%s\\n" "$scenario" "$browser_filter_manifest_sha256" "$weather_client_manifest_sha256"\n' + base = ["a" * 40, SHA, SHA, "none", "content-provider"] + for suffix, expected in ((["weather-client", SHA, SHA], 0), (["unknown", SHA, SHA], 64), + (["weather-client", "none", SHA], 64), (["weather-client", SHA], 64)): + result = subprocess.run(["/bin/sh", "-c", script, "fixture", *base, *suffix], capture_output=True, timeout=5) + self.assertEqual(result.returncode, expected, result.stderr) + if expected == 0: + self.assertEqual(result.stdout.decode().strip(), "content-provider:none:" + SHA) + + def test_control_observer_accepts_only_the_one_new_weather_prefix(self): + source = (HERE / "content-provider-smoke.sh").read_text() + body = source.split("content_provider_start_control_observer() {\n", 1)[1].split("\n}\n", 1)[0] + function = "content_provider_start_control_observer() {\n" + body + "\n}\n" + with tempfile.TemporaryDirectory() as temporary: + script = ("set -eu\nWORK=$1\nPROVIDER_CONTROL_PID=\nprovider_control_ns=fixture\n" + "provider_control_ips=fixture\nip() { :; }\nwait_observer() { :; }\n" + function + + 'content_provider_start_control_observer "$2"\n') + for name, expected in (("weather-client-delivery-control", 0), + ("content-provider-control-privacy", 0), + ("weather-client-other", 1), + ("weather-client-delivery-control-extra", 1), + ("../weather-client-delivery-control", 1)): + result = subprocess.run(["/bin/sh", "-c", script, "fixture", temporary, name], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, expected, result.stderr) + + +class WeatherEmergencyExport(unittest.TestCase): + @classmethod + def setUpClass(cls): + source = (HERE / RUNNERS[0]).read_text() + body = source.split("<<'GUEST_DIAGNOSTICS_PYTHON'\n", 1)[1].split("\nGUEST_DIAGNOSTICS_PYTHON\n", 1)[0] + cls.api = {"__name__": "inert_weather_collector"} + exec(compile(body, "weather-collector", "exec"), cls.api) + + def acquisition(self): + return dict(schema=1, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, + forecast_sha256=SHA, forecast_bytes=6000, decoder_sha256=SHA, + fields=[dict(lead_hours=step, parameter=parameter, + index_sha256=SHA, field_sha256=SHA) + for step in (3, 9, 24) for parameter in ("2t", "10u", "10v")]) + + def collect(self, raw, limit=None): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary).resolve() + home, opt, proc, cgroups = (base / name for name in ("home", "opt", "proc", "cgroups")) + for directory in (home, opt, proc, cgroups): + directory.mkdir() + (home / "alpha-output").mkdir() + work = opt / ("va." + "a" * 32 + ".ABC123") + work.mkdir() + source = work / "weather-client-delivery-acquisition.json" + source.write_bytes(raw) + source.chmod(0o600) + for name in ("weather-client-delivery-cold-consumer.json", "weather-client-delivery-input.json", + "weather-client-delivery.err", "forecast.json", "config.json"): + (work / name).write_text("PRIVATE CANARY") + overrides = {"weather_client_helper": lambda root: HERE / "weather-client-delivery.py"} + if limit is not None: + overrides["FILE_LIMIT"] = limit + # Bypass only root-owned staging in this unprivileged inert test. + # Actual production JSON validation and collector still execute. + with patch.dict(self.api, overrides): + archive = self.api["collect"](home, opt, "a" * 40, "content-provider", 124, cgroups, proc) + with tarfile.open(archive, "r:gz") as bundle: + return {item.name: bundle.extractfile(item).read() for item in bundle.getmembers()} + + def test_valid_receipt_survives_exactly_but_never_claims_vm_success(self): + raw = json.dumps(self.acquisition(), indent=2).encode() + b"\n" + files = self.collect(raw) + self.assertEqual(files["work-1/weather-client-delivery-acquisition.json"], raw) + summary = json.loads(files["vm-incomplete.json"]) + self.assertFalse(summary["success"]) + self.assertEqual(summary["cleanup"], {"complete": False, "verified": False}) + self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + + def test_private_unknown_duplicate_and_malformed_json_do_not_escape(self): + valid = self.acquisition() + bad = [b"not json", json.dumps(dict(valid, prompt="PRIVATE CANARY")).encode(), + b'{"schema":"PRIVATE CANARY",' + json.dumps(valid).encode()[1:], + json.dumps(dict(valid, forecast_sha256="PRIVATE CANARY")).encode()] + for raw in bad: + with self.subTest(raw=raw[:30]): + files = self.collect(raw) + self.assertNotIn("work-1/weather-client-delivery-acquisition.json", files) + self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + + def test_verified_json_is_not_exported_as_an_unverifiable_tail(self): + files = self.collect(json.dumps(self.acquisition()).encode(), limit=64) + self.assertFalse(any("weather-client-delivery" in name for name in files)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/weather-client-delivery.py b/tests/integration/weather-client-delivery.py new file mode 100644 index 000000000..5bd16d7b9 --- /dev/null +++ b/tests/integration/weather-client-delivery.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Joined real Weather acquisition, protected cold receipt and native UI evidence. + +Only the explicit disposable topology hook executes these operations. Publication +uses the normal CLI; the browser cannot publish, acquire origin data or see agent +caches. Export only closed summaries, never forecasts, locations, paths or logs. +""" +import datetime +import ctypes +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import selectors +import signal +import stat +import subprocess +import sys +import time + +HERE = Path(__file__).parent +B = runpy.run_path(str(HERE / "weather-client-bundle.py")) +HTTPS = runpy.run_path(str(HERE / "content-provider-https-smoke.py")) +NAME = "disposable-public-community-weather" +MAX_FORECAST = 131072 +MAX_REPORT = 65536 +VARIABLES = ["temperature_2m", "wind_speed_10m", "wind_direction_10m", "precipitation", "air_quality"] +EXPORTS = ("evidence", "acquisition", "original-report", "tree-before", "tree-after", "cache-before", "cache-after", "input-cleanup") +require, hex64 = B["require"], B["hex64"] + + +def same(value, expected): + require(type(value) is type(expected)) + if type(expected) is dict: + require(set(value) == set(expected)) + for key in expected: same(value[key], expected[key]) + elif type(expected) is list: + require(len(value) == len(expected)) + for actual, wanted in zip(value, expected): same(actual, wanted) + else: require(value == expected) + + +def raw_file(path, maximum=MAX_REPORT, mode=None): + stream, info = B["open_checked"](path, maximum, normalized=mode) + with stream: + raw = stream.read(maximum + 1) + require(len(raw) == info.st_size and B["identity"](os.fstat(stream.fileno())) == B["identity"](info)) + return raw + + +def read(path, maximum=MAX_REPORT, mode=None): + return json.loads(raw_file(path, maximum, mode), object_pairs_hook=B["B"]["unique_pairs"]) + + +def write_new(path, raw, mode=0o600): + B["new_path"](path) + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode), "wb") as output: + output.write(raw) + + +def private(root, directory=True): + B["B"]["canonical"](root) + info = root.lstat() + require((stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)) + and info.st_uid == os.getuid() and stat.S_IMODE(info.st_mode) == (0o700 if directory else 0o600)) + if not directory: require(info.st_nlink == 1) + return info + + +def boundary(parent, client, group, acquisition=False): + status = dict(line.split(":", 1) for line in Path("/proc/self/status").read_text().splitlines()) + require(os.getuid() == 985 and os.getgid() == group and not os.getgroups() + and all(int(v) == 985 for v in status["Uid"].split()) + and all(int(v) == group for v in status["Gid"].split()) + and all(int(status[k], 16) == 0 for k in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")) + and int(status["NoNewPrivs"]) == 1 and re.fullmatch(r"net:\[\d+\]", parent) + and re.fullmatch(r"net:\[\d+\]", client) and parent != client + and os.readlink("/proc/self/ns/net") == (parent if acquisition else client)) + return dict(user_uid=985, primary_control_gid=group, supplementary_groups_empty=True, + all_capabilities_dropped=True, no_new_privileges=True, client_namespace=not acquisition, + acquisition_parent_namespace=acquisition) + + +def run(command, cwd, maximum, timeout): + """Bound output/time and reap even detached decoder descendants after exit. + + This single-threaded fixture owns all children. A Linux subreaper prevents a + decoder orphan escaping when its Node parent exits before normal cleanup. + No process-name matching or unrelated process-group signalling is used. + """ + children_path = Path(f"/proc/self/task/{os.getpid()}/children") + require(not children_path.read_text().strip()) + libc = ctypes.CDLL(None, use_errno=True) + previous_reaper = ctypes.c_int() + require(libc.prctl(37, ctypes.byref(previous_reaper), 0, 0, 0) == 0) + require(libc.prctl(36, 1, 0, 0, 0) == 0) + environment = dict(PATH="/usr/bin:/bin", LANG="C.UTF-8", LC_ALL="C.UTF-8", PYTHONDONTWRITEBYTECODE="1") + process = None + data = bytearray(); deadline = time.monotonic() + timeout + previous = {} + def interrupted(_signum, _frame): raise ValueError("weather_client_interrupted") + try: + process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, env=environment, start_new_session=True) + for signum in (signal.SIGINT, signal.SIGTERM): previous[signum] = signal.signal(signum, interrupted) + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while True: + require(time.monotonic() < deadline and selector.select(max(0, deadline-time.monotonic()))) + part = os.read(process.stdout.fileno(), min(65536, maximum + 1 - len(data))) + if not part: break + data.extend(part); require(len(data) <= maximum) + require(process.wait(timeout=max(0.1, deadline-time.monotonic())) == 0 and data) + return bytes(data) + finally: + if process is not None and process.poll() is None: + try: os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: pass + try: process.wait(timeout=10) + except subprocess.TimeoutExpired: + try: os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: pass + process.wait(timeout=5) + if process is not None: process.stdout.close() + cleanup_deadline = time.monotonic() + 5 + while True: + children = children_path.read_text().split() + if not children: break + require(len(children) <= 64 and time.monotonic() < cleanup_deadline) + for child in children: + pid = int(child) + # A live direct child cannot have its PID reused until we reap + # it. There is no other thread or wait() consumer in this helper. + if os.waitpid(pid, os.WNOHANG)[0] == 0: + try: os.kill(pid, signal.SIGKILL) + except ProcessLookupError: pass + time.sleep(0.01) + require(libc.prctl(36, previous_reaper.value, 0, 0, 0) == 0) + for signum, handler in previous.items(): signal.signal(signum, handler) + + +def initialize(root): + private(root); require(root.name == "weather-publisher" and not list(root.iterdir())) + write_new(root / "passphrase", os.urandom(48).hex().encode() + b"\n") + (root / "scratch").mkdir(mode=0o700) + return dict(schema=1, initialized=True) + + +def acquisition(source, manifest_sha, root, parent, client, group): + private(root); require(root.name == "weather-publisher" and set(p.name for p in root.iterdir()) == {"passphrase", "scratch"}) + boundary(parent, client, group, True) + B["verify_tree"](source, manifest_sha) + version = run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10) + require(version.strip() == b"ecCodes Version 2.44.0") + # A prior complete 12UTC cycle, never an implicit alternate service/model retry. + hour = int(time.time()) // 43200 * 43200 - 43200 + model_run = datetime.datetime.fromtimestamp(hour, datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + command = [str(source / "runtime/node"), "--max-old-space-size=256", + str(source / B["WEATHER"] / "scripts/forecast.mjs"), "origin", "--latitude", "52.52", "--longitude", "13.41", + "--run", model_run, "--model", "ifs", "--steps", "3,9,24", "--community-area", "--include-wind", + "--execute", "--accept-data-license", "--decoder", str(source / "decoder/grib_ls"), + "--decoder-sha256", B["DECODER_SHA"], "--scratch-parent", str(root / "scratch")] + raw = run(command, root, MAX_FORECAST, 360) + value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + require(value["schema"] == "volparossa.forecast.v1" and value["source"]["provider"] == "ecmwf" + and value["source"]["acquisition"] == "origin-https" and value["source"]["model"] == "ifs" + and value["source"]["model_run_at"] == model_run and value["area"]["sharing"] == "public-community" + and [sample["lead_hours"] for sample in value["samples"]] == [3, 9, 24] + and not list((root / "scratch").iterdir())) + fields = [] + for sample in value["samples"]: + for field in (sample["evidence"], *sample["wind_10m"]["evidence"].values()): + require(field["decoder_sha256"] == B["DECODER_SHA"] and field["parameter"] in ("2t", "10u", "10v") + and hex64(field["index_sha256"]) and hex64(field["field_sha256"])) + fields.append(dict(lead_hours=sample["lead_hours"], parameter=field["parameter"], + index_sha256=field["index_sha256"], field_sha256=field["field_sha256"])) + write_new(root / "forecast.json", raw) + return dict(schema=1, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, forecast_sha256=hashlib.sha256(raw).hexdigest(), + forecast_bytes=len(raw), decoder_sha256=B["DECODER_SHA"], fields=fields) + + +def select(root, publish_path, viewer): + private(root); private(viewer) + require(root.name == "weather-publisher" and viewer.name == "weather-viewer" and not list(viewer.iterdir())) + publication = read(publish_path); raw = raw_file(root / "forecast.json", MAX_FORECAST, 0o600) + forecast = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + require(publication["operation"] == "offline_content_publish" and publication["network_publication"] is False + and publication["bytes"] == len(raw) and publication["chunks"] == 1 and hex64(publication["publisher_key_hex"]) + and publication["name"] == NAME and publication["revision"] == 1 and publication["content_type"] == "application/json" + and publication["manifest_id"] == hashlib.sha256(raw_file(root / "manifest.bin")).hexdigest()) + selected = dict(schema=1, publication=dict(publisher_key=publication["publisher_key_hex"], name=NAME, revision=1, + manifest_id=hashlib.sha256(raw_file(root / "manifest.bin")).hexdigest(), sha256=hashlib.sha256(raw).hexdigest(), + bytes=len(raw), expires_unix_seconds=publication["expires_unix_seconds"]), + query=dict(latitude=forecast["area"]["latitude"], longitude=forecast["area"]["longitude"], + run=forecast["source"]["model_run_at"], model="ifs", steps=[3, 9, 24], communityArea=True, includeWind=True)) + validate_selection(selected) + write_new(viewer / "selection.json", B["encode"](selected), 0o400) + return selected + + +def validate_selection(value): + require(type(value) is dict and set(value) == {"schema", "publication", "query"}) + same(value["schema"], 1) + publication = value["publication"] + require(type(publication) is dict and set(publication) == {"publisher_key", "name", "revision", "manifest_id", "sha256", "bytes", "expires_unix_seconds"}) + same(publication["name"], NAME); same(publication["revision"], 1) + require(all(hex64(publication[key]) for key in ("publisher_key", "manifest_id", "sha256")) + and type(publication["bytes"]) is int and 0 < publication["bytes"] <= MAX_FORECAST + and type(publication["expires_unix_seconds"]) is int and publication["expires_unix_seconds"] > 0) + query = value["query"] + same({k: v for k, v in query.items() if k != "run"}, dict(latitude=52.52, longitude=13.41, + model="ifs", steps=[3, 9, 24], communityArea=True, includeWind=True)) + require(type(query["run"]) is str and re.fullmatch(r"\d{4}-\d{2}-\d{2}T(?:00|12):00:00\.000Z", query["run"])) + return value + + +def validate_cold_receipt(receipt, publication, cache, output): + require(all(type(receipt[k]) is int and receipt[k] >= 0 for k in ("revision", "bytes", "publication_expires_unix_seconds", + "peer_bytes", "providers_used", "origin_body_bytes", "origin_range_requests", "chunks")) + and type(receipt["provider_peer_ids"]) is list and all(type(peer) is str for peer in receipt["provider_peer_ids"])) + require(receipt["operation"] == "named_content_download" and all(receipt[key] == publication[key] + for key in ("publisher_key", "name", "revision", "manifest_id", "sha256", "bytes"))) + require(receipt["publication_expires_unix_seconds"] == publication["expires_unix_seconds"] + and receipt["cache"] == str(cache) and receipt["local_output"] == str(output) + and receipt["local_delivery"] is True and receipt["output_mode"] == "0600" + and receipt["peer_bytes"] == publication["bytes"] and receipt["providers_used"] in (1, 2) + and len(receipt["provider_peer_ids"]) == len(set(receipt["provider_peer_ids"])) == receipt["providers_used"] + and receipt["origin_body_bytes"] == receipt["origin_range_requests"] == 0 + and receipt["chunks"] == 1 and all(receipt[k] is False for k in + ("ownership_changed", "origin_authenticated", "globally_latest", "cache_only"))) + + +def cold(binary, control, cache, viewer, parent, client, group): + private(viewer); require(viewer.name == "weather-viewer" and set(p.name for p in viewer.iterdir()) == {"selection.json"}) + process_boundary = boundary(parent, client, group) + selected = validate_selection(read(viewer / "selection.json", mode=0o400)); publication = selected["publication"] + output = viewer / "forecast.json" + binary_before = B["fingerprint"](binary, 256*1024**2) + raw = run([str(binary), "--control-socket", str(control), "content", "fetch-name", "--publisher-key", + publication["publisher_key"], "--name", NAME, "--min-revision", "1", "--cache", str(cache), + "--local-output", str(output), "--quota-bytes", str(MAX_FORECAST), "--max-entries", "16", + "--min-free-bytes", "67108864"], viewer, MAX_REPORT, 180) + receipt = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + validate_cold_receipt(receipt, publication, cache, output) + content = raw_file(output, MAX_FORECAST, 0o600) + require(len(content) == publication["bytes"] and hashlib.sha256(content).hexdigest() == publication["sha256"] + and time.time() < publication["expires_unix_seconds"]) + write_new(viewer / "cold-receipt.json", raw) + same(B["fingerprint"](binary, 256*1024**2), binary_before) + return dict(schema=1, boundary=process_boundary, publication=publication, receipt=receipt, + cold_receipt_sha256=hashlib.sha256(raw).hexdigest(), core_sha256=binary_before["sha256"], + cold_cache_requested=True, observed_unix_seconds=int(time.time())) + + +def prepare(source, viewer, binary, control, cache): + private(viewer) + selected = validate_selection(read(viewer / "selection.json", mode=0o400)); publication = selected["publication"] + core_sha = B["fingerprint"](binary, 256*1024**2)["sha256"] + config = dict(core=dict(mode="core-cache", executable=str(binary), controlSocket=str(control), cache=str(cache), + scratchParent=str(viewer / "browser-output/scratch"), publisherKey=publication["publisher_key"], name=NAME, + minRevision=1, query=selected["query"], reuseCache=True), variables=VARIABLES) + write_new(viewer / "config.json", B["encode"](config)) + result = run([str(source / "runtime/node"), "--max-old-space-size=256", + str(source / B["WEATHER"] / "integrations/drizzli/prepare-core-browser.mjs"), + "--forecast", str(viewer / "forecast.json"), "--receipt", str(viewer / "cold-receipt.json"), + "--config", str(viewer / "config.json"), "--core-sha256", core_sha, "--output", str(viewer / "expected.json")], + viewer, 4096, 30) + result = json.loads(result, object_pairs_hook=B["B"]["unique_pairs"]) + require(result["prepared"] is True and hex64(result["expected_sha256"]) and hex64(result["display_sha256"])) + expected = read(viewer / "expected.json", 192*1024, 0o400) + same(expected["publication"], publication) + require(expected["core_sha256"] == core_sha and hashlib.sha256(raw_file(viewer / "expected.json", 192*1024)).hexdigest() == result["expected_sha256"]) + return result + + +def cache_state(path, uid, gid): + require(path.name == "weather-cache" and path.parent.name == "state-client" and path.resolve(strict=True) == path) + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700 and info.st_uid == uid and info.st_gid == gid) + return dict(device=info.st_dev, inode=info.st_ino, mode="0700", user_uid=uid, group_gid=gid) + + +def cleanup(root, publisher): + require(root.name == ("weather-publisher" if publisher else "weather-viewer") and not root.is_symlink()) + if not root.exists(): return dict(directory_removed=True, files_removed=True) + private(root) + allowed = {"passphrase", "identity.key", "forecast.json", "manifest.bin", "publish-receipt.json", "source-cache", "scratch"} if publisher else { + "selection.json", "forecast.json", "cold-receipt.json", "config.json", "expected.json", "browser-output"} + files, directories, total = [], [], 0 + for path in root.iterdir(): + require(path.name in allowed) + if path.is_dir(): + private(path); directories.append(path) + children = list(path.iterdir()); require(len(children) <= 20) + for child in children: + require(child.name == "report.json" if path.name == "browser-output" else + path.name == "source-cache" and (re.fullmatch(r"[a-f0-9]{64}", child.name) + or child.name in (".volparossa-owner-v1", ".volparossa-index-v1"))) + total += len(raw_file(child, MAX_FORECAST)); files.append(child) + else: + total += len(raw_file(path, 192*1024)); files.append(path) + require(total <= 2*1024*1024) + for path in files: path.unlink() + for path in directories: path.rmdir() + root.rmdir() + return dict(directory_removed=True, files_removed=True) + + +def validate_packets(cold, peers, layout, route, captures, control): + receipt = cold["receipt"]; actual = set(receipt["provider_peer_ids"]) + nodes = layout["provider_nodes"] + require(len(nodes) == len(set(nodes)) == 2 and set(nodes) <= set(HTTPS["CANDIDATES"]) + and actual <= {peers[node] for node in nodes} and receipt["control_relay_peer_id"] == layout["control_relay_peer_id"]) + require(set(captures) == set(HTTPS["ROLES"])) + for role, capture in captures.items(): + HTTPS["validate_drained"](capture, allow_empty=True) + require(capture["capture_role"] == role and capture["content_provider_mode"] is True + and all(capture[key] == 0 for key in ("unexpected_outer_packets", "expected_link_down_notifications", + "unexpected_provider_application_packets", "direct_client_exit_packets")) + and set(capture["provider_application"]) == set(HTTPS["CANDIDATES"])) + require(capture["client_public_packets"] == capture["outbound_client_discovery_attempt_packets"] == 0 + if role == "exit" else capture["internet_destination_outer_packets"] == 0) + for node, counters in capture["provider_application"].items(): + if role == "exit" and node in nodes: + if peers[node] in actual: + require(counters["request_packets"] > 0 and counters["response_packets"] > 0 + and counters["response_payload_bytes"] >= receipt["bytes"]) + else: require(all(value == 0 for value in counters.values())) + control_node = next(node for node in HTTPS["PUBLIC_IPS"] if peers[node] == layout["control_relay_peer_id"]) + HTTPS["validate_control"](control, control_node, nodes, False, require_contacts=True) + paths, slots = route["paths"], route["benchmark_slots"] + require(route["transport"] == "mptcp" and len(paths) == len(slots) == 2 + and re.fullmatch(r"[a-f0-9]{32}", route["route_context_id"]) + and len({path["relay_peer_id"] for path in paths}) == 2 and {path["exit_peer_id"] for path in paths} == {peers["exit"]} + and all(path["route_context_id"] == route["route_context_id"] for path in paths) + and [slot["relay_peer_id"] for slot in slots] == [path["relay_peer_id"] for path in paths] + and all(slot["relay_node"] in HTTPS["ROLES"][1:4] and peers[slot["relay_node"]] == slot["relay_peer_id"] for slot in slots) + and {peers[node] for node in nodes}.isdisjoint({peers["client"], peers["exit"], layout["control_relay_peer_id"]} + | {path["relay_peer_id"] for path in paths}) + and any(captures[slot["relay_node"]]["client_leg_wireguard_data_datagrams"] > 0 + and captures[slot["relay_node"]]["exit_leg_wireguard_data_datagrams"] > 0 for slot in slots)) + + +def closed_report(report): + require(type(report) is dict and set(report) <= {"schema", "passed", "phase", "source", "expected", "scope", "checks", "cleanup", "outer_cleanup", "failure"}) + same(report["schema"], 1); require(type(report["passed"]) is bool) + require(report["phase"] in {"inputs", "sandbox", "bridge_start", "provider", "browser_start", "hydration", "refresh", "shutdown", "complete"}) + if "failure" in report: require(report["failure"] in ("interrupted", "check_or_runtime_failed")) + source = report["source"] + require(set(source) == {"bundle_manifest_sha256", "driver_sha256", "preparer_sha256", "built_tree_sha256", "node_sha256", "bridge_modules"}) + require(all(hex64(v) for k, v in source.items() if k != "bridge_modules")) + require(set(source["bridge_modules"]) == {"scripts/serve-provider.mjs", "src/provider-http.mjs", "src/provider-core.mjs", "src/core-cache.mjs"} + and all(hex64(v) for v in source["bridge_modules"].values())) + expected = report["expected"] + require(set(expected) == {"expected_sha256", "cold_receipt_sha256", "core_sha256", "display_sha256", "forecast_sha256", "manifest_id"} + and all(hex64(v) or (k == "expected_sha256" and v is None) for k, v in expected.items())) + require(set(report["scope"]) == {"actual_drizzli_ui", "actual_http_bridge", "real_core_adapter", "synthetic_weather", + "standalone_origin_proven", "standalone_overlay_proven", "browser_triggered_cold_fetch"} + and all(type(v) is bool for v in report["scope"].values())) + require(set(report["checks"]) <= {"api_response_sha256", "initial_dom_matches_cold_publication", "refresh_dom_matches_cold_publication", + "browser_api_requests", "provider_requests_checked", "exact_native_times_and_values", "explicit_missing_capabilities", "no_external_resources", "source_attribution", + "initial_dom_display_sha256", "refresh_dom_display_sha256"}) + for key, value in report["checks"].items(): + require(hex64(value) if key.endswith("sha256") else type(value) is int and 0 <= value <= 3 + if key.endswith("requests") or key == "provider_requests_checked" else type(value) is bool) + for name, allowed in (("cleanup", {"browser_exited", "bridge_exited", "private_state_removed", "normal_browser_stop", "normal_bridge_stop"}), + ("outer_cleanup", {"namespace_wrapper_exited", "private_state_removed", "client_namespace_unchanged"})): + if name in report: + require(set(report[name]) <= allowed and all(type(value) is bool for value in report[name].values())) + return report + + +def retain(work): + path = work / "client-fixtures/weather-viewer/browser-output/report.json" + if not path.exists(): return dict(retained=False) + raw = worker_report(path); closed_report(json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"])) + write_new(work / "weather-client-delivery-original-report.json", raw) + return dict(retained=True, sha256=hashlib.sha256(raw).hexdigest()) + + +def worker_report(path): + # Only this exact private worker output crosses into root-owned evidence. + require(path.parts[-4:] == ("client-fixtures", "weather-viewer", "browser-output", "report.json")) + with os.fdopen(os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_uid == 985 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 and 0 < info.st_size <= 16384) + raw = stream.read(16385) + require(len(raw) == info.st_size and B["identity"](os.fstat(stream.fileno())) == B["identity"](info)) + return raw + + +def validate_acquisition(value): + same({k: v for k, v in value.items() if k not in ("forecast_sha256", "forecast_bytes", "fields")}, + dict(schema=1, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, decoder_sha256=B["DECODER_SHA"])) + require(hex64(value["forecast_sha256"]) and type(value["forecast_bytes"]) is int and 0 < value["forecast_bytes"] <= MAX_FORECAST + and type(value["fields"]) is list and len(value["fields"]) == 9) + coverage = [] + for field in value["fields"]: + require(set(field) == {"lead_hours", "parameter", "index_sha256", "field_sha256"} + and type(field["lead_hours"]) is int and field["lead_hours"] in (3, 9, 24) + and field["parameter"] in ("2t", "10u", "10v") and hex64(field["index_sha256"]) and hex64(field["field_sha256"])) + coverage.append((field["lead_hours"], field["parameter"])) + require(set(coverage) == {(step, parameter) for step in (3, 9, 24) for parameter in ("2t", "10u", "10v")}) + + +def validate_publications(work, publication, nodes): + published = read(work / "weather-client-delivery-publish.json") + require(published["operation"] == "offline_content_publish" and published["network_publication"] is False + and published["publisher_key_hex"] == publication["publisher_key"] and published["content_type"] == "application/json" + and published["chunks"] == 1 and all(published[k] == publication[k] for k in + ("name", "revision", "bytes", "manifest_id", "expires_unix_seconds"))) + for node in nodes: + imported = read(work / f"weather-client-delivery-{node}-import.json") + served = read(work / f"weather-client-delivery-{node}-serve.json") + require(imported["operation"] == "content_import" and imported["complete"] is True + and imported["manifest_id"] == publication["manifest_id"] and imported["content_bytes"] == publication["bytes"] + and imported["agent_cache"] == str(work / f"state-{node}/weather-cache") and imported["public_content"] is True + and all(imported[k] is False for k in ("ownership_changed", "network_transfer", "origin_authenticated")) + and served["serving"] is True and type(served["publications"]) is int and served["publications"] == 4 + and served["replication_enabled"] is False) + + +def evidence(work, manifest_sha, binary_sha): + selected = validate_selection(read(work / "weather-client-delivery-input.json")); publication = selected["publication"] + acquisition = read(work / "weather-client-delivery-acquisition.json") + validate_acquisition(acquisition) + cold = read(work / "weather-client-delivery-cold-consumer.json") + same(cold["publication"], publication) + validate_cold_receipt(cold["receipt"], publication, work / "state-client/weather-cache", work / "client-fixtures/weather-viewer/forecast.json") + same(cold["cold_cache_requested"], True); require(cold["core_sha256"] == binary_sha) + same(cold["schema"], 1) + require(hex64(cold["cold_receipt_sha256"]) and type(cold["observed_unix_seconds"]) is int + and 0 < cold["observed_unix_seconds"] < publication["expires_unix_seconds"]) + control_gid = (work / "runtime-client/control/agent.sock").stat().st_gid + same(cold["boundary"], dict(user_uid=985, primary_control_gid=control_gid, supplementary_groups_empty=True, + all_capabilities_dropped=True, no_new_privileges=True, client_namespace=True, acquisition_parent_namespace=False)) + peers, layout = read(work / "a01-expected-peers.json"), read(work / "content-provider-layout.json") + validate_publications(work, publication, layout["provider_nodes"]) + validate_packets(cold, peers, layout, read(work / "weather-client-delivery-selection.json"), + {role: read(work / f"weather-client-delivery-privacy-{role}.json") for role in HTTPS["ROLES"]}, + read(work / "weather-client-delivery-control.json")) + same(read(work / "weather-client-delivery-publisher-cleanup.json"), dict(directory_removed=True, files_removed=True)) + same(read(work / "weather-client-delivery-input-cleanup.json"), dict(directory_removed=True, files_removed=True)) + require(all(not path.exists() and not path.is_symlink() for path in + (work / "client-fixtures/weather-publisher", work / "client-fixtures/weather-viewer"))) + before, after = (read(work / f"weather-client-delivery-cache-{phase}.json") for phase in ("before", "after")) + same(before, after); same(after, cache_state(work / "state-client/weather-cache", after["user_uid"], after["group_gid"])) + require(after["user_uid"] != 985 and cold["boundary"]["primary_control_gid"] != after["group_gid"]) + # A root-owned manifest was retained before the verified tree was handed to + # UID985. Worker verify-tree receipts check that tree before and after use; + # root never weakens owner checks or grants access to private agent caches. + manifest = B["manifest"](raw_file(work / "weather-client-delivery-manifest.json", B["MAX_MANIFEST"], 0o400), manifest_sha) + checked = dict(manifest_sha256=manifest_sha, files=len(manifest["files"]), runtime_executed=False) + for phase in ("before", "after"): + same(read(work / f"weather-client-delivery-tree-{phase}.json"), dict(version=1, action="verify-tree", **checked)) + report = closed_report(read(work / "weather-client-delivery-original-report.json", 16384)) + same(read(work / "weather-client-delivery-prepared.json"), dict(prepared=True, + expected_sha256=report["expected"]["expected_sha256"], display_sha256=report["expected"]["display_sha256"])) + require(report["passed"] is True and report["phase"] == "complete" and "failure" not in report) + same(report["scope"], dict(actual_drizzli_ui=True, actual_http_bridge=True, real_core_adapter=True, + synthetic_weather=False, standalone_origin_proven=False, standalone_overlay_proven=False, browser_triggered_cold_fetch=False)) + require(report["source"]["bundle_manifest_sha256"] == manifest_sha and report["expected"]["core_sha256"] == binary_sha + and report["expected"]["cold_receipt_sha256"] == cold["cold_receipt_sha256"] + and report["expected"]["forecast_sha256"] == acquisition["forecast_sha256"] == publication["sha256"] + and acquisition["forecast_bytes"] == publication["bytes"] and report["expected"]["manifest_id"] == publication["manifest_id"]) + for field, relative in (("driver_sha256", "integrations/drizzli/smoke_core_browser.py"), + ("preparer_sha256", "integrations/drizzli/prepare-core-browser.mjs")): + require(report["source"][field] == manifest["files"][B["WEATHER"] + relative]["sha256"]) + for relative, digest in report["source"]["bridge_modules"].items(): require(digest == manifest["files"][B["WEATHER"] + relative]["sha256"]) + require(report["source"]["node_sha256"] == B["NODE_SHA"]) + static = {name[len(B["STATIC"]):]: item["sha256"] for name, item in manifest["files"].items() if name.startswith(B["STATIC"])} + require(report["source"]["built_tree_sha256"] == hashlib.sha256(json.dumps(static, sort_keys=True, separators=(",", ":")).encode()).hexdigest()) + same(report["checks"], dict(api_response_sha256=report["checks"]["api_response_sha256"], + initial_dom_display_sha256=report["expected"]["display_sha256"], refresh_dom_display_sha256=report["expected"]["display_sha256"], + initial_dom_matches_cold_publication=True, refresh_dom_matches_cold_publication=True, browser_api_requests=2, + provider_requests_checked=1, exact_native_times_and_values=True, explicit_missing_capabilities=True, + no_external_resources=True, source_attribution=True)) + same(report["cleanup"], dict(browser_exited=True, bridge_exited=True, private_state_removed=True, normal_browser_stop=True, normal_bridge_stop=True)) + same(report["outer_cleanup"], dict(namespace_wrapper_exited=True, private_state_removed=True, client_namespace_unchanged=True)) + return dict(schema=1, success=True, kind="volparossa-weather-client-delivery", bundle_manifest_sha256=manifest_sha, + core_sha256=binary_sha, publication=publication, acquisition=acquisition, + protected_cold=dict(peer_bytes=cold["receipt"]["peer_bytes"], providers_used=cold["receipt"]["providers_used"], + origin_body_bytes=0, origin_range_requests=0, privacy_captures_validated=True, cold_receipt_sha256=cold["cold_receipt_sha256"]), + browser_report_sha256=hashlib.sha256(raw_file(work / "weather-client-delivery-original-report.json")).hexdigest(), + scope=dict(actual_acquisition=True, same_publication_joined=True, actual_ui_refresh=True, + browser_triggered_cold_fetch=False, browser_cache_only_proven=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False), + cleanup=dict(publisher_sources_removed_before_fetch=True, browser_private_state_removed=True, + viewer_removed=True, agent_cache_identity_preserved=True, agent_cache_private=True)) + + +def copy_artifacts(work, output, uid, gid): + # Closed identifiers only; original page/forecast/receipt/paths never cross this boundary. + pending = []; count = 0 + for suffix in EXPORTS: + source = work / f"weather-client-delivery-{suffix}.json" + if not source.exists(): continue + raw = raw_file(source, MAX_REPORT); value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + if suffix == "original-report": closed_report(value) + else: validate_export(value) + target = output / source.name + if target.exists() or target.is_symlink(): + require(target.resolve(strict=True) == target) + with os.fdopen(os.open(target, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as existing: + info = os.fstat(existing.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == uid and info.st_gid == gid + and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size == len(raw)) + require(existing.read(MAX_REPORT+1) == raw and B["identity"](os.fstat(existing.fileno())) == B["identity"](info)) + else: + B["new_path"](target) + pending.append((target, raw)) + count += 1 + # Reject every candidate before creating the first exported artifact. + for target, raw in pending: + write_new(target, raw); os.chown(target, uid, gid) + return dict(files=count, raw_logs_exported=False) + + +EXPORT_FIELDS = {"schema", "version", "success", "kind", "bundle_manifest_sha256", "core_sha256", "publication", "publisher_key", + "name", "revision", "manifest_id", "sha256", "bytes", "expires_unix_seconds", "acquisition", "origin_https", + "acquisition_outside_overlay", "upstream_signature_attested", "synthetic_weather", "forecast_sha256", "forecast_bytes", + "decoder_sha256", "fields", "lead_hours", "parameter", "index_sha256", "field_sha256", "protected_cold", "peer_bytes", + "providers_used", "origin_body_bytes", "origin_range_requests", "privacy_captures_validated", "cold_receipt_sha256", + "browser_report_sha256", "scope", "actual_acquisition", "same_publication_joined", "actual_ui_refresh", + "browser_triggered_cold_fetch", "browser_cache_only_proven", "two_subflow_bulk_aggregation", "production_service_complete", + "cleanup", "publisher_sources_removed_before_fetch", "browser_private_state_removed", "viewer_removed", + "agent_cache_identity_preserved", "agent_cache_private", "device", "inode", "mode", "user_uid", "group_gid", + "directory_removed", "files_removed", "action", "manifest_sha256", "files", "runtime_executed"} + + +def validate_export(value, depth=0): + require(depth <= 8) + if type(value) is dict: + require(set(value) <= EXPORT_FIELDS) + for child in value.values(): validate_export(child, depth+1) + elif type(value) is list: + require(len(value) <= 16) + for child in value: validate_export(child, depth+1) + elif type(value) is str: + require(hex64(value) or value in {NAME, "volparossa-weather-client-delivery", "verify-tree", "0700", "2t", "10u", "10v"}) + else: require(type(value) is bool or type(value) is int and 0 <= value < 2**64) + + +def main(): + action, *args = sys.argv[1:] + if action == "init" and len(args) == 1: result = initialize(Path(args[0])) + elif action == "acquire" and len(args) == 6: result = acquisition(Path(args[0]), args[1], Path(args[2]), args[3], args[4], int(args[5])) + elif action == "select" and len(args) == 3: result = select(*map(Path, args)) + elif action == "cold" and len(args) == 7: result = cold(*map(Path, args[:4]), args[4], args[5], int(args[6])) + elif action == "prepare" and len(args) == 5: result = prepare(*map(Path, args)) + elif action in ("publisher-cleanup", "user-cleanup") and len(args) == 1: result = cleanup(Path(args[0]), action == "publisher-cleanup") + elif action == "cache-state" and len(args) == 3: result = cache_state(Path(args[0]), int(args[1]), int(args[2])) + elif action == "retain" and len(args) == 1: result = retain(Path(args[0])) + elif action == "evidence" and len(args) == 3: result = evidence(Path(args[0]), args[1], args[2]) + elif action == "copy-artifacts" and len(args) == 4: result = copy_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3])) + else: raise ValueError("weather_client_delivery_rejected") + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + + +if __name__ == "__main__": + try: main() + except (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError): + print('{"success":false,"failure":"weather_client_delivery_rejected"}') + raise SystemExit(1) diff --git a/tests/integration/weather-client-delivery.sh b/tests/integration/weather-client-delivery.sh new file mode 100644 index 000000000..cc1d397f6 --- /dev/null +++ b/tests/integration/weather-client-delivery.sh @@ -0,0 +1,255 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only +# Explicit optional joined acquisition -> protected cold fetch -> native Weather UI. +# Sourced only by the disposable KVM topology; no host browser or network setup. +# shellcheck disable=SC2154,SC2034 + +# Ownership is established by this invocation, never inherited from a caller. +WEATHER_CLIENT_HOME_CREATED=no +WEATHER_CLIENT_HOME_IDENTITY= + +weather_client_private() { + setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-client-delivery.py" "$@" +} + +weather_client_cli() { + weather_socket_node=$1 + shift + timeout --signal=TERM --kill-after=5s 120s setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- "$binary_directory/volparossa" \ + --control-socket "$WORK/runtime-$weather_socket_node/control/agent.sock" "$@" +} + +weather_client_home_prepare() { + # The browser maps its isolated home to the account's real passwd home. + # Never replace HOME, change an existing home, or create this on the host. + [ "$(id -u)" -eq 0 ] && [ "$(systemd-detect-virt)" = kvm ] \ + && [ "$(sed -n '1p' /proc/1/comm)" = systemd ] || return 1 + [ "$WORKER_UID" = 985 ] \ + && [ "$(getent passwd 985 | cut -d: -f6)" = /nonexistent ] || return 1 + [ ! -L /nonexistent ] || return 1 + if [ ! -e /nonexistent ]; then + mkdir -m 0755 -- /nonexistent || return 1 + WEATHER_CLIENT_HOME_CREATED=yes + WEATHER_CLIENT_HOME_IDENTITY=$(stat -Lc '%d:%i' /nonexistent) || return 1 + fi + [ -d /nonexistent ] && [ "$(readlink -e -- /nonexistent)" = /nonexistent ] \ + && [ "$(stat -Lc '%u:%g:%a' /nonexistent)" = 0:0:755 ] || return 1 + weather_home_entry=$(find /nonexistent -mindepth 1 -maxdepth 1 -print -quit) || return 1 + [ -z "$weather_home_entry" ] +} + +weather_client_home_cleanup() { + [ "${WEATHER_CLIENT_HOME_CREATED:-no}" = yes ] || return 0 + [ "$(id -u)" -eq 0 ] && [ "$(systemd-detect-virt)" = kvm ] || return 1 + [ -n "${WEATHER_CLIENT_HOME_IDENTITY:-}" ] && [ ! -L /nonexistent ] \ + && [ -d /nonexistent ] && [ "$(readlink -e -- /nonexistent)" = /nonexistent ] \ + && [ "$(stat -Lc '%u:%g:%a' /nonexistent)" = 0:0:755 ] \ + && [ "$(stat -Lc '%d:%i' /nonexistent)" = "$WEATHER_CLIENT_HOME_IDENTITY" ] || return 1 + # rmdir refuses a nonempty/replaced directory; never recursively remove it. + rmdir -- /nonexistent || return 1 + WEATHER_CLIENT_HOME_CREATED=no + WEATHER_CLIENT_HOME_IDENTITY= +} + +weather_client_delivery_cleanup() { + weather_cleanup_status=0 + if [ -f "$WORK/bin/weather-client-delivery.py" ] && [ -n "${site_control_gid:-}" ]; then + # An interruption may enter the topology trap before the normal retain + # step. Preserve only the strictly validated closed report, never logs. + if [ ! -e "$WORK/weather-client-delivery-original-report.json" ] \ + && [ ! -L "$WORK/weather-client-delivery-original-report.json" ]; then + if [ -e "$WORK/client-fixtures/weather-viewer/browser-output/report.json" ] \ + || [ -L "$WORK/client-fixtures/weather-viewer/browser-output/report.json" ]; then + python3 -B "$WORK/bin/weather-client-delivery.py" retain "$WORK" \ + >"$WORK/weather-client-delivery-retained.json" || weather_cleanup_status=1 + fi + fi + weather_client_private publisher-cleanup "$WORK/client-fixtures/weather-publisher" \ + >"$WORK/weather-client-delivery-publisher-cleanup.json" || weather_cleanup_status=1 + weather_client_private user-cleanup "$WORK/client-fixtures/weather-viewer" \ + >"$WORK/weather-client-delivery-input-cleanup.json" || weather_cleanup_status=1 + fi + # Independent attempts: a failed user cleanup must not skip the owned home. + weather_client_home_cleanup || weather_cleanup_status=1 + return "$weather_cleanup_status" +} + +weather_client_delivery_copy_artifacts() { + [ -f "$WORK/bin/weather-client-delivery.py" ] || return 0 + python3 -B "$WORK/bin/weather-client-delivery.py" copy-artifacts "$WORK" \ + "$output_directory" "$OUTPUT_UID" "$OUTPUT_GID" >/dev/null +} + +weather_client_verify_tree() { + setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-client-bundle.py" verify-tree \ + "$weather_client_directory" "$weather_client_manifest_sha256" +} + +weather_client_delivery_run() { + if [ -z "${weather_client_bundle:-}" ] || [ -z "${weather_client_manifest_sha256:-}" ]; then + fail WEATHER_CLIENT_INPUTS_MISSING + fi + PHASE=weather-client-delivery-prepare + printf '%s\n' 'Weather trial: stage verified inputs under disposable WORK; acquire one explicit public ECMWF forecast in the guest parent namespace; publish to two existing peers; cold-fetch through the protected Client route, then render and refresh the same bytes in an isolated browser. Create the empty root-owned /nonexistent only if absent in this KVM guest; remove only that owned empty directory. No HOME override or private cache permission change.' + if [ "$WORKER_UID" != 985 ] || [ "$site_control_gid" = "$AGENT_GID" ]; then + fail WEATHER_CLIENT_USER_INVALID + fi + weather_client_directory=$WORK/client-fixtures/weather-client-source + weather_publisher=$WORK/client-fixtures/weather-publisher + weather_viewer=$WORK/client-fixtures/weather-viewer + weather_cache=$WORK/state-client/weather-cache + for weather_new in "$weather_client_directory" "$weather_publisher" "$weather_viewer" "$weather_cache" \ + "$WORK/state-$provider_node_a/weather-cache" "$WORK/state-$provider_node_b/weather-cache"; do + if [ -e "$weather_new" ] || [ -L "$weather_new" ]; then fail WEATHER_CLIENT_PATH_EXISTS; fi + done + for weather_script in weather-client-delivery.py weather-client-bundle.py browser-filter-bundle.py \ + content-provider-https-smoke.py content-network-smoke.py; do + install -o root -g root -m 0555 "$source_directory/tests/integration/$weather_script" \ + "$WORK/bin/$weather_script" || fail WEATHER_CLIENT_DRIVER_UNAVAILABLE + done + install -o root -g root -m 0600 "$weather_client_bundle" "$WORK/weather-client-inputs.tar" \ + || fail WEATHER_CLIENT_ARCHIVE_COPY_FAILED + python3 -B "$WORK/bin/weather-client-bundle.py" unpack "$WORK/weather-client-inputs.tar" \ + "$weather_client_directory" "$weather_client_manifest_sha256" \ + >"$WORK/weather-client-delivery-unpack.json" || fail WEATHER_CLIENT_UNPACK_FAILED + # Preserve proof metadata under root ownership before handing this validated + # new source tree to the worker. No existing user/agent state is chowned. + install -o root -g root -m 0400 "$weather_client_directory/INPUTS.json" \ + "$WORK/weather-client-delivery-manifest.json" || fail WEATHER_CLIENT_MANIFEST_RETENTION_FAILED + chown -R "$WORKER_UID:$site_control_gid" "$weather_client_directory" || fail WEATHER_CLIENT_OWNER_FAILED + install -d -o "$WORKER_UID" -g "$site_control_gid" -m 0700 "$weather_publisher" "$weather_viewer" \ + || fail WEATHER_CLIENT_PRIVATE_DIRECTORY_FAILED + weather_client_verify_tree >"$WORK/weather-client-delivery-tree-before.json" \ + || fail WEATHER_CLIENT_SOURCE_CHANGED + weather_client_home_prepare || fail WEATHER_CLIENT_HOME_UNSAFE + weather_core_sha256=$(sha256sum "$binary_directory/volparossa" | awk '{print $1}') + weather_client_private init "$weather_publisher" >"$WORK/weather-client-delivery-assets.json" \ + || fail WEATHER_CLIENT_INPUT_FAILED + PHASE=weather-client-delivery-acquire + timeout --signal=TERM --kill-after=10s 400s setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-client-delivery.py" acquire \ + "$weather_client_directory" "$weather_client_manifest_sha256" "$weather_publisher" \ + "$site_parent_ns" "$site_client_ns" "$site_control_gid" \ + >"$WORK/weather-client-delivery-acquisition.json" 2>"$WORK/weather-client-delivery-acquisition.err" \ + || fail WEATHER_CLIENT_ACQUISITION_FAILED + PHASE=weather-client-delivery-publish + weather_client_cli "$provider_node_a" init --identity "$weather_publisher/identity.key" \ + --passphrase-file "$weather_publisher/passphrase" >"$WORK/weather-client-delivery-init.log" \ + 2>"$WORK/weather-client-delivery-init.err" || fail WEATHER_CLIENT_IDENTITY_FAILED + weather_client_cli "$provider_node_a" content publish --input "$weather_publisher/forecast.json" \ + --name disposable-public-community-weather --revision 1 --content-type application/json \ + --lifetime-seconds 900 --identity "$weather_publisher/identity.key" \ + --passphrase-file "$weather_publisher/passphrase" --cache "$weather_publisher/source-cache" \ + --manifest "$weather_publisher/manifest.bin" >"$WORK/weather-client-delivery-publish.json" \ + 2>"$WORK/weather-client-delivery-publish.err" || fail WEATHER_CLIENT_PUBLISH_FAILED + weather_key=$(jq -er '.publisher_key_hex | select(test("^[0-9a-f]{64}$"))' \ + "$WORK/weather-client-delivery-publish.json") || fail WEATHER_CLIENT_PUBLISHER_INVALID + if [ "$weather_key" = "$provider_publisher" ] || [ "$weather_key" = "$site_key" ] \ + || [ "$weather_key" = "$filter_key" ]; then fail WEATHER_CLIENT_PUBLISHER_REUSED; fi + install -o "$WORKER_UID" -g "$site_control_gid" -m 0400 "$WORK/weather-client-delivery-publish.json" \ + "$weather_publisher/publish-receipt.json" || fail WEATHER_CLIENT_PUBLISH_RECEIPT_UNAVAILABLE + weather_client_private select "$weather_publisher" "$weather_publisher/publish-receipt.json" "$weather_viewer" \ + >"$WORK/weather-client-delivery-input.json" || fail WEATHER_CLIENT_SELECTION_FAILED + PHASE=weather-client-delivery-register + for weather_node in "$provider_node_a" "$provider_node_b"; do + case $weather_node in + relay4) weather_address=49.165.5.1; weather_hostname=provider-a.volparossa.test ;; + relay5) weather_address=50.166.6.1; weather_hostname=provider-b.volparossa.test ;; + relay3) weather_address=48.164.4.1; weather_hostname=provider-c.volparossa.test ;; + *) fail WEATHER_CLIENT_PROVIDER_INVALID ;; + esac + weather_client_cli "$weather_node" content import --public-content \ + --manifest "$weather_publisher/manifest.bin" --publisher-key "$weather_key" \ + --cache "$weather_publisher/source-cache" --agent-cache "$WORK/state-$weather_node/weather-cache" \ + >"$WORK/weather-client-delivery-$weather_node-import.json" \ + 2>"$WORK/weather-client-delivery-$weather_node-import.err" || fail WEATHER_CLIENT_IMPORT_FAILED + weather_client_cli "$weather_node" content serve --name-lookup \ + --manifest "$weather_publisher/manifest.bin" --publisher-key "$weather_key" \ + --cache "$WORK/state-$weather_node/weather-cache" --bind "$weather_address:18080" \ + --advertised-hostname "$weather_hostname" >"$WORK/weather-client-delivery-$weather_node-serve.json" \ + 2>"$WORK/weather-client-delivery-$weather_node-serve.err" || fail WEATHER_CLIENT_SERVE_FAILED + jq -e '.serving == true and .publications == 4 and .replication_enabled == false' \ + "$WORK/weather-client-delivery-$weather_node-serve.json" >/dev/null || fail WEATHER_CLIENT_NOT_READY + if [ -L "$WORK/state-$weather_node/weather-cache" ] \ + || [ "$(stat -Lc '%a:%u:%g' "$WORK/state-$weather_node/weather-cache")" != "700:$AGENT_UID:$AGENT_GID" ]; then + fail WEATHER_CLIENT_PROVIDER_CACHE_CHANGED + fi + if nsenter --target "$provider_client_pid" --mount \ + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$WORK/state-$weather_node/weather-cache"; then fail WEATHER_CLIENT_LOCAL_SHORTCUT; fi + if setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$WORK/state-$weather_node/weather-cache"; then fail WEATHER_CLIENT_AGENT_CACHE_EXPOSED; fi + done + nsenter --target "$provider_client_pid" --mount \ + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$WORK/state-client/identity.key" || fail WEATHER_CLIENT_ISOLATION_PROBE_FAILED + if nsenter --target "$provider_client_pid" --mount \ + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$weather_viewer"; then fail WEATHER_CLIENT_USER_STATE_EXPOSED; fi + # All publisher commands exited. Remove the identity, original forecast, + # manifest and source chunks before an independent Client resolves the name. + weather_client_private publisher-cleanup "$weather_publisher" \ + >"$WORK/weather-client-delivery-publisher-cleanup.json" || fail WEATHER_CLIENT_PUBLISHER_CLEANUP_FAILED + if [ -e "$weather_publisher" ] || [ -L "$weather_publisher" ]; then + fail WEATHER_CLIENT_PUBLISHER_FILES_REMAIN + fi + PHASE=weather-client-delivery-cold + start_privacy_observers weather-client-delivery-privacy || fail WEATHER_CLIENT_CAPTURE_UNAVAILABLE + content_provider_start_control_observer weather-client-delivery-control || fail WEATHER_CLIENT_CONTROL_UNAVAILABLE + timeout --signal=TERM --kill-after=10s 200s ip netns exec "$CLIENT" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-client-delivery.py" cold "$binary_directory/volparossa" \ + "$WORK/runtime-client/control/agent.sock" "$weather_cache" "$weather_viewer" \ + "$site_parent_ns" "$site_client_ns" "$site_control_gid" \ + >"$WORK/weather-client-delivery-cold-consumer.json" 2>"$WORK/weather-client-delivery-cold.err" \ + || fail WEATHER_CLIENT_COLD_FAILED + benchmark_capture_paths weather-client-delivery mptcp || fail WEATHER_CLIENT_PATHS_UNAVAILABLE + stop_privacy_observers || fail WEATHER_CLIENT_CAPTURE_INCOMPLETE + content_provider_stop_control_observer || fail WEATHER_CLIENT_CONTROL_INCOMPLETE + python3 -B "$WORK/bin/weather-client-delivery.py" cache-state "$weather_cache" "$AGENT_UID" "$AGENT_GID" \ + >"$WORK/weather-client-delivery-cache-before.json" || fail WEATHER_CLIENT_CACHE_INVALID + weather_client_private prepare "$weather_client_directory" "$weather_viewer" \ + "$binary_directory/volparossa" "$WORK/runtime-client/control/agent.sock" "$weather_cache" \ + >"$WORK/weather-client-delivery-prepared.json" || fail WEATHER_CLIENT_EXPECTED_FAILED + PHASE=weather-client-delivery-browser + weather_browser_status=0 + timeout --signal=TERM --kill-after=10s 270s ip netns exec "$CLIENT" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- env -i PATH=/usr/bin:/bin LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONDONTWRITEBYTECODE=1 \ + python3 -B "$weather_client_directory/volparossa-weather/integrations/drizzli/smoke_core_browser.py" \ + --execute --bundle-root "$weather_client_directory" --bundle-manifest-sha256 "$weather_client_manifest_sha256" \ + --config "$weather_viewer/config.json" --expected "$weather_viewer/expected.json" \ + --output "$weather_viewer/browser-output" --parent-netns "$site_parent_ns" --client-netns "$site_client_ns" \ + --control-gid "$site_control_gid" >"$WORK/weather-client-delivery-launch.json" \ + 2>"$WORK/weather-client-delivery-browser.err" || weather_browser_status=$? + PHASE=weather-client-delivery-verify + python3 -B "$WORK/bin/weather-client-delivery.py" retain "$WORK" \ + >"$WORK/weather-client-delivery-retained.json" || fail WEATHER_CLIENT_RECEIPT_RETENTION_FAILED + weather_client_delivery_cleanup || fail WEATHER_CLIENT_INPUT_CLEANUP_FAILED + [ "$weather_browser_status" -eq 0 ] || fail WEATHER_CLIENT_BROWSER_FAILED + weather_client_verify_tree >"$WORK/weather-client-delivery-tree-after.json" \ + || fail WEATHER_CLIENT_SOURCE_CHANGED + python3 -B "$WORK/bin/weather-client-delivery.py" cache-state "$weather_cache" "$AGENT_UID" "$AGENT_GID" \ + >"$WORK/weather-client-delivery-cache-after.json" || fail WEATHER_CLIENT_CACHE_INVALID + if setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$weather_cache"; then fail WEATHER_CLIENT_AGENT_CACHE_EXPOSED; fi + python3 -B "$WORK/bin/weather-client-delivery.py" evidence "$WORK" "$weather_client_manifest_sha256" \ + "$weather_core_sha256" >"$WORK/weather-client-delivery-evidence.json" || fail WEATHER_CLIENT_EVIDENCE_INVALID + PHASE=weather-client-delivery-complete +} From db5eb44fb3983047b5a8f5c8b6eaac8b8fc777f2 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:01:55 +0200 Subject: [PATCH 14/49] docs: record first joined Weather trial and precipitation evidence --- docs/IMPLEMENTATION_STATUS.md | 37 +++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index a3bdf1864..6b9562598 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -274,9 +274,9 @@ The final repository CI and CodeQL checks pass. A real core-cache publication reaching the browser remains unproven; the synthetic trial does not establish live overlay or weather-origin delivery. -The next local proof fixture is prepared: acquire fresh public ECMWF data, -publish it to peers, require a cold protected core transfer, and display and -refresh the same publication through the production bridge in Drizz.li. Origin +The local proof fixture acquires fresh public ECMWF data, +publishes it to peers, requires a cold protected core transfer, and attempts to +display and refresh the same publication through the production bridge in Drizz.li. Origin HTTPS acquisition is distinct from the subsequent overlay transfer. Explicit bundle selection, immutable input checks, private disk scratch and the existing 4 GiB guest plus 2 GiB host-memory preflight are wired into the disposable runner; @@ -289,11 +289,32 @@ independently verified without executing a runtime; its manifest is `e77f3240ab93a836f3ef19393909e4d0809a2d1a75fd143613e6a476f858c905` and archive SHA-256 is `4843f20f8f47db5f82f5902217514c184dadc31d3e5fe7c19ca01238b65add68`. -This is preparation, not a successful end-to-end Weather run. - -Overmorrow, additional location selection, precipitation and air quality remain -required work. Unsupported fields are explicitly unavailable, never substituted -with zero values. +The first joined KVM trial ran core `c1418de71b0bb3d4d73600d84277d87df49189ac` +with that bundle and failed at `weather-client-delivery-verify` +(`WEATHER_CLIENT_BROWSER_FAILED`). Real origin acquisition passed for nine +temperature/wind fields, producing a 7,149-byte forecast. The retained browser +report stopped at its outer `inputs` phase, before bridge or browser execution; +it does not prove a successful UI or independently recheckable cold transfer. +A separate minimal namespace reproduction identifies an ownership-check defect: +a root-owned executable becomes unmapped inside the unprivileged user namespace +despite unchanged bytes and a read-only mount. A narrow verifier correction +passes twelve focused tests, including the real minimal namespace case; it +keeps strict outer ownership and checks the exact authorized executable through +a read-only descriptor inside the namespace. No corrected joined run has passed yet. + +The failed trial cleaned up all owned objects. Guest-root and host network state +checks were unchanged. Retained local evidence is under +`.git/ci-evidence/weather-client-delivery-01.Dc1jXf/output`; the closed smoke +report SHA-256 is `9fc045370aa8831a4151c0a3a54febf7d97fd7f8c38164c1e0a0f3b7a01b843f`. +The earlier synthetic browser result remains valid only for its pinned scope. + +In a separate Weather candidate, native precipitation acquisition now passes for +IFS and AIFS Single with the pinned decoder. The shared provider preserves source +accumulation windows and missing values. Its updated Drizz.li component passes +five actual render/compiler tests, the full type check and a networkless build; +this does not prove precipitation browser hydration or overlay delivery. +Overmorrow, additional location selection, air quality and complete application +delivery remain required work. Absent fields are never substituted with zero. ### Citizen procedure simulation From 93dc2a64bd7157a77ee1837e4b8375f9b222fac6 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 03:25:29 +0200 Subject: [PATCH 15/49] test: retain protected weather transfer before browser execution --- docs/IMPLEMENTATION_STATUS.md | 19 ++- tests/integration/run-alpha-topology-vm.sh | 2 +- .../test-weather-client-delivery-shell.py | 40 ++++++ .../test-weather-client-delivery.py | 114 ++++++++++++++++++ .../integration/test-weather-client-runner.py | 36 +++++- tests/integration/weather-client-delivery.py | 87 ++++++++++++- tests/integration/weather-client-delivery.sh | 10 ++ 7 files changed, 297 insertions(+), 11 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 6b9562598..0a8f16a98 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -280,7 +280,7 @@ display and refresh the same publication through the production bridge in Drizz. HTTPS acquisition is distinct from the subsequent overlay transfer. Explicit bundle selection, immutable input checks, private disk scratch and the existing 4 GiB guest plus 2 GiB host-memory preflight are wired into the disposable runner; -the nine runner and closed-export tests pass. The separate real-core browser +the ten runner and closed-export tests pass. The separate real-core browser driver has four Node preparation tests and eight inert Python tests, including exact display/refresh bindings to the cold publication. Ten bundle tests, ten joined-evidence tests and eleven inert shell/cleanup tests pass. The immutable @@ -308,11 +308,24 @@ checks were unchanged. Retained local evidence is under report SHA-256 is `9fc045370aa8831a4151c0a3a54febf7d97fd7f8c38164c1e0a0f3b7a01b843f`. The earlier synthetic browser result remains valid only for its pinned scope. +The corrected fixture now validates and separately retains the protected cold +transfer before starting the browser. Its closed receipt binds acquisition and +publication bytes, core identity, provider traffic and privacy checks, and +explicitly excludes UI or end-to-end success. Fourteen helper tests and twelve +shell tests pass, including preservation across a later invalid UI receipt; +this is fixture behavior, not a newly passed joined network trial. + In a separate Weather candidate, native precipitation acquisition now passes for IFS and AIFS Single with the pinned decoder. The shared provider preserves source accumulation windows and missing values. Its updated Drizz.li component passes -five actual render/compiler tests, the full type check and a networkless build; -this does not prove precipitation browser hydration or overlay delivery. +five actual render/compiler tests, the full type check and a networkless build. +An isolated Firefox run now also passes with synthetic precipitation, preserving +missing, zero and 0.01 mm values and native windows through initial display and +refresh. Expiry, route guards and complete cleanup pass. The separate Weather +receipt `integrations/drizzli/precipitation-browser-smoke.json` has SHA-256 +`cb972cbf4b3930c87df017865c68b1da88f1fa81ed48f23008dd623ba5051a58`; +it explicitly makes no origin, core or overlay-delivery claim. A subsequent +baseline temperature/wind run on the same new build also passes. Overmorrow, additional location selection, air quality and complete application delivery remain required work. Absent fields are never substituted with zero. diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 533e0b7ef..0d6c79807 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1101,7 +1101,7 @@ BROWSER_FILTER_HELPERS = ("browser-filter-delivery.py", "browser-filter-bundle.p "content-provider-filter-smoke.py", "content-provider-site-smoke.py", "content-network-smoke.py", "content-provider-https-smoke.py") WEATHER_CLIENT_NAMES = {f"weather-client-delivery-{suffix}.json" for suffix in ( - "evidence", "acquisition", "original-report", "tree-before", "tree-after", + "cold-evidence", "evidence", "acquisition", "original-report", "tree-before", "tree-after", "cache-before", "cache-after", "input-cleanup")} WEATHER_CLIENT_HELPERS = ("weather-client-delivery.py", "weather-client-bundle.py", "browser-filter-bundle.py", "content-provider-https-smoke.py", "content-network-smoke.py") diff --git a/tests/integration/test-weather-client-delivery-shell.py b/tests/integration/test-weather-client-delivery-shell.py index b731f1a3a..e8be63924 100644 --- a/tests/integration/test-weather-client-delivery-shell.py +++ b/tests/integration/test-weather-client-delivery-shell.py @@ -139,11 +139,51 @@ def test_runtime_arguments_boundaries_and_receipt_order(self): run = source.split("weather_client_delivery_run() {", 1)[1] self.assertLess(run.index('"$WORK/weather-client-delivery-manifest.json"'), run.index('chown -R')) self.assertLess(run.index('publisher-cleanup "$weather_publisher"'), run.index('PHASE=weather-client-delivery-cold')) + self.assertLess(run.index('content_provider_stop_control_observer'), run.index(' cold-evidence "$WORK"')) + self.assertLess(run.index('weather-client-delivery-cache-before.json'), run.index(' cold-evidence "$WORK"')) + self.assertLess(run.index(' cold-evidence "$WORK"'), run.index('weather_client_private prepare')) + self.assertLess(run.index(' copy-cold-evidence "$WORK"'), run.index('weather_client_private prepare')) + self.assertLess(run.index(' cold-evidence "$WORK"'), run.index('PHASE=weather-client-delivery-browser')) self.assertLess(run.index(' retain "$WORK"'), run.index('weather_client_delivery_cleanup')) self.assertLess(run.index('weather_client_delivery_cleanup'), run.index(' evidence "$WORK"')) self.assertIn('weather-client-delivery-tree-before.json', run) self.assertIn('weather-client-delivery-tree-after.json', run) + def test_cold_evidence_is_preserved_before_prepare_and_failure_prevents_browser_preparation(self): + run = SOURCE.read_text().split('weather_client_delivery_run() {', 1)[1] + segment = run[run.index(' PHASE=weather-client-delivery-cold-verify'): + run.index(' PHASE=weather-client-delivery-browser')] + for status in (0, 1): + with self.subTest(cold_validation_status=status): + result = self.call(''' +weather_client_manifest_sha256=inert-manifest +weather_core_sha256=inert-core +weather_client_directory=$WORK/inert-source +weather_viewer=$WORK/inert-viewer +weather_cache=$WORK/inert-cache +binary_directory=$WORK/inert-bin +output_directory=$WORK/inert-output +OUTPUT_UID=985 +OUTPUT_GID=987 +python3() { + case "$3" in cold-evidence|copy-cold-evidence) ;; *) exit 90 ;; esac + printf '%s\n' "$3" >> "$WORK/calls" + printf '{"inert_test_only":true}\n' + if [ "$3" = cold-evidence ]; then return "$TEST_COLD_STATUS"; fi +} +weather_client_private() { + [ "$1" = prepare ] || exit 91 + [ -s "$WORK/weather-client-delivery-cold-evidence.json" ] || exit 92 + printf 'prepare\n' >> "$WORK/calls" +} +fail() { printf '%s\n' "$1" >&2; exit 77; } +''' + segment, TEST_COLD_STATUS=str(status)) + self.assertEqual(result.returncode, 0 if status == 0 else 77, result.stderr) + self.assertEqual((self.root / 'calls').read_text().splitlines(), + ['cold-evidence', 'copy-cold-evidence', 'prepare'] if status == 0 else ['cold-evidence']) + if status: self.assertIn('WEATHER_CLIENT_COLD_EVIDENCE_INVALID', result.stderr) + (self.root / 'calls').unlink() + def test_interrupt_retains_before_cleanup_and_keeps_attempting_on_error(self): (self.root / "bin").mkdir() (self.root / "bin/weather-client-delivery.py").touch() diff --git a/tests/integration/test-weather-client-delivery.py b/tests/integration/test-weather-client-delivery.py index 440c808d7..c5c9dee37 100644 --- a/tests/integration/test-weather-client-delivery.py +++ b/tests/integration/test-weather-client-delivery.py @@ -111,6 +111,8 @@ def joined_fixture(work): socket = work / "runtime-client/control/agent.sock"; socket.parent.mkdir(parents=True) A["write_new"](socket, b"inert stat-only fixture") for suffix, value in files.items(): save(work / f"weather-client-delivery-{suffix}.json", value) + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda *args: cache): + save(work / "weather-client-delivery-cold-evidence.json", A["cold_evidence"](work, manifest_sha, SHA)) return manifest_sha, cache @@ -127,6 +129,8 @@ def test_join_requires_same_publication_dom_hash_and_closed_real_evidence(self): ("cold-consumer", ("boundary", "all_capabilities_dropped"), False), ("cold-consumer", ("receipt", "origin_body_bytes"), 1), ("publisher-cleanup", ("files_removed",), False), + ("cold-evidence", ("forecast_sha256",), "f"*64), + ("cold-evidence", ("scope", "end_to_end_proven"), True), ("tree-after", ("manifest_sha256",), "f"*64), ("privacy-client", ("direct_client_exit_packets",), 1), ("privacy-exit", ("packet_socket_drops",), 1), @@ -153,6 +157,116 @@ def test_join_requires_same_publication_dom_hash_and_closed_real_evidence(self): self.assertFalse(proof["scope"]["two_subflow_bulk_aggregation"]) A["validate_export"](proof) + def test_cold_evidence_survives_missing_or_failed_ui_without_claiming_end_to_end(self): + for browser_outcome in ("missing", "failed"): + with self.subTest(browser=browser_outcome), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, cache = joined_fixture(work) + for suffix in ("original-report", "tree-after", "cache-after", "prepared", "input-cleanup"): + (work / f"weather-client-delivery-{suffix}.json").unlink() + if browser_outcome == "failed": + failed = report(); failed.update(passed=False, phase="inputs", checks={}, cleanup={}) + failed["scope"].update(actual_drizzli_ui=False, actual_http_bridge=False, real_core_adapter=False) + save(work / "weather-client-delivery-original-report.json", failed) + # The browser has not removed its private working inputs yet. + (work / "client-fixtures/weather-viewer").mkdir(parents=True) + original = (work / "weather-client-delivery-cold-evidence.json").read_bytes() + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda *args: cache): + proof = A["cold_evidence"](work, digest, SHA) + self.assertEqual(json.loads(original), proof) + with self.assertRaises((OSError, ValueError)): + A["evidence"](work, digest, SHA) + self.assertTrue(proof["scope"]["protected_cold_transfer"]) + self.assertEqual(proof["forecast_sha256"], publication()["sha256"]) + for key in ("actual_drizzli_ui", "actual_http_bridge", "actual_ui_refresh", "end_to_end_proven"): + self.assertIs(proof["scope"][key], False) + output = work / "output"; output.mkdir() + A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual((output / "weather-client-delivery-cold-evidence.json").read_bytes(), original) + self.assertFalse((output / "weather-client-delivery-evidence.json").exists()) + + def test_cold_evidence_requires_full_acquisition_publication_transport_and_privacy_join(self): + mutations = [ + ("acquisition", ("forecast_sha256",), "f"*64), + ("acquisition", ("forecast_bytes",), 1), + ("acquisition", ("synthetic_weather",), True), + ("acquisition", ("fields",), []), + ("cold-consumer", ("core_sha256",), "f"*64), + ("cold-consumer", ("boundary", "all_capabilities_dropped"), False), + ("cold-consumer", ("receipt", "peer_bytes"), 0), + ("cold-consumer", ("receipt", "origin_body_bytes"), 1), + ("cold-consumer", ("receipt", "local_output"), "/private/unrelated"), + ("publish", ("manifest_id",), "f"*64), + ("publisher-cleanup", ("files_removed",), False), + ("tree-before", ("manifest_sha256",), "f"*64), + ("cache-before", ("user_uid",), 985), + ("privacy-client", ("direct_client_exit_packets",), 1), + ("privacy-exit", ("packet_socket_drops",), 1), + ("control", ("unexpected_provider_application_packets",), 1)] + for suffix, keys, value in mutations: + with self.subTest(file=suffix, field=keys), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, cache = joined_fixture(work) + (work / "weather-client-delivery-original-report.json").unlink() + path = work / f"weather-client-delivery-{suffix}.json" + data = json.loads(path.read_bytes()); target = data + for key in keys[:-1]: target = target[key] + target[keys[-1]] = value + path.unlink(); save(path, data) + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda *args: cache): + with self.assertRaises(ValueError): A["cold_evidence"](work, digest, SHA) + + def test_cold_evidence_rejects_publisher_residue_and_closed_export_violations(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, cache = joined_fixture(work) + publisher = work / "client-fixtures/weather-publisher"; publisher.parent.mkdir() + for symlink in (False, True): + if symlink: publisher.symlink_to(work / "absent") + else: publisher.mkdir() + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda *args: cache): + with self.assertRaises(ValueError): A["cold_evidence"](work, digest, SHA) + if symlink: publisher.unlink() + else: publisher.rmdir() + path = work / "weather-client-delivery-cold-evidence.json"; original = path.read_bytes() + for keys, value in [(('path',), '/private/forecast'), (('forecast_sha256',), 'coordinates'), + (('forecast_bytes',), True), (('scope', 'end_to_end_proven'), True), + (('scope', 'actual_drizzli_ui'), True), (('protected_cold', 'privacy_captures_validated'), False), + (('protected_cold', 'peer_bytes'), 1), (('protected_cold', 'providers_used'), True)]: + bad = json.loads(original); target = bad + for key in keys[:-1]: target = target[key] + target[keys[-1]] = value + with self.subTest(field=keys), self.assertRaises(ValueError): A["validate_cold_export"](bad) + bad = json.loads(original); bad["private"] = "DO NOT EXPORT" + path.unlink(); save(path, bad) + output = work / "output"; output.mkdir() + with self.assertRaises(ValueError): A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + + def test_early_cold_export_survives_later_invalid_ui_evidence_without_replacement(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); joined_fixture(work) + output = work / "output"; output.mkdir() + proof = work / "weather-client-delivery-cold-evidence.json" + wanted = proof.read_bytes() + result = A["copy_artifacts"](work, output, os.getuid(), os.getgid(), only_cold=True) + self.assertEqual(result, dict(files=1, raw_logs_exported=False)) + target = output / proof.name; inode = target.stat().st_ino + # Even a malformed later UI/final receipt cannot erase the already + # exported cold proof or turn the final join into a success. + bad = work / "weather-client-delivery-evidence.json" + save(bad, dict(success=False, failure="inert_late_ui_failure")) + with self.assertRaises(ValueError): A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), [target]) + self.assertEqual(target.read_bytes(), wanted) + self.assertEqual(target.stat().st_ino, inode) + self.assertEqual(A["copy_artifacts"](work, output, os.getuid(), os.getgid(), only_cold=True), result) + modified = json.loads(wanted); modified["core_sha256"] = "f"*64 + proof.unlink(); save(proof, modified) + with self.assertRaises(ValueError): + A["copy_artifacts"](work, output, os.getuid(), os.getgid(), only_cold=True) + self.assertEqual(target.read_bytes(), wanted) + proof.unlink() + with self.assertRaises(ValueError): + A["copy_artifacts"](work, output, os.getuid(), os.getgid(), only_cold=True) + def test_selection_has_no_authority_or_query_fallback(self): A["validate_selection"](selected()) for key, value in (("latitude", 52), ("communityArea", False), ("includeWind", False), diff --git a/tests/integration/test-weather-client-runner.py b/tests/integration/test-weather-client-runner.py index ca43da030..d580ba98a 100644 --- a/tests/integration/test-weather-client-runner.py +++ b/tests/integration/test-weather-client-runner.py @@ -148,18 +148,26 @@ def acquisition(self): index_sha256=SHA, field_sha256=SHA) for step in (3, 9, 24) for parameter in ("2t", "10u", "10v")]) - def collect(self, raw, limit=None): + def collect(self, raw, limit=None, published=None, late_invalid=False): with tempfile.TemporaryDirectory() as temporary: base = Path(temporary).resolve() home, opt, proc, cgroups = (base / name for name in ("home", "opt", "proc", "cgroups")) for directory in (home, opt, proc, cgroups): directory.mkdir() (home / "alpha-output").mkdir() + if published is not None: + early = home / "alpha-output/weather-client-delivery-cold-evidence.json" + early.write_bytes(published) + early.chmod(0o600) work = opt / ("va." + "a" * 32 + ".ABC123") work.mkdir() source = work / "weather-client-delivery-acquisition.json" source.write_bytes(raw) source.chmod(0o600) + if late_invalid: + invalid = work / "weather-client-delivery-evidence.json" + invalid.write_text('{"unknown":"PRIVATE CANARY"}') + invalid.chmod(0o600) for name in ("weather-client-delivery-cold-consumer.json", "weather-client-delivery-input.json", "weather-client-delivery.err", "forecast.json", "config.json"): (work / name).write_text("PRIVATE CANARY") @@ -197,6 +205,32 @@ def test_verified_json_is_not_exported_as_an_unverifiable_tail(self): files = self.collect(json.dumps(self.acquisition()).encode(), limit=64) self.assertFalse(any("weather-client-delivery" in name for name in files)) + def test_previously_published_cold_proof_survives_later_invalid_ui_without_success_claim(self): + cold = dict(schema=1, success=True, kind="volparossa-weather-protected-cold", + bundle_manifest_sha256=SHA, core_sha256=SHA, forecast_sha256=SHA, + forecast_bytes=6000, manifest_id=SHA, cold_receipt_sha256=SHA, + decoder_sha256="3a3b64581e6922db4287445f217f993f6967dea5fcc45ff66944b0bc883408f4", + acquired_fields=9, + protected_cold=dict(peer_bytes=6000, providers_used=2, origin_body_bytes=0, + origin_range_requests=0, privacy_captures_validated=True), + scope=dict(actual_acquisition=True, acquisition_outside_overlay=True, + same_publication_joined=True, protected_cold_transfer=True, actual_drizzli_ui=False, + actual_http_bridge=False, actual_ui_refresh=False, end_to_end_proven=False, + browser_triggered_cold_fetch=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False), + cleanup=dict(publisher_sources_removed_before_fetch=True)) + raw = json.dumps(cold, indent=2).encode() + b"\n" + files = self.collect(json.dumps(self.acquisition()).encode(), published=raw, late_invalid=True) + self.assertEqual(files["published/weather-client-delivery-cold-evidence.json"], raw) + self.assertNotIn("work-1/weather-client-delivery-evidence.json", files) + self.assertNotIn("work-1/weather-client-delivery-acquisition.json", files) + self.assertFalse(json.loads(files["vm-incomplete.json"])["success"]) + self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + for invalid in (dict(cold, prompt="PRIVATE CANARY"), + dict(cold, scope=dict(cold["scope"], end_to_end_proven=True))): + files = self.collect(b"invalid", published=json.dumps(invalid).encode(), late_invalid=True) + self.assertFalse(any("weather-client-delivery" in name for name in files)) + if __name__ == "__main__": unittest.main() diff --git a/tests/integration/weather-client-delivery.py b/tests/integration/weather-client-delivery.py index 5bd16d7b9..e37c485e6 100644 --- a/tests/integration/weather-client-delivery.py +++ b/tests/integration/weather-client-delivery.py @@ -28,7 +28,7 @@ MAX_FORECAST = 131072 MAX_REPORT = 65536 VARIABLES = ["temperature_2m", "wind_speed_10m", "wind_direction_10m", "precipitation", "air_quality"] -EXPORTS = ("evidence", "acquisition", "original-report", "tree-before", "tree-after", "cache-before", "cache-after", "input-cleanup") +EXPORTS = ("cold-evidence", "evidence", "acquisition", "original-report", "tree-before", "tree-after", "cache-before", "cache-after", "input-cleanup") require, hex64 = B["require"], B["hex64"] @@ -426,10 +426,18 @@ def validate_publications(work, publication, nodes): and served["replication_enabled"] is False) -def evidence(work, manifest_sha, binary_sha): +def validate_cold_phase(work, manifest_sha, binary_sha): + """Full cold-phase join, independent of whether a browser ever starts. + + The final join calls the same validator again; retaining this earlier proof + must not turn a later browser failure into successful end-to-end evidence. + """ + require(hex64(binary_sha)) selected = validate_selection(read(work / "weather-client-delivery-input.json")); publication = selected["publication"] acquisition = read(work / "weather-client-delivery-acquisition.json") validate_acquisition(acquisition) + require(acquisition["forecast_sha256"] == publication["sha256"] + and acquisition["forecast_bytes"] == publication["bytes"]) cold = read(work / "weather-client-delivery-cold-consumer.json") same(cold["publication"], publication) validate_cold_receipt(cold["receipt"], publication, work / "state-client/weather-cache", work / "client-fixtures/weather-viewer/forecast.json") @@ -446,16 +454,77 @@ def evidence(work, manifest_sha, binary_sha): {role: read(work / f"weather-client-delivery-privacy-{role}.json") for role in HTTPS["ROLES"]}, read(work / "weather-client-delivery-control.json")) same(read(work / "weather-client-delivery-publisher-cleanup.json"), dict(directory_removed=True, files_removed=True)) + publisher = work / "client-fixtures/weather-publisher" + require(not publisher.exists() and not publisher.is_symlink()) + before = read(work / "weather-client-delivery-cache-before.json") + same(before, cache_state(work / "state-client/weather-cache", before["user_uid"], before["group_gid"])) + require(before["user_uid"] != 985 and cold["boundary"]["primary_control_gid"] != before["group_gid"]) + # Retained by root before UID985 receives the input tree. This binds the + # acquisition source/runtime pins without requiring a later UI report. + manifest = B["manifest"](raw_file(work / "weather-client-delivery-manifest.json", B["MAX_MANIFEST"], 0o400), manifest_sha) + same(read(work / "weather-client-delivery-tree-before.json"), dict(version=1, action="verify-tree", + manifest_sha256=manifest_sha, files=len(manifest["files"]), runtime_executed=False)) + return publication, acquisition, cold, manifest, before + + +def cold_summary(publication, acquisition, cold, manifest_sha, binary_sha): + result = dict(schema=1, success=True, kind="volparossa-weather-protected-cold", + bundle_manifest_sha256=manifest_sha, core_sha256=binary_sha, + forecast_sha256=publication["sha256"], forecast_bytes=publication["bytes"], + manifest_id=publication["manifest_id"], cold_receipt_sha256=cold["cold_receipt_sha256"], + decoder_sha256=acquisition["decoder_sha256"], acquired_fields=len(acquisition["fields"]), + protected_cold=dict(peer_bytes=cold["receipt"]["peer_bytes"], providers_used=cold["receipt"]["providers_used"], + origin_body_bytes=0, origin_range_requests=0, privacy_captures_validated=True), + scope=dict(actual_acquisition=True, acquisition_outside_overlay=True, same_publication_joined=True, + protected_cold_transfer=True, actual_drizzli_ui=False, actual_http_bridge=False, actual_ui_refresh=False, + end_to_end_proven=False, browser_triggered_cold_fetch=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False), + cleanup=dict(publisher_sources_removed_before_fetch=True)) + validate_cold_export(result) + return result + + +def validate_cold_export(value): + """Exact closed schema: no paths, names, coordinates, peers or raw receipts.""" + require(type(value) is dict and set(value) == {"schema", "success", "kind", "bundle_manifest_sha256", + "core_sha256", "forecast_sha256", "forecast_bytes", "manifest_id", "cold_receipt_sha256", + "decoder_sha256", "acquired_fields", "protected_cold", "scope", "cleanup"}) + same(value["schema"], 1); same(value["success"], True) + same(value["kind"], "volparossa-weather-protected-cold") + require(all(hex64(value[key]) for key in ("bundle_manifest_sha256", "core_sha256", "forecast_sha256", + "manifest_id", "cold_receipt_sha256", "decoder_sha256")) and value["decoder_sha256"] == B["DECODER_SHA"] + and type(value["forecast_bytes"]) is int and 0 < value["forecast_bytes"] <= MAX_FORECAST) + same(value["acquired_fields"], 9) + transfer = value["protected_cold"] + require(type(transfer) is dict and type(transfer.get("providers_used")) is int and transfer["providers_used"] in (1, 2)) + same(transfer, dict(peer_bytes=value["forecast_bytes"], providers_used=transfer["providers_used"], + origin_body_bytes=0, origin_range_requests=0, privacy_captures_validated=True)) + same(value["scope"], dict(actual_acquisition=True, acquisition_outside_overlay=True, same_publication_joined=True, + protected_cold_transfer=True, actual_drizzli_ui=False, actual_http_bridge=False, actual_ui_refresh=False, + end_to_end_proven=False, browser_triggered_cold_fetch=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False)) + same(value["cleanup"], dict(publisher_sources_removed_before_fetch=True)) + + +def cold_evidence(work, manifest_sha, binary_sha): + publication, acquisition, cold, _manifest, _before = validate_cold_phase(work, manifest_sha, binary_sha) + return cold_summary(publication, acquisition, cold, manifest_sha, binary_sha) + + +def evidence(work, manifest_sha, binary_sha): + publication, acquisition, cold, manifest, before = validate_cold_phase(work, manifest_sha, binary_sha) + retained_cold = read(work / "weather-client-delivery-cold-evidence.json") + validate_cold_export(retained_cold) + same(retained_cold, cold_summary(publication, acquisition, cold, manifest_sha, binary_sha)) same(read(work / "weather-client-delivery-input-cleanup.json"), dict(directory_removed=True, files_removed=True)) require(all(not path.exists() and not path.is_symlink() for path in (work / "client-fixtures/weather-publisher", work / "client-fixtures/weather-viewer"))) - before, after = (read(work / f"weather-client-delivery-cache-{phase}.json") for phase in ("before", "after")) + after = read(work / "weather-client-delivery-cache-after.json") same(before, after); same(after, cache_state(work / "state-client/weather-cache", after["user_uid"], after["group_gid"])) require(after["user_uid"] != 985 and cold["boundary"]["primary_control_gid"] != after["group_gid"]) # A root-owned manifest was retained before the verified tree was handed to # UID985. Worker verify-tree receipts check that tree before and after use; # root never weakens owner checks or grants access to private agent caches. - manifest = B["manifest"](raw_file(work / "weather-client-delivery-manifest.json", B["MAX_MANIFEST"], 0o400), manifest_sha) checked = dict(manifest_sha256=manifest_sha, files=len(manifest["files"]), runtime_executed=False) for phase in ("before", "after"): same(read(work / f"weather-client-delivery-tree-{phase}.json"), dict(version=1, action="verify-tree", **checked)) @@ -495,14 +564,18 @@ def evidence(work, manifest_sha, binary_sha): viewer_removed=True, agent_cache_identity_preserved=True, agent_cache_private=True)) -def copy_artifacts(work, output, uid, gid): +def copy_artifacts(work, output, uid, gid, only_cold=False): # Closed identifiers only; original page/forecast/receipt/paths never cross this boundary. + # The earlier cold-only export is independent of later UI receipt failures. + # Full export still prevalidates every candidate and never replaces a file. + if only_cold: require((work / "weather-client-delivery-cold-evidence.json").exists()) pending = []; count = 0 - for suffix in EXPORTS: + for suffix in (("cold-evidence",) if only_cold else EXPORTS): source = work / f"weather-client-delivery-{suffix}.json" if not source.exists(): continue raw = raw_file(source, MAX_REPORT); value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) if suffix == "original-report": closed_report(value) + elif suffix == "cold-evidence": validate_cold_export(value) else: validate_export(value) target = output / source.name if target.exists() or target.is_symlink(): @@ -557,7 +630,9 @@ def main(): elif action in ("publisher-cleanup", "user-cleanup") and len(args) == 1: result = cleanup(Path(args[0]), action == "publisher-cleanup") elif action == "cache-state" and len(args) == 3: result = cache_state(Path(args[0]), int(args[1]), int(args[2])) elif action == "retain" and len(args) == 1: result = retain(Path(args[0])) + elif action == "cold-evidence" and len(args) == 3: result = cold_evidence(Path(args[0]), args[1], args[2]) elif action == "evidence" and len(args) == 3: result = evidence(Path(args[0]), args[1], args[2]) + elif action == "copy-cold-evidence" and len(args) == 4: result = copy_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3]), only_cold=True) elif action == "copy-artifacts" and len(args) == 4: result = copy_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3])) else: raise ValueError("weather_client_delivery_rejected") print(json.dumps(result, sort_keys=True, separators=(",", ":"))) diff --git a/tests/integration/weather-client-delivery.sh b/tests/integration/weather-client-delivery.sh index cc1d397f6..4586505da 100644 --- a/tests/integration/weather-client-delivery.sh +++ b/tests/integration/weather-client-delivery.sh @@ -222,6 +222,16 @@ weather_client_delivery_run() { content_provider_stop_control_observer || fail WEATHER_CLIENT_CONTROL_INCOMPLETE python3 -B "$WORK/bin/weather-client-delivery.py" cache-state "$weather_cache" "$AGENT_UID" "$AGENT_GID" \ >"$WORK/weather-client-delivery-cache-before.json" || fail WEATHER_CLIENT_CACHE_INVALID + # Preserve the fully validated cold transport separately before any browser + # preparation or execution. Later UI failure must not erase this evidence, + # and this record explicitly makes no UI or end-to-end success claim. + PHASE=weather-client-delivery-cold-verify + python3 -B "$WORK/bin/weather-client-delivery.py" cold-evidence "$WORK" "$weather_client_manifest_sha256" \ + "$weather_core_sha256" >"$WORK/weather-client-delivery-cold-evidence.json" \ + || fail WEATHER_CLIENT_COLD_EVIDENCE_INVALID + python3 -B "$WORK/bin/weather-client-delivery.py" copy-cold-evidence "$WORK" \ + "$output_directory" "$OUTPUT_UID" "$OUTPUT_GID" >/dev/null \ + || fail WEATHER_CLIENT_COLD_RETENTION_FAILED weather_client_private prepare "$weather_client_directory" "$weather_viewer" \ "$binary_directory/volparossa" "$WORK/runtime-client/control/agent.sock" "$weather_cache" \ >"$WORK/weather-client-delivery-prepared.json" || fail WEATHER_CLIENT_EXPECTED_FAILED From 44c50563e3f5db99473a7f6d685833a7b238a1e9 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:00:12 +0200 Subject: [PATCH 16/49] fix: preserve complementary chunks after slow provider probes --- .../src/transfer/parallel.rs | 19 ++- .../src/transfer/parallel/adaptive.rs | 5 +- .../src/transfer/parallel/tests/adaptive.rs | 146 +++++++++++++++++- docs/IMPLEMENTATION_STATUS.md | 55 ++++++- 4 files changed, 210 insertions(+), 15 deletions(-) diff --git a/crates/volparossa-content/src/transfer/parallel.rs b/crates/volparossa-content/src/transfer/parallel.rs index 1d0b9c205..95a04de13 100644 --- a/crates/volparossa-content/src/transfer/parallel.rs +++ b/crates/volparossa-content/src/transfer/parallel.rs @@ -43,6 +43,7 @@ struct Peer { pending: Option, state: adaptive::PeerState, probing: bool, + coverage_only: bool, } /// A bounded writer/coordinator; workers never own or reopen the destination cache. @@ -319,13 +320,25 @@ impl ParallelDownload { } fn assign(&mut self, index: usize) -> Result<(), TransferError> { - let peer = &mut self.peers[index]; + let peer = &self.peers[index]; if peer.state != adaptive::PeerState::Active || peer.pending.is_some() { return Ok(()); } let eligible = |item: &PendingChunk| { - !item.complete && item.owner.is_none() && !item.attempted.contains(&index) + !item.complete + && item.owner.is_none() + && !item.attempted.contains(&index) + && (!peer.coverage_only + || !self.peers.iter().enumerate().any(|(other, candidate)| { + other != index + && candidate.state == adaptive::PeerState::Active + && !candidate.coverage_only + && !item.attempted.contains(&other) + })) }; + // A failed speed probe keeps its existing, counted lease only to cover gaps. + // Let ordinary active peers try first; a slow probe is not evidence that it + // lacks later shards. Dormant or closing peers cannot promise live coverage. // A failed/missing request is useful evidence for exploration; manifest membership // alone never claims that a provider actually stores a chunk. Prefer that unresolved // coverage for the probe without reassigning anybody's still-in-flight request. @@ -339,6 +352,7 @@ impl ParallelDownload { let Some(position) = missing.or_else(|| self.chunks.iter().position(eligible)) else { return Ok(()); }; + let peer = &mut self.peers[index]; let pending = &mut self.chunks[position]; match peer.requests.try_send(Work::Chunk(pending.chunk.clone())) { Ok(()) => { @@ -442,6 +456,7 @@ fn peer( adaptive::PeerState::Dormant }, probing: false, + coverage_only: false, }, ChunkWorker { manifest: manifest.clone(), diff --git a/crates/volparossa-content/src/transfer/parallel/adaptive.rs b/crates/volparossa-content/src/transfer/parallel/adaptive.rs index 179b80d56..b2e32bb5a 100644 --- a/crates/volparossa-content/src/transfer/parallel/adaptive.rs +++ b/crates/volparossa-content/src/transfer/parallel/adaptive.rs @@ -108,7 +108,10 @@ impl ParallelDownload { fn finish_probe(&mut self, index: usize, beneficial: bool) { self.peers[index].probing = false; if !beneficial { - self.finish_peer(index); + // Throughput says nothing about possession of untested complementary shards. + // Stop speculative credits, but retain bounded gap coverage until exhausted + // or resource pressure requires closure. This lease still counts at full width. + self.peers[index].coverage_only = true; self.growth_blocked = true; } self.probe_baseline = None; diff --git a/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs b/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs index 11e671517..91cbe7873 100644 --- a/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs +++ b/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs @@ -23,6 +23,9 @@ enum Case { NoResources, Many, Pressure, + RetiredCoverage, + RetiredDuplicate, + PressureCoverage, } struct Fixture { @@ -90,7 +93,7 @@ fn fixture(case: Case) -> Fixture { for (position, chunk) in manifest.chunks().iter().take(9).enumerate() { let owns = match case { Case::Failed => index > 0, - Case::Pressure => true, + Case::Pressure | Case::RetiredDuplicate => true, Case::Many => index == 9, _ => position % 3 == index, }; @@ -250,6 +253,86 @@ async fn adaptive_original_indexes_use_three_providers_and_preserve_progress_und } } +#[tokio::test] +async fn slow_probe_retirement_must_not_discard_later_complementary_chunks() { + tokio::time::timeout(Duration::from_secs(10), transfer(Case::RetiredCoverage)) + .await + .expect("bounded complementary coverage after an unsuccessful rate probe"); +} + +#[tokio::test] +async fn unsuccessful_probe_gets_no_speculative_credits_when_other_peers_cover_the_object() { + tokio::time::timeout(Duration::from_secs(10), transfer(Case::RetiredDuplicate)) + .await + .expect("bounded transfer without unnecessary probe payload"); +} + +#[tokio::test] +async fn hard_pressure_preserves_partial_progress_but_does_not_claim_full_object_success() { + tokio::time::timeout(Duration::from_secs(10), transfer(Case::PressureCoverage)) + .await + .expect("hard resource pressure takes precedence over complementary coverage"); +} + +fn reject_speed_probe(download: &mut ParallelDownload) { + // Fix only the performance history, not any content/transport response: + // a third admitted probe did not improve the preceding aggregate rate. + // Its untested complementary shards still need the actual v1 streams. + // No wall-clock race decides whether this branch is exercised. + download.peers[2].state = super::super::adaptive::PeerState::Active; + download.peers[2].probing = true; + download.probe_baseline = Some(u64::MAX); + for index in 0..3 { + download.record_sample(index, 1, false); + } + assert!(download.growth_blocked); + assert!(download.peers[2].coverage_only); + assert!(!download.peers[2].probing); +} + +#[test] +fn coverage_only_peer_waits_for_alternative_misses_and_never_duplicates_an_assignment() { + let Fixture { + _directory, + manifest, + mut output, + .. + } = fixture(Case::Complementary); + let (mut download, mut workers) = + ParallelDownload::new_adaptive(&manifest, 3, &mut output, TransferLimits::default(), 3) + .unwrap(); + reject_speed_probe(&mut download); + download.assign_ready().unwrap(); + assert_eq!(download.peers[0].pending, Some(0)); + assert_eq!(download.peers[1].pending, Some(1)); + assert_eq!(download.peers[2].pending, None); + assert!(workers[2].requests.try_recv().is_err()); + let mut progress = vec![TransferProgress::default(); 3]; + // A completed miss releases a chunk, but the other ordinary peer still gets + // its own attempt first. Remove only the consumed queue credit each time. + for index in [0, 1, 1] { + workers[index].requests.try_recv().unwrap(); + download + .accept(index, Some(Ok(None)), &mut output, &mut progress) + .unwrap(); + download.assign_ready().unwrap(); + } + assert_eq!(download.peers[2].pending, Some(0)); + assert_eq!(download.chunks[0].owner, Some(2)); + assert_eq!( + download.chunks[0] + .attempted + .iter() + .copied() + .collect::>(), + [0, 1, 2] + ); + workers[2].requests.try_recv().unwrap(); + download.assign_ready().unwrap(); + assert!(workers[2].requests.try_recv().is_err()); + assert_eq!(download.peers[2].pending, Some(0)); +} + async fn transfer(case: Case) { let Fixture { _directory, @@ -269,7 +352,7 @@ async fn transfer(case: Case) { ..TransferLimits::default() }; let refs: Vec<_> = transports.iter().collect(); - let (download, workers) = ParallelDownload::new_adaptive_with_transport_manifests( + let (mut download, workers) = ParallelDownload::new_adaptive_with_transport_manifests( &manifest, &refs, &mut output, @@ -285,8 +368,24 @@ async fn transfer(case: Case) { .count() <= 2 ); + if matches!(case, Case::RetiredCoverage | Case::RetiredDuplicate) { + reject_speed_probe(&mut download); + } + if matches!(case, Case::PressureCoverage) { + reject_speed_probe(&mut download); + // Pressure also closes a coverage-only peer; it cannot bypass a hard + // allowance just because it might own the only remaining fragments. + download.adjust_width(1); + } let counters = Counters::default(); - counters.allowance.store(allowance, Ordering::SeqCst); + counters.allowance.store( + if matches!(case, Case::PressureCoverage) { + 1 + } else { + allowance + }, + Ordering::SeqCst, + ); let mut drivers = Vec::new(); for (index, (worker, provider)) in workers.into_iter().zip(providers).enumerate() { drivers.push(tokio::spawn(drive( @@ -309,8 +408,45 @@ async fn transfer(case: Case) { for driver in drivers { sent += driver.await.unwrap().bytes; } + check_result( + case, + &counters, + &progress, + sent, + &manifest, + &mut output, + &bytes, + ); +} + +fn check_result( + case: Case, + counters: &Counters, + progress: &[TransferProgress], + sent: u64, + manifest: &VerifiedManifest, + output: &mut ChunkStore, + bytes: &[u8], +) { assert_eq!(counters.active.load(Ordering::SeqCst), 0); assert_eq!(sent, progress.iter().map(|p| p.bytes).sum::()); + if matches!(case, Case::RetiredDuplicate) { + assert_eq!(progress[2], TransferProgress::default()); + assert_eq!(counters.opened.load(Ordering::SeqCst), 2); + } + if matches!(case, Case::PressureCoverage) { + assert_eq!(counters.opened.load(Ordering::SeqCst), 1); + assert_eq!(counters.maximum.load(Ordering::SeqCst), 1); + assert_eq!(progress.iter().map(|p| p.chunks).sum::(), 3); + assert_eq!(sent, 3 * CHUNK_BYTES as u64); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let result = reassemble(manifest, &mut [output], now, &mut Vec::new()); + assert!(matches!(result, Err(crate::Error::MissingChunk(_)))); + return; + } if matches!(case, Case::Pressure) { assert_eq!(counters.maximum.load(Ordering::SeqCst), 3); assert_eq!(counters.allowance.load(Ordering::SeqCst), 1); @@ -326,7 +462,7 @@ async fn transfer(case: Case) { } else { check_progress( case, - &progress, + progress, sent, counters.maximum.load(Ordering::SeqCst), ); @@ -335,7 +471,7 @@ async fn transfer(case: Case) { .unwrap() .as_secs(); let mut reconstructed = Vec::new(); - reassemble(&manifest, &mut [&mut output], now, &mut reconstructed).unwrap(); + reassemble(manifest, &mut [output], now, &mut reconstructed).unwrap(); assert_eq!(reconstructed, bytes); } } diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 0a8f16a98..036bb5ed7 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -272,9 +272,8 @@ correction. The persistent bridge requires an explicitly initialized, owned cach with reuse enabled, rather than trying to create it again on every refresh. The final repository CI and CodeQL checks pass. -A real core-cache publication reaching the browser remains unproven; the -synthetic trial does not establish live overlay or weather-origin delivery. -The local proof fixture acquires fresh public ECMWF data, +The corrected joined KVM trial now proves one real temperature/wind publication +reaching the browser and surviving refresh. The local proof fixture acquires public ECMWF data, publishes it to peers, requires a cold protected core transfer, and attempts to display and refresh the same publication through the production bridge in Drizz.li. Origin HTTPS acquisition is distinct from the subsequent overlay transfer. Explicit @@ -300,7 +299,8 @@ a root-owned executable becomes unmapped inside the unprivileged user namespace despite unchanged bytes and a read-only mount. A narrow verifier correction passes twelve focused tests, including the real minimal namespace case; it keeps strict outer ownership and checks the exact authorized executable through -a read-only descriptor inside the namespace. No corrected joined run has passed yet. +a read-only descriptor inside the namespace. The corrected Weather phase passes +in the second trial described below; the complete topology run still fails later. The failed trial cleaned up all owned objects. Guest-root and host network state checks were unchanged. Retained local evidence is under @@ -313,7 +313,40 @@ transfer before starting the browser. Its closed receipt binds acquisition and publication bytes, core identity, provider traffic and privacy checks, and explicitly excludes UI or end-to-end success. Fourteen helper tests and twelve shell tests pass, including preservation across a later invalid UI receipt; -this is fixture behavior, not a newly passed joined network trial. +these targeted tests establish fixture behavior independently of the live trial. + +The second joined trial on core `93dc2a64bd7157a77ee1837e4b8375f9b222fac6` +uses bundle manifest `a3dab90657e742ce181994cb0be6741eccaaaaf63464026a09da67b49db7cb6e`. +Its Weather phase passes: nine real ECMWF fields produce a 7,149-byte forecast, +then a protected cold transfer obtains those same bytes from a peer after the +publisher sources are removed, with zero origin requests or bytes in that phase. +The actual Drizz.li page and production bridge display and refresh the exact +publication, bound by matching hashes; browser, bridge, viewer, private state +and agent-cache identity checks pass. This does not prove a browser-triggered +cold fetch, simultaneous bulk throughput on two subflows, precipitation delivery, +AQI, or a complete weather service. Origin acquisition occurs outside the overlay. + +Evidence is retained under `.git/ci-evidence/weather-client-delivery-02.kusV5m/output`: +`weather-client-delivery-evidence.json` has SHA-256 +`f8a0a792e8e392cc8981c293b548bcec34852255cbad6b5aa7f6e3fc39d659d4`, +and its browser report hash is +`a869419e5302cb079240f68acbe327f46157c63c102d427b1701b48b38e5836b`. +The **overall runner exits 1** afterwards at the separate 15 MiB adaptive-provider +fetch with `CONTENT_UNAVAILABLE`. It must not be described as an all-green topology +run. A15 reports unchanged state outside the owned disposable namespaces, and the +runner and QEMU exit. + +A deterministic regression reproduces one cause of incomplete adaptive transfer: +a peer rejected by the speed probe still owns untested complementary chunks, +but closing it leaves only six of nine chunks. The local correction retains that +already-counted peer for gap coverage, after ordinary active peers have attempted +the missing chunk. It does not grant extra credits, duplicate an in-flight request +or extend the original deadline. Nine parallel-transfer tests pass, including +all nine complementary chunks, no speculative payload from a redundant peer, +single-owner assignment and hard-pressure partial progress. Hard pressure can +still stop a transfer; the final reassembly gate explicitly rejects the incomplete +object rather than calling it complete. The correction has an independent review +but still requires a new live topology run; the earlier failing receipt is retained. In a separate Weather candidate, native precipitation acquisition now passes for IFS and AIFS Single with the pinned decoder. The shared provider preserves source @@ -326,8 +359,16 @@ receipt `integrations/drizzli/precipitation-browser-smoke.json` has SHA-256 `cb972cbf4b3930c87df017865c68b1da88f1fa81ed48f23008dd623ba5051a58`; it explicitly makes no origin, core or overlay-delivery claim. A subsequent baseline temperature/wind run on the same new build also passes. -Overmorrow, additional location selection, air quality and complete application -delivery remain required work. Absent fields are never substituted with zero. +The Weather air-quality candidate now has a separate NOAA GEFS-Aerosols +PM2.5/PM10 contract, exact-range source adapter and shared core-cache consumer. +A real CLI/decoder trial verifies two native instantaneous forecast points +(steps 3 and 6) using 2,841,174 field bytes; it does not derive AQI or observations. +Ninety-one Weather tests pass, including subprocess-based cache/CLI boundaries, +query/expiry checks and cancellation. The particulate core-cache tests use a +synthetic core process; real overlay delivery and application display remain +unproved. Overmorrow, additional location selection, precipitation delivery and +complete application delivery remain required work. Absent fields are never +substituted with zero. ### Citizen procedure simulation From 6a8275c9821135ab42791c19cac420b603affc3f Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:06:02 +0200 Subject: [PATCH 17/49] test: join precipitation and particulate delivery to native weather UI --- docs/IMPLEMENTATION_STATUS.md | 113 +++- docs/README.md | 1 + tests/integration/content-provider-smoke.sh | 5 +- tests/integration/kvm-alpha-topology.sh | 6 +- tests/integration/run-alpha-topology-vm.sh | 29 +- .../integration/test-weather-client-bundle.py | 273 ++++++++- .../integration/test-weather-client-runner.py | 124 +++- .../test-weather-products-delivery-shell.py | 149 +++++ .../test-weather-products-delivery.py | 315 ++++++++++ tests/integration/weather-client-bundle.py | 137 ++++- tests/integration/weather-client-delivery.sh | 16 + .../integration/weather-products-delivery.py | 569 ++++++++++++++++++ .../integration/weather-products-delivery.sh | 282 +++++++++ 13 files changed, 1965 insertions(+), 54 deletions(-) create mode 100644 tests/integration/test-weather-products-delivery-shell.py create mode 100644 tests/integration/test-weather-products-delivery.py create mode 100644 tests/integration/weather-products-delivery.py create mode 100644 tests/integration/weather-products-delivery.sh diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 036bb5ed7..6dea14568 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -7,14 +7,55 @@ This is the repository's source of truth for implementation progress. A checked detailed evidence ledger, including historical failures—not an installation guide. [Current work](#current-integration-and-active-work) · +[Execution plan](#execution-priorities-and-next-milestones) · [Earlier milestones](#earlier-milestone-evidence) · [Original v1 scorecard](#fixed-alpha-v1-scorecard) · [Completion requirements](#definition-of-done) -Last updated: 2026-10-08 +Last updated: 2026-10-11 ## Current integration and active work +### Execution priorities and next milestones + +Continue the existing core-cache and Weather integration; the prioritization +revision does not restart development or interrupt useful work. Choose the next +slice across repositories by shared blockers, architectural uncertainty and a +demonstrable end-to-end result, not by the order of applications in the goal or +the number of changes. Develop shared core functions with their actual consumers +rather than waiting for the entire core to be complete. + +The immediate milestones build on the retained candidates and evidence below: + +| Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | +| --- | --- | --- | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify the adaptive-provider correction in the real disposable topology, preserving the joined temperature/wind-to-Drizz.li proof, traffic checks and cleanup. | Core candidate `44c50563` passes targeted tests, but the corrected live run remains outstanding. Trial 03 stopped before VM startup at the unchanged 4 GiB guest plus 2 GiB host-reserve gate; it produced no new network evidence. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The combined UI/bridge passes with synthetic data. The two-product joined driver and exact input bundle now pass targeted checks; the next step is the real disposable trial when the unchanged memory gate permits. Real peer delivery remains unproved; existing temperature/wind evidence does not cover it. | +| Extend the same provider to Overmorrow — `volparossa-weather` | Build and exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | Reuse the provider contracts and core retrieval path; client adaptation, a reproducible client build and its own runtime proof remain required. Do not create a second network scheduler or source-acquisition implementation. | + +Parallel work may continue on the Drizz.li field adapter and Overmorrow client +boundaries while the core trial waits for sufficient memory. Keep the trial's +inputs frozen, give shared interface changes one owner and integrate dependent +consumer changes together. Do not launch competing heavy builds or lower the +host reserve to make a trial start. A resource wait delays that proof, not all +implementation work. + +After each milestone, or when a substantive blocker changes, reassess the next +shared dependency against the existing evidence and application worktrees. +Protected compute for Code/Browser, private storage for Chat/Image/Cloud, +message custody for Chat/Mail, public content for Map/Weather, and transaction +or civic simulation dependencies remain in scope; this is not a new serial +queue or a claim that those integrations are complete. Architecture-defining +checks can proceed alongside useful integration when they do not conflict with +shared interfaces or available resources. + +Keep this plan concise: record the selected milestone, affected repositories, +dependencies and remaining blocker. When changing course, record why, where +unfinished work is retained and what permits its continuation. No integration +is interrupted solely because of this planning revision. Existing functionality, +privacy boundaries and completion criteria are unchanged; local candidates and +component checks do not become merged or end-to-end verified by appearing here. + ### Browser consent cooperation and public filter snapshots Browser [PR8](https://github.com/VOLPAROSSA/volparossa-browser/pull/8) and @@ -365,11 +406,77 @@ A real CLI/decoder trial verifies two native instantaneous forecast points (steps 3 and 6) using 2,841,174 field bytes; it does not derive AQI or observations. Ninety-one Weather tests pass, including subprocess-based cache/CLI boundaries, query/expiry checks and cancellation. The particulate core-cache tests use a -synthetic core process; real overlay delivery and application display remain -unproved. Overmorrow, additional location selection, precipitation delivery and +synthetic core process; real particulate overlay delivery remains unproved. +Overmorrow, additional location selection, precipitation delivery and complete application delivery remain required work. Absent fields are never substituted with zero. +The next local Weather slice connects that separate particulate product to the +actual Drizz.li week route through `/api/volparossa/air-quality`. Owner-selected +weather and particulate configurations must use the same public area, while +their publishers, source runs, native times and expiry remain independent. +Both endpoints use one bounded retrieval slot, no origin fallback and no browser +selection of locations or publishers. Timeout failure follows backend cancellation +cleanup; the next product request does not race a still-occupied slot. + +The complete client type check and networkless production build pass. A total +of 126 targeted Node tests (including eight actual Svelte render/compiler tests) +and thirteen inert harness checks pass. The actual isolated Firefox trial with +synthetic precipitation and PM2.5/PM10 also passes: eleven requests per product, +exact missing/zero concentrations, independent native timestamps, refresh, +separate expiry/failure handling, route guards and complete private-state/process +cleanup. The same build passes the existing temperature/wind scenario with AQ +unconfigured. Weather's retained `integrations/drizzli/air-quality-browser-smoke.json` +has SHA-256 `7977c31ad53d5ed12455ec7147daf220a05987aca71273344a3a536fe0a76d3d`. +This is real UI and HTTP-bridge evidence with synthetic values, not a new NOAA +origin trial or precipitation/particulate peer-delivery result. The frozen +temperature/wind inputs for the outstanding core topology rerun are unchanged. + +The local v2 input-bundle preparation now preserves the complete precipitation +and particulate module set, both application licences, source overlays, staged +copies and the pinned decoder/runtime. It checks exact provenance groups against +the combined browser receipt and the separate ECMWF and NOAA acquisition +receipts. Capturing a file does not establish that it has run in the joined test. +The actual 193-file archive (433,030,799 payload bytes) passes capture, readonly +unpack and a complete tree recheck. Its manifest SHA-256 is +`6165442537ddacf0e287118447670a285ac2d1cb1bf33fa37afe42bfaa037b28`; +archive SHA-256 is +`97facd0bd87160094774143d016cc4e6659fb69fe984280e3a1812a2c7de3989`. +The earlier v1 archive still passes unchanged. Twenty bundle tests and 37 +existing delivery/shell/runner checks pass, including the new profile gate. +Four Weather preparation tests also pass after their temporary test copies were +updated to include the imported particulate modules; no assertion was relaxed. + +The two-product joined driver now acquires both products before publishing, +creates distinct publications and agent caches, and removes both original +forecasts and the publisher identity before either cold fetch. Each product +requires its own drained packet/control captures, selected protected route, +byte-matched receipt and private-cache checks. The browser then checks both +provider responses, initial panels and refresh against those cold receipts. +Native NOAA timestamps remain independent of ECMWF; these selected model points +are not presented as current observations or guaranteed future coverage. + +A shared 840-second post-acquisition budget bounds publication, transfers and +browser work, without extending either signed 900-second publication or source +expiry. Helper timeouts allow bounded descendant cleanup before a hard kill. +Closed cold-transfer evidence is retained before browser preparation, and +interruption cleanup attempts every owned resource independently. Eighty-two +core bundle/driver/shell/runner tests and the seven Weather preparation plus +sixteen browser-driver tests pass; cross-repository checks confirm the matching +report contract. The wider Weather check passes 133 Node tests. + +Both topology runners now accept exactly the validated v1 and v2 input profiles, +selecting the matching driver. The updated 193-file archive has 433,046,126 +payload bytes, manifest SHA-256 +`ec789a95fb07e2eba17f87253f7387acd4a5c488a23703c1d90f11808a48d102` +and archive SHA-256 +`f1509e305236e37afecfc7287e517c99a6714487f036af4b25db5770a0bc5493`. +Capture, full runtime-input verification, read-only unpack and tree verification +pass; the frozen v1 archive remains unchanged and accepted. This is fixture and +input evidence, not a completed live two-product trial. The unchanged 4 GiB guest +plus 2 GiB host reserve is still required; no VM or new origin acquisition was +started for this preparation. + ### Citizen procedure simulation Citizen [PR1](https://github.com/VOLPAROSSA/volparossa-citizen/pull/1) is merged diff --git a/docs/README.md b/docs/README.md index 0dc78a35b..30e344efb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -88,6 +88,7 @@ or go directly to a topic when you already know what you need. - [Operations](development/OPERATIONS.md): configuration, commands, packaging and removal. - [Testing](development/TESTING.md): focused checks, disposable topologies and acceptance requirements. - [Implementation status](IMPLEMENTATION_STATUS.md): detailed evidence ledger, including failed trials. +- [Execution plan](IMPLEMENTATION_STATUS.md#execution-priorities-and-next-milestones): next milestones, shared dependencies and current blockers. - [Applications](applications/APPLICATIONS.md): boundaries and links for the application repositories. - [Contributing](../CONTRIBUTING.md): how to propose and verify a useful change. - [Security policy](../SECURITY.md): private vulnerability reporting. diff --git a/tests/integration/content-provider-smoke.sh b/tests/integration/content-provider-smoke.sh index 64fa89834..72b6ce151 100644 --- a/tests/integration/content-provider-smoke.sh +++ b/tests/integration/content-provider-smoke.sh @@ -82,7 +82,10 @@ content_provider_control_underlay() { content_provider_start_control_observer() { provider_control_prefix=$1 - case $provider_control_prefix in content-provider-*|weather-client-delivery-control) ;; *) return 1 ;; esac + case $provider_control_prefix in + content-provider-*|weather-client-delivery-control|weather-products-delivery-weather-control|weather-products-delivery-air-quality-control) ;; + *) return 1 ;; + esac [ -z "$PROVIDER_CONTROL_PID" ] || return 1 ip netns exec "$provider_control_ns" python3 "$WORK/bin/privacy-observer.py" \ content-control "$WORK/$provider_control_prefix.json" \ diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index ac9982f82..650a45ece 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1002,10 +1002,10 @@ if [ -n "$weather_client_bundle" ]; then && [ "$(stat -Lc '%a' "$weather_client_bundle")" = 600 ] || exit 69 weather_client_input_uid=$(stat -Lc '%u' "$weather_client_bundle") if [ "$weather_client_input_uid" -eq 0 ]; then - python3 -B "$source_directory/tests/integration/weather-client-bundle.py" verify \ + python3 -B "$source_directory/tests/integration/weather-client-bundle.py" verify-runtime \ "$weather_client_bundle" "$weather_client_manifest_sha256" || exit 69 elif [ "$weather_client_input_uid" -eq "$(id -u vpci)" ]; then - runuser -u vpci -- python3 -B "$source_directory/tests/integration/weather-client-bundle.py" verify \ + runuser -u vpci -- python3 -B "$source_directory/tests/integration/weather-client-bundle.py" verify-runtime \ "$weather_client_bundle" "$weather_client_manifest_sha256" || exit 69 else exit 69 @@ -6187,6 +6187,8 @@ start_privacy_observers() { [ "$scenario" = content ] || [ "$scenario" = content-message ] || return 1 ;; content-https-complete-privacy|content-https-missing-privacy) [ "$scenario" = content-https ] || return 1 ;; + weather-products-delivery-weather-privacy|weather-products-delivery-air-quality-privacy) + [ "$scenario" = content-provider ] || return 1 ;; content-provider-privacy|content-provider-adaptive-privacy|content-provider-adaptive-https-privacy|content-provider-https-complete-privacy|content-provider-https-missing-privacy|content-provider-https-baseline-privacy|content-provider-https-origin-only-privacy|content-provider-https-auto-privacy|content-provider-https-digest-origin-only-privacy|content-provider-https-digest-peers-first-privacy|content-provider-https-limited-origin-only-privacy|content-provider-https-limited-peers-first-privacy|content-provider-https-limited-auto-privacy|content-provider-https-checksum-origin-only-privacy|content-provider-https-checksum-peers-first-privacy|content-provider-https-checksum-mismatch-privacy|content-provider-user-privacy|content-provider-named-privacy|content-provider-site-privacy|content-provider-site-cache-only-privacy|content-provider-filter-cold-privacy|content-provider-filter-warm-privacy|weather-client-delivery-privacy) [ "$scenario" = content-provider ] || return 1 ;; content-message-publication-privacy) diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 0d6c79807..45fafa43a 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -821,12 +821,12 @@ WEATHER_CLIENT_ARCHIVE=$RUN_DIRECTORY/weather-client.tar WEATHER_CLIENT_ARCHIVE_SHA256=none WEATHER_CLIENT_MANIFEST_SHA256=none if [ -n "$weather_client_bundle" ]; then - python3 -B "$HERE/weather-client-bundle.py" verify "$weather_client_bundle" "$weather_client_manifest_sha256" + python3 -B "$HERE/weather-client-bundle.py" verify-runtime "$weather_client_bundle" "$weather_client_manifest_sha256" install -m 0600 "$weather_client_bundle" "$WEATHER_CLIENT_ARCHIVE" - python3 -B "$HERE/weather-client-bundle.py" verify "$WEATHER_CLIENT_ARCHIVE" "$weather_client_manifest_sha256" + python3 -B "$HERE/weather-client-bundle.py" verify-runtime "$WEATHER_CLIENT_ARCHIVE" "$weather_client_manifest_sha256" WEATHER_CLIENT_ARCHIVE_SHA256=$(sha256sum "$WEATHER_CLIENT_ARCHIVE" | awk '{ print $1 }') WEATHER_CLIENT_MANIFEST_SHA256=$weather_client_manifest_sha256 - printf '%s\n' 'Additional proof: fresh public ECMWF HTTPS acquisition, protected peer transfer and real Drizz.li UI; origin acquisition is not an overlay claim.' + printf '%s\n' 'Additional proof: public model HTTPS acquisition (ECMWF, plus NOAA for the two-product profile), protected peer transfer and real Drizz.li UI; origin acquisition is not an overlay claim.' fi CODE_ARCHIVE_SHA256=none CODE_MANIFEST_SHA256=none @@ -1105,6 +1105,9 @@ WEATHER_CLIENT_NAMES = {f"weather-client-delivery-{suffix}.json" for suffix in ( "cache-before", "cache-after", "input-cleanup")} WEATHER_CLIENT_HELPERS = ("weather-client-delivery.py", "weather-client-bundle.py", "browser-filter-bundle.py", "content-provider-https-smoke.py", "content-network-smoke.py") +WEATHER_PRODUCTS_NAMES = {name.replace("weather-client-delivery-", "weather-products-delivery-", 1) + for name in WEATHER_CLIENT_NAMES} +WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", *WEATHER_CLIENT_HELPERS) def joined_client_helper(work, names, entrypoint): @@ -1133,11 +1136,23 @@ def browser_filter_helper(work): def weather_client_helper(work): - return joined_client_helper(work, WEATHER_CLIENT_HELPERS, "weather-client-delivery.py") + # A present v2 entrypoint selects the complete v2 validation closure. If + # unsafe, reject it rather than silently interpreting v2 evidence as v1. + try: + (work / "bin/weather-products-delivery.py").lstat() + except FileNotFoundError: + return joined_client_helper(work, WEATHER_CLIENT_HELPERS, "weather-client-delivery.py") + except OSError: + return None + return joined_client_helper(work, WEATHER_PRODUCTS_HELPERS, "weather-products-delivery.py") def weather_client_candidates(root, label, helper, target): # No raw forecast, request/config, CLI receipt or process log is exported. + names = {"weather-client-delivery.py": WEATHER_CLIENT_NAMES, + "weather-products-delivery.py": WEATHER_PRODUCTS_NAMES}.get(helper.name) + if names is None: + return [] validated = Path(tempfile.mkdtemp(prefix="weather-client-validated.", dir=target)) try: subprocess.run(["/usr/bin/python3", "-I", "-B", str(helper), "copy-artifacts", str(root), @@ -1146,7 +1161,7 @@ def weather_client_candidates(root, label, helper, target): cwd="/", env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, timeout=3, check=True) except (OSError, subprocess.SubprocessError): return [] - return [(validated / name, f"{label}/{name}") for name in sorted(WEATHER_CLIENT_NAMES) + return [(validated / name, f"{label}/{name}") for name in sorted(names) if (validated / name).is_file() and not (validated / name).is_symlink()] @@ -1447,7 +1462,7 @@ def collect(home, opt, revision, scenario, guest_status, if observed is None: continue data, original_size = observed - if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES and original_size > len(data): + if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES and original_size > len(data): continue # Preserve complete validated original bytes, never a JSON tail. if scenario == "browser-network" and path.name in browser_names and original_size > len(data): # A truncated JSON tail is not reconstructible structured evidence. @@ -1714,7 +1729,7 @@ fi if [ "$weather_client_manifest_sha256" != none ]; then guest_phase weather-client-inputs printf '%s /home/vpci/weather-client.tar\n' "$weather_client_archive_sha256" | sha256sum --check --strict - - python3 -B tests/integration/weather-client-bundle.py verify /home/vpci/weather-client.tar "$weather_client_manifest_sha256" + python3 -B tests/integration/weather-client-bundle.py verify-runtime /home/vpci/weather-client.tar "$weather_client_manifest_sha256" fi if [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperative-code-proposal ]; then guest_phase code-inputs diff --git a/tests/integration/test-weather-client-bundle.py b/tests/integration/test-weather-client-bundle.py index ba8d10f3a..4050ee27b 100644 --- a/tests/integration/test-weather-client-bundle.py +++ b/tests/integration/test-weather-client-bundle.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-only """Inert archive/parser tests. No real binaries, acquisition, browser or VM.""" import copy +from contextlib import redirect_stdout import hashlib import io import json @@ -27,15 +28,18 @@ def setUp(self): self.pins.start() self.addCleanup(self.pins.stop) self.addCleanup(self.temp.cleanup) - self.files = {name: dict(bytes=len(self.raw), mode=W["mode"](name), sha256=self.digest) - for name in W["FIXED"] | {W["STATIC"] + "index.html"}} + self.files = self.profile_files() - def manifest(self, value=None): - raw = W["encode"](value or dict(version=1, kind=W["KIND"], files=self.files)) + def profile_files(self, version=1): + return {name: dict(bytes=len(self.raw), mode=W["mode"](name), sha256=self.digest) + for name in W["FIXED" if version == 1 else "FIXED_V2"] | {W["STATIC"] + "index.html"}} + + def manifest(self, value=None, version=1): + raw = W["encode"](value or dict(version=version, kind=W["KIND"], files=self.files)) return raw, hashlib.sha256(raw).hexdigest() - def archive(self, suffix=b"\0" * 1024): - raw, expected = self.manifest() + def archive(self, suffix=b"\0" * 1024, version=1): + raw, expected = self.manifest(version=version) archive = self.root / "inputs.tar" with archive.open("wb") as target: for name, data, mode in [("INPUTS.json", raw, 0o400), @@ -60,6 +64,119 @@ def test_exact_roundtrip_readonly_tree_no_execution(self): self.assertEqual((destination / name).stat().st_mode & 0o777, item["mode"]) self.assertEqual(destination.stat().st_mode & 0o777, 0o500) + def test_v1_runtime_gate_preserves_existing_archive_and_schema(self): + archive, expected = self.archive() + before = archive.read_bytes() + checked = W["checked_archive"](archive, expected, runtime=True) + self.assertEqual(checked, W["checked_archive"](archive, expected)) + self.assertEqual(before, archive.read_bytes()) + raw, digest = self.manifest() + self.assertEqual(W["manifest"](raw, digest), dict(version=1, kind=W["KIND"], files=self.files)) + self.assertNotIn(W["WEATHER"] + "src/air-quality.mjs", self.files) + self.assertIn(W["WEATHER"] + "integrations/drizzli/browser-smoke.json", self.files) + + def test_runtime_version_action_checks_whole_archive_without_executing_or_changing_profiles(self): + for version in (1, 2): + self.files = self.profile_files(version) + archive, expected = self.archive(version=version) + before = archive.read_bytes() + output = io.StringIO() + with patch.object(W["sys"], "argv", ["bundle", "runtime-version", str(archive), expected]), redirect_stdout(output): + W["main"]() + self.assertEqual(json.loads(output.getvalue()), dict(version=1, action="runtime-version", + bundle_version=version, runtime_executed=False, + **W["checked_archive"](archive, expected))) + self.assertEqual(before, archive.read_bytes()) + self.assertEqual([path.name for path in self.root.iterdir()], ["inputs.tar"]) + # A valid manifest prefix is not enough: damaged body/trailer or a + # wrong retained digest must be rejected before reporting a version. + damaged = bytearray(before) + damaged[before.rfind(self.raw)] ^= 1 + for raw, digest in ((bytes(damaged), expected), (before + b"\0" * 512, expected), + (before, "f" * 64)): + archive.write_bytes(raw) + rejected = io.StringIO() + with self.subTest(version=version), patch.object(W["sys"], "argv", + ["bundle", "runtime-version", str(archive), digest]), redirect_stdout(rejected): + with self.assertRaises(ValueError): + W["main"]() + self.assertEqual(rejected.getvalue(), "") + archive.unlink() + + def test_v2_roundtrip_is_readonly_and_never_executes_captured_inputs(self): + self.files = self.profile_files(2) + archive, expected = self.archive(version=2) + before = archive.read_bytes() + destination = self.root / "v2-tree" + result = W["checked_archive"](archive, expected, destination) + self.assertEqual(result["files"], len(self.files)) + checked = W["verify_tree"](destination, expected) + self.assertEqual(checked, dict(manifest_sha256=expected, files=len(self.files), runtime_executed=False)) + self.assertEqual(before, archive.read_bytes()) + self.assertEqual(json.loads((destination / "INPUTS.json").read_bytes())["version"], 2) + for name, item in self.files.items(): + self.assertEqual((destination / name).read_bytes(), self.raw) + self.assertEqual((destination / name).stat().st_mode & 0o777, item["mode"]) + self.assertTrue(all(path.stat().st_mode & 0o777 == 0o500 + for path in (destination, *destination.rglob("*")) if path.is_dir())) + + def test_v2_runtime_gate_checks_complete_archive_and_preserves_existing_roots(self): + self.files = self.profile_files(2) + archive, expected = self.archive(version=2) + before = archive.read_bytes() + original_open = W["open_checked"] + opened = [] + def observed_open(*args, **kwargs): + result = original_open(*args, **kwargs) + opened.append(result[0]) + return result + destination = self.root / "existing" + destination.mkdir(mode=0o700) + (destination / "keep").write_bytes(b"untouched") + with patch.dict(W["checked_archive"].__globals__, open_checked=observed_open): + checked = W["checked_archive"](archive, expected, runtime=True) + self.assertEqual(checked, W["checked_archive"](archive, expected)) + self.assertEqual(archive.read_bytes(), before) + with self.assertRaises(ValueError): + W["checked_archive"](archive, expected, destination, runtime=True) + self.assertEqual([p.name for p in destination.iterdir()], ["keep"]) + self.assertEqual((destination / "keep").read_bytes(), b"untouched") + damaged = bytearray(before) + damaged[before.rfind(self.raw)] ^= 1 + archive.write_bytes(damaged) + with self.assertRaises(ValueError): + W["checked_archive"](archive, expected, runtime=True) + # Eligibility still requires the whole exact v2 source closure. + del self.files[W["WEATHER"] + "src/air-quality-core.mjs"] + archive, incomplete = self.archive(version=2) + fresh = self.root / "must-not-create" + with self.assertRaises(ValueError): + W["checked_archive"](archive, incomplete, fresh, runtime=True) + self.assertFalse(fresh.exists()) + self.assertTrue(opened and all(stream.closed for stream in opened)) + + def test_version_profiles_are_exact_not_optional_or_interchangeable(self): + old_proof = W["WEATHER"] + "integrations/drizzli/browser-smoke.json" + new_proof = W["WEATHER"] + "integrations/drizzli/air-quality-browser-smoke.json" + for version in (1, 2): + for fault in ("other_profile", "missing", "cross_proof", "unknown", "version_bool", "version_zero", "version_three", "version_string"): + files = self.profile_files(version) + value = dict(version=version, kind=W["KIND"], files=files) + if fault == "other_profile": value["version"] = 3 - version + elif fault == "missing": del files[old_proof if version == 1 else new_proof] + elif fault == "cross_proof": files[new_proof if version == 1 else old_proof] = dict(files["runtime/node"], mode=0o400) + elif fault == "unknown": files[W["WEATHER"] + "src/unreviewed.mjs"] = dict(files["runtime/node"], mode=0o400) + else: value["version"] = {"version_bool": True, "version_zero": 0, "version_three": 3, "version_string": str(version)}[fault] + raw, expected = self.manifest(value) + with self.subTest(version=version, fault=fault), self.assertRaises(ValueError): + W["manifest"](raw, expected) + for name in W["FIXED_V2"] - W["FIXED"]: + files = self.profile_files(2) + del files[name] + raw, expected = self.manifest(dict(version=2, kind=W["KIND"], files=files)) + with self.subTest(missing_v2=name), self.assertRaises(ValueError): + W["manifest"](raw, expected) + def test_manifest_rejects_missing_unknown_escape_and_mode(self): base = dict(version=1, kind=W["KIND"], files=self.files) for fault in ("missing", "unknown", "escape", "mode", "size_bool", "extra", "version_bool", "zero"): @@ -113,9 +230,10 @@ def test_changed_or_extra_tree_inputs_rejected(self): directory.chmod(0o500) with self.assertRaises(ValueError): W["verify_tree"](destination, expected) - def proof_fixture(self): - selected = copy.deepcopy(self.files) - module_hashes = {name: self.digest for name in W["PROVEN_WEATHER_MODULES"]} + def proof_fixture(self, version=1): + selected = self.profile_files(version) + modules = W["PROVEN_WEATHER_MODULES"] | (W["AQ_MODULES"] if version == 2 else set()) + module_hashes = {name: self.digest for name in modules} runtime = {name: self.digest for name in W["PROVEN_RUNTIME_FILES"]} static = {"index.html": self.digest} proof = dict(passed=True, scope=dict(synthetic_weather=True, core_executed=False), runtime=dict(sha256=runtime), @@ -132,14 +250,143 @@ def proof_fixture(self): build = dict(compiled=True, embedded_definitions=True, required_environment={}, source_pin_sha256=self.digest, build_script_sha256=self.digest, original_sources_unchanged=True, artifacts=[dict( path="build/eccodes/eccodes-build/bin/grib_ls", bytes=31328216, sha256=self.digest)]) - return selected, {"browser-smoke.json": proof, "upstream.json": upstream, - "volparossa-staging.json": metadata, "build-receipt.json": build} + records = {"browser-smoke.json" if version == 1 else "air-quality-browser-smoke.json": proof, + "upstream.json": upstream, "volparossa-staging.json": metadata, "build-receipt.json": build} + if version == 2: + proof.update(phase="complete", scenario="precipitation", air_quality_scenario=True, wrapper_outcome="completed") + proof["scope"].update(synthetic_air_quality=True, actual_drizzli_ui=True, actual_http_bridge=True, + overlay_proven=False, upstream_origin_proven=False) + proof["runtime"]["node_sha256"] = self.digest + proof["source"]["bridge_modules"]["src/air-quality-core.mjs"] = self.digest + proof["source"]["overlay_sha256"] = {name: self.digest for name in W["OVERLAY_FILES"]} + proof["source"]["fixture_modules"] = {name: self.digest for name in W["FIXTURE_MODULES"]} + proof["source"]["fixture_sha256"] = self.digest + proof["cleanup"] = dict.fromkeys(("browser_exited", "fixture_exited", "private_profile_removed", + "normal_browser_stop", "normal_fixture_stop"), True) + proof["outer_cleanup"] = dict.fromkeys(("namespace_wrapper_exited", "private_profile_removed", "host_netns_unchanged"), True) + proof["checks"] = dict.fromkeys(("precipitation_native_windows", "precipitation_missing_zero_small_amount", + "precipitation_no_hourly_rate", "air_quality_is_separate_product", "air_quality_native_times", + "air_quality_missing_zero_concentrations", "air_quality_not_aqi_or_observations", "actual_refresh", + "no_external_resources", "weather_failure_preserves_air_quality", "weather_expiry_preserves_air_quality", + "aq_unavailable_weather_preserved", "aq_stale_weather_preserved", "air_quality_expiry_preserves_weather"), True) + proof["checks"].update(provider_requests=11, air_quality_provider_requests=11) + for product in ("precipitation", "air_quality"): + for phase in ("initial", "refresh"): + proof["checks"][f"{product}_{phase}_dom_sha256"] = self.digest + records["noaa-gefs-smoke.json"] = dict(version=1, kind="direct-noaa-native-particulate-forecast", + success=True, origin="https://noaa-gefs-pds.s3.amazonaws.com", node_sha256=self.digest, + decoder_sha256=self.digest, source_modules={name: self.digest for name in W["NOAA_MODULES"]}, + scope=dict(actual_origin_acquisition=True, actual_pinned_native_decoder=True, + core_executed=False, overlay_proven=False, browser_executed=False, accuracy_proven=False)) + records["ecmwf-precipitation-smoke.json"] = dict(version=1, + kind="direct-ecmwf-native-precipitation-forecast", success=True, + source="https://data.ecmwf.int/forecasts", node_sha256=self.digest, decoder_sha256=self.digest, + native_accumulation_windows_preserved=True, hourly_rates_inferred=False, + core_overlay_roundtrip_proven=False, browser_execution_proven=False, + source_modules={name: self.digest for name in W["ECMWF_MODULES"]}) + return selected, records - def check_proof(self, selected, records): + def check_proof(self, selected, records, version=1): # No real source/binary capture or import. Only inert JSON-shaped values. with patch.dict(W["source_proof"].__globals__, small_json=lambda path: records[path.name]): return W["source_proof"](Path("/inert/weather"), Path("/inert/browser"), - Path("/inert/ui"), Path("/inert/runtime"), selected) + Path("/inert/ui"), Path("/inert/runtime"), selected, version) + + def test_v2_complete_provenance_binds_expanded_modules_and_separate_proofs(self): + self.check_proof(*self.proof_fixture(2), version=2) + names = {*(W["WEATHER"] + relative for relative in + W["AQ_MODULES"] | W["NOAA_MODULES"] | W["ECMWF_MODULES"] | W["FIXTURE_MODULES"] | {"src/air-quality-core.mjs"}), + *(W["WEATHER"] + "integrations/drizzli/overlay/" + relative for relative in W["OVERLAY_FILES"]), + *(W["UI_STAGE"] + relative for relative in W["OVERLAY_FILES"]), + *(W["UI_STAGE"] + "src/lib/services/volparossa/" + Path(relative).name for relative in W["AQ_MODULES"]), + W["WEATHER"] + "integrations/drizzli/browser-fixture.mjs", W["UI_STAGE"] + "LICENSE.VOLPAROSSA", + W["WEATHER"] + "LICENSE"} + for name in names: + selected, records = self.proof_fixture(2) + selected[name]["sha256"] = "f" * 64 + with self.subTest(name=name), self.assertRaises(ValueError): + self.check_proof(selected, records, version=2) + + def test_v2_expanded_provenance_maps_are_exact(self): + paths = [("air-quality-browser-smoke.json", "source", "bridge_modules"), + ("air-quality-browser-smoke.json", "source", "weather_modules"), + ("air-quality-browser-smoke.json", "source", "overlay_sha256"), + ("air-quality-browser-smoke.json", "source", "fixture_modules"), + ("upstream.json", "weather_modules"), ("noaa-gefs-smoke.json", "source_modules"), + ("ecmwf-precipitation-smoke.json", "source_modules")] + for path in paths: + for fault in ("empty", "missing", "extra", "hash", "none", "list"): + selected, records = self.proof_fixture(2) + parent = records + for key in path[:-1]: parent = parent[key] + changed = dict(parent[path[-1]]) + if fault == "empty": changed = {} + elif fault == "missing": changed.pop(next(iter(changed))) + elif fault == "extra": changed["PRIVATE"] = self.digest + elif fault == "hash": changed[next(iter(changed))] = "PRIVATE" + elif fault == "none": changed = None + else: changed = list(changed) + parent[path[-1]] = changed + with self.subTest(path=path, fault=fault), self.assertRaises(ValueError): + self.check_proof(selected, records, version=2) + + def test_v2_scenario_scope_cleanup_and_checks_cannot_be_relaxed(self): + selected, records = self.proof_fixture(2) + proof = records["air-quality-browser-smoke.json"] + mutations = [(key, [False, None, "wrong"]) for key in + ("phase", "scenario", "air_quality_scenario", "wrapper_outcome")] + paths = [((key,), bad) for key, bad_values in mutations for bad in bad_values] + for group in ("scope", "cleanup", "outer_cleanup"): + paths.extend(((group, name), bad) for name, value in proof[group].items() + for bad in (not value, int(value))) + for name, value in proof["checks"].items(): + bad_values = (False, 1) if type(value) is bool else (10, 12, True, "11") if type(value) is int else ("f" * 64, "PRIVATE") + paths.extend((("checks", name), bad) for bad in bad_values) + paths.append((("runtime", "node_sha256"), "f" * 64)) + for path, bad in paths: + selected, records = self.proof_fixture(2) + parent = records["air-quality-browser-smoke.json"] + for key in path[:-1]: parent = parent[key] + parent[path[-1]] = bad + with self.subTest(path=path, bad=bad), self.assertRaises(ValueError): + self.check_proof(selected, records, version=2) + + def test_v2_noaa_proof_preserves_source_scope_and_runtime_authority(self): + selected, records = self.proof_fixture(2) + noaa = records["noaa-gefs-smoke.json"] + paths = [(("version",), bad) for bad in (True, 0, 2, "1")] + paths.extend(((name,), bad) for name, bad in ( + ("kind", "other"), ("success", False), ("success", 1), ("origin", "https://untrusted.invalid"), + ("node_sha256", "f" * 64), ("decoder_sha256", "f" * 64))) + paths.extend((("scope", name), bad) for name, value in noaa["scope"].items() + for bad in (not value, int(value))) + for path, bad in paths: + selected, records = self.proof_fixture(2) + parent = records["noaa-gefs-smoke.json"] + for key in path[:-1]: parent = parent[key] + parent[path[-1]] = bad + with self.subTest(path=path, bad=bad), self.assertRaises(ValueError): + self.check_proof(selected, records, version=2) + + def test_proof_profile_version_rejects_unknown_and_boolean_values(self): + for version in (True, False, 0, 3, "1", "2", None): + with self.subTest(version=version), self.assertRaises(ValueError): + self.check_proof(*self.proof_fixture(), version=version) + + def test_v2_ecmwf_proof_preserves_accumulation_and_separate_source_scope(self): + faults = [("version", value) for value in (True, 0, 2, "1")] + faults.extend((name, value) for name, value in ( + ("kind", "other"), ("success", False), ("success", 1), + ("source", "https://untrusted.invalid"), ("node_sha256", "f" * 64), ("decoder_sha256", "f" * 64), + ("native_accumulation_windows_preserved", False), ("native_accumulation_windows_preserved", 1), + ("hourly_rates_inferred", True), ("hourly_rates_inferred", 0), + ("core_overlay_roundtrip_proven", True), ("core_overlay_roundtrip_proven", 0), + ("browser_execution_proven", True), ("browser_execution_proven", 0))) + for name, value in faults: + selected, records = self.proof_fixture(2) + records["ecmwf-precipitation-smoke.json"][name] = value + with self.subTest(name=name, value=value), self.assertRaises(ValueError): + self.check_proof(selected, records, version=2) def test_complete_provenance_binds_all_known_module_groups_without_source_execution(self): self.check_proof(*self.proof_fixture()) diff --git a/tests/integration/test-weather-client-runner.py b/tests/integration/test-weather-client-runner.py index d580ba98a..109af7614 100644 --- a/tests/integration/test-weather-client-runner.py +++ b/tests/integration/test-weather-client-runner.py @@ -8,6 +8,7 @@ import subprocess import tarfile import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -23,6 +24,14 @@ def call(self, runner, *args, env=None): return subprocess.run(["/bin/sh", str(HERE / runner), *args], capture_output=True, timeout=5, env=env) + def test_runners_require_the_joined_runtime_profile_not_only_valid_archive(self): + for runner, calls in (("run-alpha-topology-vm.sh", 3), ("kvm-alpha-topology.sh", 2)): + source = (HERE / runner).read_text() + verifier_lines = [line for line in source.splitlines() + if 'weather-client-bundle.py" verify' in line or 'weather-client-bundle.py verify' in line] + self.assertEqual(len(verifier_lines), calls) + self.assertTrue(all(" verify-runtime " in line for line in verifier_lines)) + def required(self, runner, scenario="content-provider"): args = ["--execute", "--yes", "--scenario", scenario, "--output", "/unread/output", "--expected-commit", "a" * 40, @@ -140,6 +149,66 @@ def setUpClass(cls): cls.api = {"__name__": "inert_weather_collector"} exec(compile(body, "weather-collector", "exec"), cls.api) + def test_products_helper_requires_complete_root_owned_closure_without_v1_fallback(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary).resolve() + directory = work / "bin" + directory.mkdir() + for name in self.api["WEATHER_PRODUCTS_HELPERS"]: + path = directory / name + path.write_text("raise RuntimeError('inert; must not execute')\n") + path.chmod(0o555) + original = Path.lstat + bad_path, fault = None, {} + + def fixture_stat(path): + actual = original(path) + # Simulate trusted root staging without requiring root, mounts, + # chown, or changing the host. The selected fault stays real to + # the production closure checker. + values = dict(st_mode=actual.st_mode & ~0o022, st_uid=0, + st_nlink=actual.st_nlink, st_size=actual.st_size) + if path == bad_path: + values.update(fault) + return SimpleNamespace(**values) + + with patch.object(Path, "lstat", fixture_stat): + self.assertEqual(self.api["weather_client_helper"](work), directory / "weather-products-delivery.py") + for name in self.api["WEATHER_PRODUCTS_HELPERS"]: + bad_path = directory / name + for fault in (dict(st_uid=1000), dict(st_nlink=2), + dict(st_mode=stat.S_IFREG | 0o666), dict(st_size=262145), + dict(st_mode=stat.S_IFLNK | 0o555)): + with self.subTest(name=name, fault=fault): + self.assertIsNone(self.api["weather_client_helper"](work)) + bad_path, fault = None, {} + products = directory / "weather-products-delivery.py" + products.unlink() + self.assertEqual(self.api["weather_client_helper"](work), directory / "weather-client-delivery.py") + products.symlink_to("missing-products-driver") + self.assertIsNone(self.api["weather_client_helper"](work)) + + def test_products_candidate_allowlist_never_exports_private_or_other_profile_files(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + target = root / "output" + target.mkdir() + + def inert_copy(command, **kwargs): + self.assertEqual(command[4:6], ["copy-artifacts", str(root)]) + copied = Path(command[6]) + for name in self.api["WEATHER_CLIENT_NAMES"] | self.api["WEATHER_PRODUCTS_NAMES"] | { + "weather-products-delivery-config.json", "forecast.json", "private.log"}: + (copied / name).write_text("{}") + + with patch.object(self.api["subprocess"], "run", inert_copy): + candidates = self.api["weather_client_candidates"](root, "work-1", + root / "weather-products-delivery.py", target) + self.assertEqual({name for _, name in candidates}, + {"work-1/" + name for name in self.api["WEATHER_PRODUCTS_NAMES"]}) + with patch.object(self.api["subprocess"], "run", side_effect=AssertionError("must not execute")): + self.assertEqual(self.api["weather_client_candidates"](root, "work-1", root / "unknown.py", target), []) + def acquisition(self): return dict(schema=1, success=True, origin_https=True, acquisition_outside_overlay=True, upstream_signature_attested=False, synthetic_weather=False, @@ -148,30 +217,46 @@ def acquisition(self): index_sha256=SHA, field_sha256=SHA) for step in (3, 9, 24) for parameter in ("2t", "10u", "10v")]) - def collect(self, raw, limit=None, published=None, late_invalid=False): + def products_acquisition(self): + return dict(schema=2, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, synthetic_air_quality=False, + independent_product_runs=True, + decoder_sha256="3a3b64581e6922db4287445f217f993f6967dea5fcc45ff66944b0bc883408f4", + products={name: dict(forecast_sha256=SHA, forecast_bytes=6000, + fields=[dict(lead_hours=step, parameter=parameter, + index_sha256=SHA, field_sha256=SHA) + for step in steps for parameter in parameters]) + for name, steps, parameters in ( + ("weather", (3, 9, 24), ("2t", "10u", "10v", "tp")), + ("air_quality", (3, 6, 12), ("PMTF", "PMTC")))}) + + def collect(self, raw, limit=None, published=None, late_invalid=False, products=False): with tempfile.TemporaryDirectory() as temporary: + prefix = "weather-products-delivery" if products else "weather-client-delivery" base = Path(temporary).resolve() home, opt, proc, cgroups = (base / name for name in ("home", "opt", "proc", "cgroups")) for directory in (home, opt, proc, cgroups): directory.mkdir() (home / "alpha-output").mkdir() if published is not None: - early = home / "alpha-output/weather-client-delivery-cold-evidence.json" + early = home / "alpha-output" / (prefix + "-cold-evidence.json") early.write_bytes(published) early.chmod(0o600) work = opt / ("va." + "a" * 32 + ".ABC123") work.mkdir() - source = work / "weather-client-delivery-acquisition.json" + source = work / (prefix + "-acquisition.json") source.write_bytes(raw) source.chmod(0o600) if late_invalid: - invalid = work / "weather-client-delivery-evidence.json" + invalid = work / (prefix + "-evidence.json") invalid.write_text('{"unknown":"PRIVATE CANARY"}') invalid.chmod(0o600) for name in ("weather-client-delivery-cold-consumer.json", "weather-client-delivery-input.json", - "weather-client-delivery.err", "forecast.json", "config.json"): + "weather-client-delivery.err", "forecast.json", "config.json", + "weather-products-delivery-input.json", "weather-products-delivery-air_quality-cold-consumer.json", + "weather-products-delivery.err", "air-quality.json"): (work / name).write_text("PRIVATE CANARY") - overrides = {"weather_client_helper": lambda root: HERE / "weather-client-delivery.py"} + overrides = {"weather_client_helper": lambda root: HERE / (prefix + ".py")} if limit is not None: overrides["FILE_LIMIT"] = limit # Bypass only root-owned staging in this unprivileged inert test. @@ -190,6 +275,33 @@ def test_valid_receipt_survives_exactly_but_never_claims_vm_success(self): self.assertEqual(summary["cleanup"], {"complete": False, "verified": False}) self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + def test_products_valid_receipt_survives_exactly_without_inventing_joined_success(self): + raw = json.dumps(self.products_acquisition(), indent=2).encode() + b"\n" + files = self.collect(raw, products=True) + self.assertEqual(files["work-1/weather-products-delivery-acquisition.json"], raw) + self.assertFalse(json.loads(files["vm-incomplete.json"])["success"]) + self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + self.assertFalse(any("weather-client-delivery" in name for name in files)) + + def test_products_export_rejects_unknown_private_duplicate_and_mixed_source_data(self): + valid = self.products_acquisition() + wrong_field = self.products_acquisition() + wrong_field["products"]["air_quality"]["fields"][0]["parameter"] = "2t" + bad = [b"not json", json.dumps(dict(valid, prompt="PRIVATE CANARY")).encode(), + b'{"schema":"PRIVATE CANARY",' + json.dumps(valid).encode()[1:], + json.dumps(dict(valid, synthetic_air_quality=True)).encode(), + json.dumps(dict(valid, decoder_sha256="PRIVATE CANARY")).encode(), + json.dumps(wrong_field).encode()] + for raw in bad: + with self.subTest(raw=raw[:50]): + files = self.collect(raw, products=True) + self.assertNotIn("work-1/weather-products-delivery-acquisition.json", files) + self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + files = self.collect(json.dumps(valid).encode(), products=True, limit=64) + self.assertFalse(any("weather-products-delivery" in name for name in files)) + files = self.collect(json.dumps(valid).encode(), products=True, late_invalid=True) + self.assertFalse(any("weather-products-delivery" in name for name in files)) + def test_private_unknown_duplicate_and_malformed_json_do_not_escape(self): valid = self.acquisition() bad = [b"not json", json.dumps(dict(valid, prompt="PRIVATE CANARY")).encode(), diff --git a/tests/integration/test-weather-products-delivery-shell.py b/tests/integration/test-weather-products-delivery-shell.py new file mode 100644 index 000000000..1c19e3669 --- /dev/null +++ b/tests/integration/test-weather-products-delivery-shell.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert two-product dispatch, timing and cleanup contracts; no network or VM.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +SOURCE = HERE / "weather-products-delivery.sh" +LEGACY = HERE / "weather-client-delivery.sh" + + +class ProductsShell(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="weather-products-shell-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + + def call(self, body, source=None): + return subprocess.run(["/bin/sh", "-s"], input="set -eu\n" + + (SOURCE.read_text() if source is None else source) + "\n" + body, + text=True, capture_output=True, timeout=5, + env=dict(PATH="/usr/bin:/bin", WORK=str(self.root), site_control_gid="987")) + + def ok(self, result): + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_syntax_and_inert_sourcing(self): + subprocess.run(["/bin/sh", "-n", str(SOURCE)], check=True, timeout=5) + self.ok(self.call(":")) + self.assertEqual(list(self.root.iterdir()), []) + + def test_closed_independent_product_mapping(self): + result = self.call(''' +for kind in weather air_quality; do + weather_products_select "$kind" + printf '%s %s %s %s %s %s\n' "$weather_product" "$weather_product_file" \ + "$weather_product_manifest" "$weather_product_source_cache" "$weather_publications" "$weather_product_prefix" +done +if weather_products_select arbitrary; then exit 90; fi +''') + self.ok(result) + self.assertEqual(result.stdout.splitlines(), [ + "weather forecast.json manifest.bin source-cache 4 weather-products-delivery-weather", + "air_quality air-quality.json air-quality-manifest.bin air-quality-source-cache 5 weather-products-delivery-air-quality", + ]) + + def test_single_deadline_caps_commands_and_refuses_expired_stage(self): + self.ok(self.call(''' +date() { printf '1000\n'; } +[ "$(weather_products_remaining 120)" = 120 ] +WEATHER_PRODUCTS_DEADLINE=1840 +[ "$(weather_products_remaining 120)" = 120 ] +WEATHER_PRODUCTS_DEADLINE=1031 +[ "$(weather_products_remaining 200)" = 31 ] +[ "$(weather_products_remaining 270)" = 31 ] +WEATHER_PRODUCTS_DEADLINE=1000 +if weather_products_remaining 200; then exit 90; fi +WEATHER_PRODUCTS_DEADLINE=999 +if weather_products_remaining 270; then exit 90; fi +''')) + + def test_cleanup_retains_before_all_independent_attempts(self): + (self.root / "bin").mkdir() + (self.root / "bin/weather-products-delivery.py").touch() + output = self.root / "client-fixtures/weather-viewer/browser-output" + output.mkdir(parents=True) + (output / "report.json").touch() + self.ok(self.call(''' +python3() { printf 'retain\n' >> "$WORK/calls"; return 1; } +weather_products_private() { printf '%s\n' "$1" >> "$WORK/calls"; return 1; } +weather_client_home_cleanup() { printf 'home\n' >> "$WORK/calls"; } +if weather_client_delivery_cleanup; then exit 90; fi +''')) + self.assertEqual((self.root / "calls").read_text().splitlines(), + ["retain", "publisher-cleanup", "user-cleanup", "home"]) + + def test_already_retained_report_not_replaced(self): + (self.root / "bin").mkdir() + (self.root / "bin/weather-products-delivery.py").touch() + retained = self.root / "weather-products-delivery-original-report.json" + retained.write_bytes(b"retained") + self.ok(self.call(''' +python3() { exit 90; } +weather_products_private() { :; } +weather_client_home_cleanup() { :; } +weather_client_delivery_cleanup +''')) + self.assertEqual(retained.read_bytes(), b"retained") + + def test_verified_closed_dispatch_before_any_v1_work(self): + source = LEGACY.read_text() + head = source.split("weather_client_delivery_run() {", 1)[1].split(" PHASE=weather-client-delivery-prepare", 1)[0] + helper_root = self.root / "tests/integration" + helper_root.mkdir(parents=True) + (helper_root / "weather-products-delivery.sh").write_text( + 'weather_products_delivery_run() { printf "v2\\n"; }\n') + for version in (1, 2, 3, "2"): + with self.subTest(version=version, value_type=type(version)): + import json + value = json.dumps({"bundle_version": version}) + result = self.call(''' +source_directory=$WORK +weather_client_bundle=inert +weather_client_manifest_sha256=inert +python3() { printf '%s\n' 'JSON'; } +fail() { exit 77; } +run_selected +'''.replace("JSON", value), source="run_selected() {\n" + head + '\nprintf "v1\\n"\n}\n') + self.assertEqual(result.returncode, 0 if type(version) is int and version in (1, 2) else 77) + if result.returncode == 0: self.assertEqual(result.stdout, f"v{version}\n") + self.assertLess(head.index("runtime-version"), head.index(".bundle_version")) + self.assertLess(head.index(".bundle_version"), head.index(' . "$source_directory')) + + def test_publication_cleanup_independent_captures_and_retained_proof_order(self): + source = SOURCE.read_text() + run = source.split("weather_products_delivery_run() {", 1)[1] + self.assertLess(run.index(' acquire \\'), run.index('WEATHER_PRODUCTS_DEADLINE=')) + self.assertLess(run.index('WEATHER_PRODUCTS_DEADLINE='), run.index(' content publish')) + self.assertLess(run.index('weather-products-delivery-manifest.json'), run.index('chown -R')) + self.assertLess(run.index('publisher-cleanup "$weather_publisher"'), run.index('PHASE=weather-products-delivery-cold-')) + self.assertIn('cold "$weather_product"', run) + self.assertIn('start_privacy_observers "$weather_product_prefix-privacy"', run) + self.assertIn('content_provider_start_control_observer "$weather_product_prefix-control"', run) + self.assertLess(run.index('content_provider_stop_control_observer'), run.index(' cold-evidence "$WORK"')) + self.assertLess(run.index(' copy-cold-evidence "$WORK"'), run.index(' prepare "$weather_client_directory"')) + self.assertLess(run.index(' retain "$WORK"'), run.index('weather_client_delivery_cleanup')) + self.assertLess(run.index('weather_client_delivery_cleanup'), run.index(' evidence "$WORK"')) + self.assertIn('--lifetime-seconds 900', run) + self.assertIn('WEATHER_PRODUCTS_DEADLINE=$(($(date +%s) + 840))', run) + for maximum in (120, 200, 40, 270): self.assertIn(f'weather_products_remaining {maximum}', source) + self.assertEqual(source.count('--kill-after=30s'), 4) + self.assertNotIn('--kill-after=10s', source) + self.assertNotIn('env HOME=', source) + self.assertNotIn('--groups=', source) + + def test_observers_accept_only_exact_new_product_prefixes(self): + control = (HERE / "content-provider-smoke.sh").read_text() + privacy = (HERE / "kvm-alpha-topology.sh").read_text() + for suffix in ("weather", "air-quality"): + self.assertIn(f"weather-products-delivery-{suffix}-control", control) + self.assertIn(f"weather-products-delivery-{suffix}-privacy", privacy) + self.assertNotIn('weather-products-*', control) + self.assertNotIn('weather-products-*', privacy) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-weather-products-delivery.py b/tests/integration/test-weather-products-delivery.py new file mode 100644 index 000000000..a466d4c59 --- /dev/null +++ b/tests/integration/test-weather-products-delivery.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Synthetic adversarial evidence tests, not an origin/overlay/UI proof.""" +import copy +import datetime +import hashlib +import json +import os +from pathlib import Path +import runpy +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +A = runpy.run_path(str(HERE / "weather-products-delivery.py")) +OLD = runpy.run_path(str(HERE / "test-weather-client-delivery.py")) +SHA = "a" * 64 + + +def selected(): + result = dict(schema=2, products={}) + for key, p in A["PRODUCTS"].items(): + value = OLD["publication"]() + value.update(name=p["name"], manifest_id=("b" if key == "weather" else "d")*64, + sha256=("c" if key == "weather" else "e")*64, bytes=6000 if key == "weather" else 4000) + result["products"][key] = dict(publication=value, + query=A["fixed_query"](key, "2026-10-10T12:00:00.000Z" if key == "weather" else "2026-10-10T00:00:00.000Z")) + return result + + +def acquisition(): + selection = selected() + return dict(schema=2, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, synthetic_air_quality=False, + independent_product_runs=True, decoder_sha256=A["B"]["DECODER_SHA"], products={ + key: dict(forecast_sha256=selection["products"][key]["publication"]["sha256"], + forecast_bytes=selection["products"][key]["publication"]["bytes"], + fields=[dict(lead_hours=step, parameter=parameter, index_sha256=SHA, field_sha256=SHA) + for step in p["steps"] for parameter in p["parameters"]]) for key, p in A["PRODUCTS"].items()}) + + +def report(): + legacy = OLD["report"](); expected = legacy["expected"] + result = dict(schema=2, passed=True, phase="complete", source=legacy["source"], + expected={key: expected[key] for key in ("expected_sha256", "core_sha256", "display_sha256")}, + products={}, scope=dict(legacy["scope"], synthetic_air_quality=False, separate_native_runs=True), + checks=dict(no_external_resources=True, provider_requests_checked=2, browser_api_requests=4, + initial_dom_display_sha256=SHA, refresh_dom_display_sha256=SHA), + cleanup=legacy["cleanup"], outer_cleanup=legacy["outer_cleanup"]) + result["source"]["bridge_modules"] = {name: SHA for name in A["BRIDGE_MODULES"]} + for key in A["PRODUCTS"]: + publication = selected()["products"][key]["publication"] + result["products"][key] = dict(expected=dict(cold_receipt_sha256=SHA, display_sha256=SHA, + forecast_sha256=publication["sha256"], manifest_id=publication["manifest_id"]), checks=copy.deepcopy(legacy["checks"])) + return result + + +def save(path, value, mode=0o600): + A["write_new"](path, json.dumps(value).encode(), mode) + + +def joined_fixture(work): + f = OLD["FILTER"]["fixture"]() + selection = selected() + common = {"input": selection, "acquisition": acquisition(), + "publisher-cleanup": dict(directory_removed=True, files_removed=True), + "input-cleanup": dict(directory_removed=True, files_removed=True), + "prepared": dict(prepared=True, expected_sha256=SHA, display_sha256=SHA)} + caches = {key: dict(device=1, inode=10+i, mode="0700", user_uid=1234, group_gid=os.getgid()+1) + for i, key in enumerate(A["PRODUCTS"])} + for key, p in A["PRODUCTS"].items(): + publication = selection["products"][key]["publication"] + receipt = OLD["receipt"](work / "state-client" / p["cache"], work / "client-fixtures/weather-viewer" / p["filename"]) + receipt.update({k: v for k, v in publication.items() if k != "expires_unix_seconds"}) + receipt.update(peer_bytes=publication["bytes"], provider_peer_ids=f["cold"]["application"]["receipt"]["provider_peer_ids"], + control_relay_peer_id=f["layout"]["control_relay_peer_id"]) + files = { + "selection": f["selected_route"], "control": f["cold"]["control_privacy"], + "cold-consumer": dict(schema=2, product=key, receipt=receipt, publication=publication, + cold_cache_requested=True, core_sha256=SHA, cold_receipt_sha256=SHA, + observed_unix_seconds=publication["expires_unix_seconds"]-60, + boundary=dict(user_uid=985, primary_control_gid=os.getgid(), supplementary_groups_empty=True, + all_capabilities_dropped=True, no_new_privileges=True, client_namespace=True, acquisition_parent_namespace=False)), + "publish": dict(operation="offline_content_publish", network_publication=False, + publisher_key_hex=publication["publisher_key"], content_type="application/json", chunks=1, + **{k: publication[k] for k in ("name", "revision", "bytes", "manifest_id", "expires_unix_seconds")}), + "cache-before": caches[key], "cache-after": caches[key], + } + files.update({"privacy-" + role: captured for role, captured in f["cold"]["privacy"].items()}) + for node in f["layout"]["provider_nodes"]: + files[node + "-import"] = dict(f["imports"][node], manifest_id=publication["manifest_id"], + content_bytes=publication["bytes"], agent_cache=str(work / f"state-{node}" / p["cache"])) + files[node + "-serve"] = dict(f["serves"][node], publications=4 if key == "weather" else 5) + for suffix, value in files.items(): save(A["artifact"](work, suffix, key), value) + B = A["B"] + items = {name: dict(bytes=1, mode=B["mode"](name), sha256=SHA) for name in B["FIXED_V2"] | {B["STATIC"]+"index.html"}} + for name, digest in (("runtime/node", B["NODE_SHA"]), ("decoder/grib_ls", B["DECODER_SHA"]), + (B["WEATHER"]+"third_party/eccodes-sources.json", B["PIN_SHA"]), (B["WEATHER"]+"scripts/build_eccodes.py", B["BUILD_SHA"])): + items[name]["sha256"] = digest + manifest = B["encode"](dict(version=2, kind=B["KIND"], files=items)); digest = hashlib.sha256(manifest).hexdigest() + A["write_new"](A["artifact"](work, "manifest"), manifest, 0o400) + for phase in ("before", "after"): + common["tree-"+phase] = dict(version=1, action="verify-tree", manifest_sha256=digest, files=len(items), runtime_executed=False) + common["cache-"+phase] = dict(schema=2, products=caches) + browser = report(); browser["source"]["bundle_manifest_sha256"] = digest + browser["source"]["built_tree_sha256"] = hashlib.sha256(json.dumps({"index.html": SHA}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + common["original-report"] = browser + save(work / "a01-expected-peers.json", f["expected_peers"]); save(work / "content-provider-layout.json", f["layout"]) + socket = work / "runtime-client/control/agent.sock"; socket.parent.mkdir(parents=True) + A["write_new"](socket, b"inert stat-only fixture") + for suffix, value in common.items(): save(A["artifact"](work, suffix), value) + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda key, *args: caches[key]): + save(A["artifact"](work, "cold-evidence"), A["cold_evidence"](work, digest, SHA)) + return digest, caches + + +class Products(unittest.TestCase): + def test_acquisition_commands_include_precipitation_and_independent_noaa_cycle_with_one_deadline(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) + source = base / "inputs"; source.mkdir(mode=0o700) + A["initialize"](root); A["write_new"](source / "INPUTS.json", b"fixture", 0o400) + calls = [] + def execute(command, cwd, maximum, timeout): + calls.append((command, timeout)) + if command[-1] == "-V": return b"ecCodes Version 2.44.0\n" + key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" + p = A["PRODUCTS"][key] + def field(parameter): + return dict(parameter=parameter, decoder_sha256=A["B"]["DECODER_SHA"], index_sha256=SHA, field_sha256=SHA) + samples = [] + for step in p["steps"]: + samples.append(dict(lead_hours=step, **(dict(evidence=field("2t"), + wind_10m=dict(evidence=dict(eastward=field("10u"), northward=field("10v"))), + precipitation=dict(evidence=field("tp"))) if key == "weather" else + dict(evidence=dict(pm2_5=field("PMTF"), pm10=field("PMTC")))))) + return json.dumps(dict(schema=p["schema"], source=dict(provider=p["provider"], acquisition="origin-https", + model=p["model"], model_run_at=command[command.index("--run")+1]), + area=dict(latitude=52.52, longitude=13.41, sharing="public-community"), samples=samples)).encode() + clock = datetime.datetime(2026, 10, 11, 14, tzinfo=datetime.timezone.utc).timestamp() + with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None), \ + patch.dict(A["B"], verify_tree=lambda *args: None, manifest=lambda *args: {"version": 2}), \ + patch.object(A["time"], "time", return_value=clock), \ + patch.object(A["time"], "monotonic", side_effect=[100, 120, 480]): + result = A["acquisition"](source, SHA, root, "net:[1]", "net:[2]", 1000) + A["validate_acquisition"](result) + weather, aq = calls[1][0], calls[2][0] + self.assertIn("--include-precipitation", weather); self.assertNotIn("--include-precipitation", aq) + self.assertEqual(weather[weather.index("--run")+1], "2026-10-11T00:00:00.000Z") + self.assertEqual(aq[aq.index("--run")+1], "2026-10-11T00:00:00.000Z") + # Same cycle time is permissible when independently selected; + # unlike their native samples, the times need not differ by force. + self.assertEqual(weather[weather.index("--steps")+1], "3,9,24") + self.assertEqual(aq[aq.index("--steps")+1], "3,6,12") + self.assertEqual([call[1] for call in calls], [10, 360, 340]) + self.assertTrue((root / "forecast.json").exists()); self.assertTrue((root / "air-quality.json").exists()) + + def test_preparer_uses_both_receipts_and_separate_caches(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); viewer = base / "weather-viewer"; viewer.mkdir(mode=0o700) + selection = selected(); save(viewer / "selection.json", selection, 0o400) + def execute(command, cwd, maximum, timeout): + self.assertEqual(command[command.index("--air-quality")+1], str(viewer / "air-quality.json")) + self.assertEqual(command[command.index("--air-quality-receipt")+1], str(viewer / "air-quality-cold-receipt.json")) + config = A["read"](viewer / "config.json") + self.assertNotEqual(config["core"]["cache"], config["airQuality"]["core"]["cache"]) + self.assertEqual(config["core"]["scratchParent"], config["airQuality"]["core"]["scratchParent"]) + self.assertTrue(config["core"]["query"]["includePrecipitation"]) + expected = dict(schema=2, core_sha256=SHA, display_sha256=SHA, + products={key: {"publication": value["publication"]} for key, value in selection["products"].items()}) + raw = A["B"]["encode"](expected); A["write_new"](viewer / "expected.json", raw, 0o400) + return json.dumps(dict(prepared=True, expected_sha256=hashlib.sha256(raw).hexdigest(), display_sha256=SHA)).encode() + with patch.dict(A["prepare"].__globals__, run=execute), patch.dict(A["B"], fingerprint=lambda *args: {"sha256": SHA}): + result = A["prepare"](base / "source", viewer, base / "binary", base / "agent.sock", + base / "state-client/weather-cache", base / "state-client/air-quality-cache") + self.assertTrue(result["prepared"]) + + def test_partial_browser_report_is_retainable_but_never_success(self): + failed = report(); failed.update(passed=False, phase="provider", checks={}, cleanup={}) + failed["scope"].update(actual_drizzli_ui=False, real_core_adapter=False, separate_native_runs=False) + for value in failed["products"].values(): value["checks"] = {} + A["closed_report"](failed) + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, caches = joined_fixture(work) + path = A["artifact"](work, "original-report"); path.unlink(); save(path, failed) + with patch.dict(A["evidence"].__globals__, cache_state=lambda key, *args: caches[key]): + with self.assertRaises(ValueError): A["evidence"](work, digest, SHA) + + def test_exact_selection_keeps_separate_native_runs_and_no_cross_product_alias(self): + value = selected(); A["validate_selection"](value) + self.assertNotEqual(value["products"]["weather"]["query"]["run"], value["products"]["air_quality"]["query"]["run"]) + mutations = [("air_quality", "query", "steps", [3, 9, 24]), ("weather", "query", "includePrecipitation", False), + ("air_quality", "publication", "name", A["PRODUCTS"]["weather"]["name"]), + ("air_quality", "publication", "manifest_id", value["products"]["weather"]["publication"]["manifest_id"])] + for key, group, field, changed in mutations: + bad = copy.deepcopy(value); bad["products"][key][group][field] = changed + with self.subTest(field=field), self.assertRaises(ValueError): A["validate_selection"](bad) + + def test_acquisition_requires_twelve_weather_and_six_native_particulate_fields(self): + good = acquisition(); A["validate_acquisition"](good) + for key in A["PRODUCTS"]: + for field in ("missing", "duplicate", "wrong_parameter", "wrong_step"): + value = copy.deepcopy(good); fields = value["products"][key]["fields"] + if field == "missing": fields.pop() + elif field == "duplicate": fields[0] = fields[1] + elif field == "wrong_parameter": fields[0]["parameter"] = "private-source-url" + else: fields[0]["lead_hours"] = 1 + with self.subTest(key=key, field=field), self.assertRaises(ValueError): A["validate_acquisition"](value) + + def test_full_join_rejects_product_specific_origin_privacy_and_ui_failures(self): + mutations = [None, + ("cold-consumer", "air_quality", ("receipt", "origin_body_bytes"), 1), + ("cold-consumer", "air_quality", ("publication", "sha256"), "f"*64), + ("cold-consumer", "weather", ("boundary", "no_new_privileges"), False), + ("cold-consumer", "air_quality", ("observed_unix_seconds",), 1900000000), + ("privacy-client", "air_quality", ("direct_client_exit_packets",), 1), + ("privacy-exit", "air_quality", ("packet_socket_drops",), 1), + ("control", "air_quality", ("unexpected_provider_application_packets",), 1), + ("original-report", None, ("products", "air_quality", "checks", "browser_api_requests"), 1), + ("original-report", None, ("products", "weather", "checks", "refresh_dom_display_sha256"), "f"*64), + ("original-report", None, ("products", "air_quality", "expected", "cold_receipt_sha256"), "f"*64), + ("original-report", None, ("source", "bridge_modules", "src/air-quality-provider.mjs"), "f"*64), + ("original-report", None, ("scope", "synthetic_air_quality"), True), + ("original-report", None, ("scope", "separate_native_runs"), False), + ("original-report", None, ("checks", "browser_api_requests"), 6), + ("original-report", None, ("checks", "refresh_dom_display_sha256"), "f"*64), + ("cold-evidence", None, ("products", "air_quality", "forecast_sha256"), "f"*64), + ("acquisition", None, ("products", "air_quality", "forecast_sha256"), "f"*64), + ("publisher-cleanup", None, ("files_removed",), False), + ("cache-after", "air_quality", ("inode",), 123)] + for mutation in mutations: + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, caches = joined_fixture(work) + if mutation: + suffix, key, keys, changed = mutation; path = A["artifact"](work, suffix, key) + value = json.loads(path.read_bytes()); target = value + for field in keys[:-1]: target = target[field] + target[keys[-1]] = changed; path.unlink(); save(path, value) + with patch.dict(A["evidence"].__globals__, cache_state=lambda key, *args: caches[key]): + if mutation: + with self.assertRaises(ValueError): A["evidence"](work, digest, SHA) + else: + proof = A["evidence"](work, digest, SHA); A["validate_evidence_export"](proof) + self.assertTrue(proof["scope"]["end_to_end_proven"]) + self.assertFalse(proof["scope"]["browser_triggered_cold_fetch"]) + self.assertEqual(proof["products"]["weather"]["acquired_fields"], 12) + self.assertEqual(proof["products"]["air_quality"]["acquired_fields"], 6) + + def test_weather_only_capture_or_receipt_never_proves_air_quality(self): + for suffix in ("cold-consumer", "selection", "privacy-client", "privacy-exit", "control", "cache-before"): + with self.subTest(suffix=suffix), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, caches = joined_fixture(work) + A["artifact"](work, suffix, "air_quality").unlink() + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda key, *args: caches[key]): + with self.assertRaises((OSError, ValueError)): A["cold_evidence"](work, digest, SHA) + + def test_cold_evidence_survives_missing_ui_without_claiming_end_to_end(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, caches = joined_fixture(work) + for suffix in ("original-report", "prepared", "tree-after", "input-cleanup"): + A["artifact"](work, suffix).unlink() + with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda key, *args: caches[key]): + result = A["cold_evidence"](work, digest, SHA) + self.assertFalse(result["scope"]["end_to_end_proven"]) + with self.assertRaises((ValueError, OSError)): A["evidence"](work, digest, SHA) + output = work / "output"; output.mkdir() + A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertTrue(A["artifact"](output, "cold-evidence").exists()) + self.assertFalse(A["artifact"](output, "evidence").exists()) + + def test_closed_exports_reject_location_and_raw_receipts_before_any_write(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); joined_fixture(work); output = work / "output"; output.mkdir() + path = A["artifact"](work, "acquisition"); value = json.loads(path.read_bytes()) + value["coordinates"] = [52.52, 13.41]; path.unlink(); save(path, value) + with self.assertRaises(ValueError): A["copy_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + value = report(); value["products"]["air_quality"]["receipt"] = {"private": "raw"} + with self.assertRaises(ValueError): A["closed_report"](value) + + def test_cleanup_validates_all_targets_before_deleting_two_product_sources(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "weather-publisher"; root.mkdir(mode=0o700) + for name in ("forecast.json", "air-quality.json", "identity.key"): + A["write_new"](root / name, b"fixture") + A["write_new"](root / "unrelated", b"keep") + with self.assertRaises(ValueError): A["cleanup"](root, True) + self.assertTrue((root / "forecast.json").exists()); self.assertTrue((root / "air-quality.json").exists()) + (root / "unrelated").unlink() + self.assertEqual(A["cleanup"](root, True), dict(directory_removed=True, files_removed=True)) + self.assertFalse(root.exists()) + + def test_cleanup_accepts_larger_two_product_expected_but_not_links(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "weather-viewer"; root.mkdir(mode=0o700) + A["write_new"](root / "expected.json", b"x" * (300*1024), 0o400) + (root / "air-quality.json").symlink_to(root / "expected.json") + with self.assertRaises(ValueError): A["cleanup"](root, False) + self.assertTrue((root / "expected.json").exists()) + (root / "air-quality.json").unlink(); A["cleanup"](root, False) + + def test_cache_metadata_is_distinct_private_and_closed(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); state = work / "state-client"; state.mkdir() + values = {} + for key, p in A["PRODUCTS"].items(): + path = state / p["cache"]; path.mkdir(mode=0o700) + values[key] = A["cache_state"](key, path, os.getuid(), os.getgid()) + self.assertNotEqual(values["weather"]["inode"], values["air_quality"]["inode"]) + A["validate_cache_export"](dict(schema=2, products=values)) + with self.assertRaises(ValueError): A["cache_state"]("air_quality", state / "weather-cache", os.getuid(), os.getgid()) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/integration/weather-client-bundle.py b/tests/integration/weather-client-bundle.py index 87be2d48e..ab1b3fbfd 100644 --- a/tests/integration/weather-client-bundle.py +++ b/tests/integration/weather-client-bundle.py @@ -51,6 +51,27 @@ FIXED = frozenset({*(WEATHER + name for name in WEATHER_FILES), *(BROWSER + name for name in BROWSER_FILES), *(STAGE + name for name in RUNTIME_FILES), *(UI_STAGE + name for name in CLIENT_FILES), "runtime/node", "decoder/grib_ls"}) +# Version 1 remains the exact frozen temperature/wind input contract. Version 2 +# selects the separate precipitation/particulate runner. Input eligibility does +# not constitute evidence that either runtime has completed a live proof. +AQ_MODULES = frozenset(("src/air-quality.mjs", "src/air-quality-provider.mjs")) +OVERLAY_FILES = frozenset(("src/lib/services/volparossa/client.mjs", + "src/lib/services/volparossa/client.d.mts", "src/lib/services/volparossa-mode.ts", + "src/routes/weather/week/VolparossaForecast.svelte", "src/routes/weather/week/VolparossaWeek.svelte", + "src/routes/weather/week/VolparossaAirQuality.svelte")) +FIXTURE_MODULES = frozenset(("tests/fixtures.mjs", "tests/air-quality-fixtures.mjs", "src/ecmwf.mjs")) +NOAA_MODULES = frozenset(("src/noaa-gefs.mjs", "src/air-quality.mjs", "src/core-cache.mjs", "scripts/air-quality.mjs")) +ECMWF_MODULES = frozenset(("src/ecmwf.mjs", "src/forecast.mjs", "src/provider.mjs", "scripts/forecast.mjs")) +WEATHER_FILES_V2 = (WEATHER_FILES - {"integrations/drizzli/browser-smoke.json"}) | AQ_MODULES | NOAA_MODULES | FIXTURE_MODULES | { + "src/air-quality-core.mjs", "third_party/noaa-gefs.json", "third_party/noaa-gefs-smoke.json", + "third_party/ecmwf-precipitation-smoke.json", + "integrations/drizzli/air-quality-browser-smoke.json", "integrations/drizzli/browser-fixture.mjs", + *("integrations/drizzli/overlay/" + name for name in OVERLAY_FILES)} +CLIENT_FILES_V2 = CLIENT_FILES | OVERLAY_FILES | {"LICENSE.VOLPAROSSA", + *("src/lib/services/volparossa/" + Path(name).name for name in AQ_MODULES)} +FIXED_V2 = frozenset({*(WEATHER + name for name in WEATHER_FILES_V2), + *(BROWSER + name for name in BROWSER_FILES), *(STAGE + name for name in RUNTIME_FILES), + *(UI_STAGE + name for name in CLIENT_FILES_V2), "runtime/node", "decoder/grib_ls"}) EXECUTABLES = frozenset({"runtime/node", "decoder/grib_ls", *(STAGE + name for name in ("firefox-esr", "firefox-bin", "crashhelper", "crashreporter", "glxtest", "vaapitest", "pingsender"))}) MAX_FILES = 512 @@ -83,10 +104,11 @@ def manifest(raw, expected): require(hex64(expected) and 0 < len(raw) <= MAX_MANIFEST and hashlib.sha256(raw).hexdigest() == expected) value = json.loads(raw, object_pairs_hook=B["unique_pairs"]) require(type(value) is dict and set(value) == {"version", "kind", "files"} - and type(value["version"]) is int and value["version"] == 1 and value["kind"] == KIND - and type(value["files"]) is dict and FIXED <= set(value["files"]) + and type(value["version"]) is int and value["version"] in (1, 2) and value["kind"] == KIND + and type(value["files"]) is dict and 1 <= len(value["files"]) <= MAX_FILES and encode(value) == raw) - require(all(name in FIXED or static_name(name) for name in value["files"]) + fixed = FIXED if value["version"] == 1 else FIXED_V2 + require(fixed <= set(value["files"]) and all(name in fixed or static_name(name) for name in value["files"]) and STATIC + "index.html" in value["files"]) total = static_bytes = 0 for name, item in value["files"].items(): @@ -144,9 +166,11 @@ def proven_hashes(value, names): return value -def source_proof(weather, browser, ui, stage, selected): +def source_proof(weather, browser, ui, stage, selected, version=1): """Check retained build provenance; do not run either project's validators.""" - proof = small_json(weather / "integrations/drizzli/browser-smoke.json") + require(type(version) is int and version in (1, 2)) + proof = small_json(weather / ("integrations/drizzli/browser-smoke.json" if version == 1 + else "integrations/drizzli/air-quality-browser-smoke.json")) upstream = small_json(weather / "integrations/drizzli/upstream.json") require(proof["passed"] is True and proof["scope"]["synthetic_weather"] is True and proof["scope"]["core_executed"] is False) @@ -159,15 +183,20 @@ def source_proof(weather, browser, ui, stage, selected): and selected[WEATHER + "integrations/drizzli/upstream.patch"]["sha256"] == upstream["upstream_patch_sha256"] == proof["source"]["patch_sha256"] and selected[WEATHER + "integrations/drizzli/smoke_browser.py"]["sha256"] == proof["source"]["driver_sha256"]) - weather_modules = proven_hashes(upstream["weather_modules"], PROVEN_WEATHER_MODULES) - require(weather_modules == proven_hashes(proof["source"]["weather_modules"], PROVEN_WEATHER_MODULES)) - for module in ("forecast", "provider"): - digest = selected[WEATHER + "src/" + module + ".mjs"]["sha256"] - require(digest == selected[UI_STAGE + "src/lib/services/volparossa/" + module + ".mjs"]["sha256"] - == weather_modules["src/" + module + ".mjs"]) + module_names = PROVEN_WEATHER_MODULES | (AQ_MODULES if version == 2 else set()) + weather_modules = proven_hashes(upstream["weather_modules"], module_names) + require(weather_modules == proven_hashes(proof["source"]["weather_modules"], module_names)) + for module, expected_sha in weather_modules.items(): + digest = selected[WEATHER + module]["sha256"] + require(digest == selected[UI_STAGE + "src/lib/services/volparossa/" + Path(module).name]["sha256"] + == expected_sha) + overlay = proof["source"]["overlay_sha256"] + if version == 2: + overlay = proven_hashes(overlay, OVERLAY_FILES) require(selected[UI_STAGE + "src/lib/services/volparossa/client.mjs"]["sha256"] - == proof["source"]["overlay_sha256"]["src/lib/services/volparossa/client.mjs"]) - for relative, digest in proven_hashes(proof["source"]["bridge_modules"], PROVEN_BRIDGE_MODULES).items(): + == overlay["src/lib/services/volparossa/client.mjs"]) + bridge_names = PROVEN_BRIDGE_MODULES | ({"src/air-quality-core.mjs"} if version == 2 else set()) + for relative, digest in proven_hashes(proof["source"]["bridge_modules"], bridge_names).items(): require(selected[WEATHER + relative]["sha256"] == digest) for relative, digest in proven_hashes(proof["source"]["browser_helpers"], PROVEN_BROWSER_HELPERS).items(): require(selected[BROWSER + relative]["sha256"] == digest) @@ -191,24 +220,78 @@ def source_proof(weather, browser, ui, stage, selected): artifacts[item["path"]] = item require(artifacts.get("build/eccodes/eccodes-build/bin/grib_ls") == dict(path="build/eccodes/eccodes-build/bin/grib_ls", bytes=31328216, sha256=DECODER_SHA)) + if version == 2: + extended_source_proof(weather, selected, proof) + + +def extended_source_proof(weather, selected, proof): + """Bind separate historical source/UI proofs, never infer joined delivery.""" + require(proof["phase"] == "complete" and proof["scenario"] == "precipitation" + and proof["air_quality_scenario"] is True and proof["wrapper_outcome"] == "completed" + and proof["scope"]["synthetic_air_quality"] is True + and proof["scope"]["actual_drizzli_ui"] is True and proof["scope"]["actual_http_bridge"] is True + and proof["scope"]["overlay_proven"] is False and proof["scope"]["upstream_origin_proven"] is False + and proof["runtime"]["node_sha256"] == NODE_SHA) + for group, names in (("cleanup", ("browser_exited", "fixture_exited", "private_profile_removed", + "normal_browser_stop", "normal_fixture_stop")), + ("outer_cleanup", ("namespace_wrapper_exited", "private_profile_removed", "host_netns_unchanged"))): + require(all(proof[group][name] is True for name in names)) + for name in ("precipitation_native_windows", "precipitation_missing_zero_small_amount", "precipitation_no_hourly_rate", + "air_quality_is_separate_product", "air_quality_native_times", "air_quality_missing_zero_concentrations", + "air_quality_not_aqi_or_observations", "actual_refresh", "no_external_resources", + "weather_failure_preserves_air_quality", "weather_expiry_preserves_air_quality", + "aq_unavailable_weather_preserved", "aq_stale_weather_preserved", "air_quality_expiry_preserves_weather"): + require(proof["checks"][name] is True) + for name in ("precipitation", "air_quality"): + initial = proof["checks"][name + "_initial_dom_sha256"] + require(hex64(initial) and initial == proof["checks"][name + "_refresh_dom_sha256"]) + for name in ("provider_requests", "air_quality_provider_requests"): + require(type(proof["checks"][name]) is int and proof["checks"][name] == 11) + for relative, digest in proven_hashes(proof["source"]["overlay_sha256"], OVERLAY_FILES).items(): + require(selected[WEATHER + "integrations/drizzli/overlay/" + relative]["sha256"] + == selected[UI_STAGE + relative]["sha256"] == digest) + for relative, digest in proven_hashes(proof["source"]["fixture_modules"], FIXTURE_MODULES).items(): + require(selected[WEATHER + relative]["sha256"] == digest) + require(selected[WEATHER + "integrations/drizzli/browser-fixture.mjs"]["sha256"] == proof["source"]["fixture_sha256"] + and selected[UI_STAGE + "LICENSE.VOLPAROSSA"]["sha256"] == selected[WEATHER + "LICENSE"]["sha256"]) + noaa = small_json(weather / "third_party/noaa-gefs-smoke.json") + require(type(noaa["version"]) is int and noaa["version"] == 1 + and noaa["kind"] == "direct-noaa-native-particulate-forecast" and noaa["success"] is True + and noaa["origin"] == "https://noaa-gefs-pds.s3.amazonaws.com" + and noaa["node_sha256"] == NODE_SHA and noaa["decoder_sha256"] == DECODER_SHA + and noaa["scope"]["actual_origin_acquisition"] is True + and noaa["scope"]["actual_pinned_native_decoder"] is True + and all(noaa["scope"][key] is False for key in ("core_executed", "overlay_proven", "browser_executed", "accuracy_proven"))) + for relative, digest in proven_hashes(noaa["source_modules"], NOAA_MODULES).items(): + require(selected[WEATHER + relative]["sha256"] == digest) + ecmwf = small_json(weather / "third_party/ecmwf-precipitation-smoke.json") + require(type(ecmwf["version"]) is int and ecmwf["version"] == 1 + and ecmwf["kind"] == "direct-ecmwf-native-precipitation-forecast" and ecmwf["success"] is True + and ecmwf["source"] == "https://data.ecmwf.int/forecasts" + and ecmwf["node_sha256"] == NODE_SHA and ecmwf["decoder_sha256"] == DECODER_SHA + and ecmwf["native_accumulation_windows_preserved"] is True and ecmwf["hourly_rates_inferred"] is False + and ecmwf["core_overlay_roundtrip_proven"] is False and ecmwf["browser_execution_proven"] is False) + for relative, digest in proven_hashes(ecmwf["source_modules"], ECMWF_MODULES).items(): + require(selected[WEATHER + relative]["sha256"] == digest) -def capture(weather, ui, browser, stage, node, archive): +def capture(weather, ui, browser, stage, node, archive, version=1): + require(type(version) is int and version in (1, 2)) for root in (weather, ui, browser, stage): B["canonical"](root) inventory(stage, exact=RUNTIME_FILES) - sources = {**{WEATHER + name: weather / name for name in WEATHER_FILES}, + sources = {**{WEATHER + name: weather / name for name in (WEATHER_FILES if version == 1 else WEATHER_FILES_V2)}, **{BROWSER + name: browser / name for name in BROWSER_FILES}, **{STAGE + name: stage / name for name in RUNTIME_FILES}, - **{UI_STAGE + name: ui / name for name in CLIENT_FILES}, + **{UI_STAGE + name: ui / name for name in (CLIENT_FILES if version == 1 else CLIENT_FILES_V2)}, **{STATIC + name: path for name, path in inventory(ui / "build").items()}, "runtime/node": node, "decoder/grib_ls": weather / "build/eccodes/eccodes-build/bin/grib_ls"} selected = {name: dict(**fingerprint(path, bound(name)), mode=mode(name)) for name, path in sorted(sources.items())} for name in EXECUTABLES: require(sources[name].stat().st_mode & 0o100) - raw = encode(dict(version=1, kind=KIND, files=selected)); expected = hashlib.sha256(raw).hexdigest() + raw = encode(dict(version=version, kind=KIND, files=selected)); expected = hashlib.sha256(raw).hexdigest() manifest(raw, expected) - source_proof(weather, browser, ui, stage, selected) + source_proof(weather, browser, ui, stage, selected, version) new_path(archive) with os.fdopen(os.open(archive, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "wb") as output: for name in ("INPUTS.json", *sorted(selected)): @@ -228,7 +311,7 @@ def capture(weather, ui, browser, stage, node, archive): return checked_archive(archive, expected) -def checked_archive(archive, expected, root=None): +def checked_archive(archive, expected, root=None, *, runtime=False, report_version=False): source, before = open_checked(archive, MAX_ARCHIVE) with source: require(before.st_size % 512 == 0) @@ -239,6 +322,9 @@ def checked_archive(archive, expected, root=None): raw = read_exact(source, item.size) require(read_exact(source, -item.size % 512) == b"\0" * (-item.size % 512)) selected = manifest(raw, expected) + # Both closed profiles have distinct joined-runtime dispatch. Preserve + # this explicit gate when future archive-only profiles are introduced. + require(not runtime or selected["version"] in (1, 2)) if root is not None: new_path(root); root.mkdir(mode=0o700) for name, record in sorted(selected["files"].items()): @@ -268,7 +354,8 @@ def checked_archive(archive, expected, root=None): for directory in sorted([root, *(p for p in root.rglob("*") if p.is_dir())], key=lambda p: len(p.parts), reverse=True): directory.chmod(0o500) return dict(manifest_sha256=expected, archive_sha256=archive_sha, files=len(selected["files"]), - bytes=sum(item["bytes"] for item in selected["files"].values())) + bytes=sum(item["bytes"] for item in selected["files"].values()), + **({"bundle_version": selected["version"]} if report_version else {})) def verify_tree(root, expected): @@ -285,8 +372,14 @@ def verify_tree(root, expected): def main(): action, *args = sys.argv[1:] - if action == "capture" and len(args) == 6: - result = capture(*map(Path, args)) + if action in ("capture", "capture-v2") and len(args) == 6: + result = capture(*map(Path, args), version=2 if action == "capture-v2" else 1) + elif action == "verify-runtime" and len(args) == 2: + result = checked_archive(Path(args[0]), args[1], runtime=True) + elif action == "runtime-version" and len(args) == 2: + # Dispatch only after checking every member and the complete archive. + # Identifying a valid input profile does not authorize its execution. + result = checked_archive(Path(args[0]), args[1], report_version=True) elif action == "unpack" and len(args) == 3: result = checked_archive(Path(args[0]), args[2], Path(args[1])) elif action in ("verify", "verify-tree") and len(args) == 2: diff --git a/tests/integration/weather-client-delivery.sh b/tests/integration/weather-client-delivery.sh index 4586505da..b22265a7a 100644 --- a/tests/integration/weather-client-delivery.sh +++ b/tests/integration/weather-client-delivery.sh @@ -96,6 +96,22 @@ weather_client_delivery_run() { if [ -z "${weather_client_bundle:-}" ] || [ -z "${weather_client_manifest_sha256:-}" ]; then fail WEATHER_CLIENT_INPUTS_MISSING fi + # Select only after the complete immutable archive has been validated. The + # legacy profile must keep its original driver and evidence contract. + weather_profile=$(python3 -B "$source_directory/tests/integration/weather-client-bundle.py" \ + runtime-version "$weather_client_bundle" "$weather_client_manifest_sha256") \ + || fail WEATHER_CLIENT_PROFILE_INVALID + weather_profile_version=$(printf '%s\n' "$weather_profile" | \ + jq -er '.bundle_version | select(type == "number" and (. == 1 or . == 2))') \ + || fail WEATHER_CLIENT_PROFILE_INVALID + case $weather_profile_version in + 1) ;; + 2) + . "$source_directory/tests/integration/weather-products-delivery.sh" + weather_products_delivery_run + return ;; + *) fail WEATHER_CLIENT_PROFILE_INVALID ;; + esac PHASE=weather-client-delivery-prepare printf '%s\n' 'Weather trial: stage verified inputs under disposable WORK; acquire one explicit public ECMWF forecast in the guest parent namespace; publish to two existing peers; cold-fetch through the protected Client route, then render and refresh the same bytes in an isolated browser. Create the empty root-owned /nonexistent only if absent in this KVM guest; remove only that owned empty directory. No HOME override or private cache permission change.' if [ "$WORKER_UID" != 985 ] || [ "$site_control_gid" = "$AGENT_GID" ]; then diff --git a/tests/integration/weather-products-delivery.py b/tests/integration/weather-products-delivery.py new file mode 100644 index 000000000..f9e97e2c3 --- /dev/null +++ b/tests/integration/weather-products-delivery.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Join two real source products to independent cold transfers and native UI. + +Version one is deliberately untouched. This closed version-two fixture shares +its process, ownership and packet-boundary checks, never its single-product +evidence. Coordinates and raw receipts stay inside the disposable guest. +""" +import datetime +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import stat +import subprocess +import sys +import time + +HERE = Path(__file__).parent +V1 = runpy.run_path(str(HERE / "weather-client-delivery.py")) +B, HTTPS = V1["B"], V1["HTTPS"] +require, hex64, same = V1["require"], V1["hex64"], V1["same"] +raw_file, read, write_new, private = (V1[k] for k in ("raw_file", "read", "write_new", "private")) +run, boundary = V1["run"], V1["boundary"] +MAX_FORECAST, MAX_REPORT = V1["MAX_FORECAST"], V1["MAX_REPORT"] +MAX_EXPECTED = 384 * 1024 +PREFIX = "weather-products-delivery" +PRODUCTS = { + "weather": dict(suffix="weather", name="disposable-public-community-weather", filename="forecast.json", + manifest="manifest.bin", published="publish-receipt.json", cache="weather-cache", source_cache="source-cache", + receipt="cold-receipt.json", script="forecast.mjs", schema="volparossa.forecast.v1", provider="ecmwf", + model="ifs", steps=[3, 9, 24], parameters=["2t", "10u", "10v", "tp"]), + "air_quality": dict(suffix="air-quality", name="disposable-public-community-air-quality", filename="air-quality.json", + manifest="air-quality-manifest.bin", published="air-quality-publish-receipt.json", cache="air-quality-cache", + source_cache="air-quality-source-cache", receipt="air-quality-cold-receipt.json", script="air-quality.mjs", + schema="volparossa.air-quality.forecast.v1", provider="noaa", model="gefs-aerosols", + steps=[3, 6, 12], parameters=["PMTF", "PMTC"]), +} +BRIDGE_MODULES = {"scripts/serve-provider.mjs", "src/provider-http.mjs", "src/provider-core.mjs", "src/core-cache.mjs", + "src/air-quality.mjs", "src/air-quality-provider.mjs", "src/air-quality-core.mjs"} +EXPORTS = ("cold-evidence", "evidence", "acquisition", "original-report", "tree-before", "tree-after", + "cache-before", "cache-after", "input-cleanup") + + +def product(key): + require(type(key) is str and key in PRODUCTS) + return PRODUCTS[key] + + +def artifact(work, suffix, key=None): + return work / f"{PREFIX}-{product(key)['suffix'] + '-' if key else ''}{suffix}.json" + + +def initialize(root): + result = V1["initialize"](root) + result["schema"] = 2 + return result + + +def fixed_query(key, model_run): + p = product(key) + result = dict(latitude=52.52, longitude=13.41, run=model_run, model=p["model"], + steps=p["steps"], communityArea=True) + if key == "weather": result.update(includeWind=True, includePrecipitation=True) + return result + + +def acquisition(source, manifest_sha, root, parent, client, group): + private(root) + require(root.name == "weather-publisher" and set(p.name for p in root.iterdir()) == {"passphrase", "scratch"}) + boundary(parent, client, group, True) + B["verify_tree"](source, manifest_sha) + require(B["manifest"](raw_file(source / "INPUTS.json", B["MAX_MANIFEST"], 0o400), manifest_sha)["version"] == 2) + deadline = time.monotonic() + 720 + require(run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10).strip() == b"ecCodes Version 2.44.0") + now = int(time.time()) + # Select independently completed cycles. NOAA's 00UTC cycle with an explicit + # twelve-hour availability margin is not + # rewritten to match IFS and its native 3/6/12h samples are not interpolated. + cycles = dict(weather=now // 43200 * 43200 - 43200, air_quality=(now - 43200) // 86400 * 86400) + products = {} + for key, p in PRODUCTS.items(): + model_run = datetime.datetime.fromtimestamp(cycles[key], datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + command = [str(source / "runtime/node"), "--max-old-space-size=256", + str(source / B["WEATHER"] / "scripts" / p["script"]), "origin", "--latitude", "52.52", "--longitude", "13.41", + "--run", model_run, "--steps", ",".join(map(str, p["steps"])), "--community-area", + "--execute", "--accept-data-license", "--decoder", str(source / "decoder/grib_ls"), + "--decoder-sha256", B["DECODER_SHA"], "--scratch-parent", str(root / "scratch")] + if key == "weather": command += ["--model", "ifs", "--include-wind", "--include-precipitation"] + remaining = deadline - time.monotonic(); require(remaining > 0) + raw = run(command, root, MAX_FORECAST, min(360, remaining)) + value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + require(value["schema"] == p["schema"] and value["source"]["provider"] == p["provider"] + and value["source"]["acquisition"] == "origin-https" and value["source"]["model"] == p["model"] + and value["source"]["model_run_at"] == model_run) + same(value["area"], dict(latitude=52.52, longitude=13.41, sharing="public-community")) + same([sample["lead_hours"] for sample in value["samples"]], p["steps"]) + require(not list((root / "scratch").iterdir())) + fields = [] + for sample in value["samples"]: + evidence = (sample["evidence"], *sample["wind_10m"]["evidence"].values(), sample["precipitation"]["evidence"]) \ + if key == "weather" else tuple(sample["evidence"].values()) + for field in evidence: + require(field["decoder_sha256"] == B["DECODER_SHA"] and field["parameter"] in p["parameters"] + and hex64(field["index_sha256"]) and hex64(field["field_sha256"])) + fields.append(dict(lead_hours=sample["lead_hours"], parameter=field["parameter"], + index_sha256=field["index_sha256"], field_sha256=field["field_sha256"])) + products[key] = dict(forecast_sha256=hashlib.sha256(raw).hexdigest(), forecast_bytes=len(raw), fields=fields) + write_new(root / p["filename"], raw) + result = dict(schema=2, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, synthetic_air_quality=False, + independent_product_runs=True, decoder_sha256=B["DECODER_SHA"], products=products) + validate_acquisition(result) + return result + + +def validate_publication(value, key): + p = product(key) + require(type(value) is dict and set(value) == {"publisher_key", "name", "revision", "manifest_id", "sha256", "bytes", "expires_unix_seconds"}) + same(value["name"], p["name"]); same(value["revision"], 1) + require(all(hex64(value[k]) for k in ("publisher_key", "manifest_id", "sha256")) + and type(value["bytes"]) is int and 0 < value["bytes"] <= MAX_FORECAST + and type(value["expires_unix_seconds"]) is int and value["expires_unix_seconds"] > 0) + + +def validate_selection(value): + require(type(value) is dict and set(value) == {"schema", "products"}) + same(value["schema"], 2); require(type(value["products"]) is dict and set(value["products"]) == set(PRODUCTS)) + for key, selected in value["products"].items(): + require(type(selected) is dict and set(selected) == {"publication", "query"}) + validate_publication(selected["publication"], key) + query = selected["query"] + same(query, fixed_query(key, query["run"])) + require(type(query["run"]) is str and re.fullmatch(r"\d{4}-\d{2}-\d{2}T(?:00|12):00:00\.000Z" if key == "weather" + else r"\d{4}-\d{2}-\d{2}T00:00:00\.000Z", query["run"])) + require(value["products"]["weather"]["publication"]["manifest_id"] != value["products"]["air_quality"]["publication"]["manifest_id"]) + return value + + +def select(root, viewer): + private(root); private(viewer) + require(root.name == "weather-publisher" and viewer.name == "weather-viewer" and not list(viewer.iterdir())) + selected = dict(schema=2, products={}) + for key, p in PRODUCTS.items(): + published = read(root / p["published"]) + raw = raw_file(root / p["filename"], MAX_FORECAST, 0o600) + forecast = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + manifest_id = hashlib.sha256(raw_file(root / p["manifest"])).hexdigest() + require(published["operation"] == "offline_content_publish" and published["network_publication"] is False + and published["bytes"] == len(raw) and published["chunks"] == 1 and hex64(published["publisher_key_hex"]) + and published["name"] == p["name"] and published["revision"] == 1 and published["content_type"] == "application/json" + and published["manifest_id"] == manifest_id and forecast["schema"] == p["schema"]) + same(forecast["area"], dict(latitude=52.52, longitude=13.41, sharing="public-community")) + selected["products"][key] = dict(publication=dict(publisher_key=published["publisher_key_hex"], name=p["name"], + revision=1, manifest_id=manifest_id, sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw), + expires_unix_seconds=published["expires_unix_seconds"]), query=fixed_query(key, forecast["source"]["model_run_at"])) + validate_selection(selected) + write_new(viewer / "selection.json", B["encode"](selected), 0o400) + return selected + + +def cold(key, binary, control, cache, viewer, parent, client, group): + p = product(key); private(viewer) + require(viewer.name == "weather-viewer" and cache.name == p["cache"]) + allowed = {"selection.json"} | ({PRODUCTS["weather"]["filename"], PRODUCTS["weather"]["receipt"]} if key == "air_quality" else set()) + require(set(path.name for path in viewer.iterdir()) == allowed) + process_boundary = boundary(parent, client, group) + publication = validate_selection(read(viewer / "selection.json", mode=0o400))["products"][key]["publication"] + output = viewer / p["filename"] + binary_before = B["fingerprint"](binary, 256*1024**2) + raw = run([str(binary), "--control-socket", str(control), "content", "fetch-name", "--publisher-key", + publication["publisher_key"], "--name", p["name"], "--min-revision", "1", "--cache", str(cache), + "--local-output", str(output), "--quota-bytes", str(MAX_FORECAST), "--max-entries", "16", "--min-free-bytes", "67108864"], + viewer, MAX_REPORT, 180) + receipt = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + V1["validate_cold_receipt"](receipt, publication, cache, output) + content = raw_file(output, MAX_FORECAST, 0o600) + require(len(content) == publication["bytes"] and hashlib.sha256(content).hexdigest() == publication["sha256"] + and time.time() < publication["expires_unix_seconds"]) + write_new(viewer / p["receipt"], raw) + same(B["fingerprint"](binary, 256*1024**2), binary_before) + return dict(schema=2, product=key, boundary=process_boundary, publication=publication, receipt=receipt, + cold_receipt_sha256=hashlib.sha256(raw).hexdigest(), core_sha256=binary_before["sha256"], + cold_cache_requested=True, observed_unix_seconds=int(time.time())) + + +def prepare(source, viewer, binary, control, weather_cache, air_quality_cache): + private(viewer) + selected = validate_selection(read(viewer / "selection.json", mode=0o400)) + core_sha = B["fingerprint"](binary, 256*1024**2)["sha256"] + def core_config(key, cache): + publication = selected["products"][key]["publication"] + require(cache.name == product(key)["cache"]) + return dict(mode="core-cache", executable=str(binary), controlSocket=str(control), cache=str(cache), + scratchParent=str(viewer / "browser-output/scratch"), publisherKey=publication["publisher_key"], + name=publication["name"], minRevision=1, query=selected["products"][key]["query"], reuseCache=True) + config = dict(core=core_config("weather", weather_cache), variables=V1["VARIABLES"], + airQuality=dict(core=core_config("air_quality", air_quality_cache), variables=["pm2_5", "pm10"])) + write_new(viewer / "config.json", B["encode"](config)) + raw = run([str(source / "runtime/node"), "--max-old-space-size=256", + str(source / B["WEATHER"] / "integrations/drizzli/prepare-core-browser.mjs"), + "--forecast", str(viewer / "forecast.json"), "--receipt", str(viewer / "cold-receipt.json"), + "--air-quality", str(viewer / "air-quality.json"), "--air-quality-receipt", str(viewer / "air-quality-cold-receipt.json"), + "--config", str(viewer / "config.json"), "--core-sha256", core_sha, "--output", str(viewer / "expected.json")], viewer, 4096, 30) + result = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + same(set(result), {"prepared", "expected_sha256", "display_sha256"}) + require(result["prepared"] is True and hex64(result["expected_sha256"]) and hex64(result["display_sha256"])) + expected_raw = raw_file(viewer / "expected.json", MAX_EXPECTED, 0o400) + expected = json.loads(expected_raw, object_pairs_hook=B["B"]["unique_pairs"]) + require(set(expected) == {"schema", "core_sha256", "products", "display_sha256"} and expected["schema"] == 2 + and expected["core_sha256"] == core_sha and set(expected["products"]) == set(PRODUCTS) + and hashlib.sha256(expected_raw).hexdigest() == result["expected_sha256"] + and expected["display_sha256"] == result["display_sha256"]) + for key in PRODUCTS: same(expected["products"][key]["publication"], selected["products"][key]["publication"]) + return result + + +def cache_state(key, path, uid, gid): + require(path.name == product(key)["cache"] and path.parent.name == "state-client" and path.resolve(strict=True) == path) + info = path.lstat() + require(stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o700 and info.st_uid == uid and info.st_gid == gid) + return dict(device=info.st_dev, inode=info.st_ino, mode="0700", user_uid=uid, group_gid=gid) + + +def cache_summary(work, phase): + require(phase in ("before", "after")) + result = dict(schema=2, products={key: read(artifact(work, "cache-" + phase, key)) for key in PRODUCTS}) + validate_cache_export(result) + return result + + +def cleanup(root, publisher): + require(root.name == ("weather-publisher" if publisher else "weather-viewer") and not root.is_symlink()) + if not root.exists(): return dict(directory_removed=True, files_removed=True) + private(root) + allowed = {"passphrase", "identity.key", "scratch"} if publisher else {"selection.json", "config.json", "expected.json", "browser-output"} + allowed |= {p[field] for p in PRODUCTS.values() for field in + (("filename", "manifest", "published", "source_cache") if publisher else ("filename", "receipt"))} + files, directories, total = [], [], 0 + for path in root.iterdir(): + require(path.name in allowed) + if path.is_dir(): + private(path); directories.append(path) + children = list(path.iterdir()); require(len(children) <= 20) + for child in children: + require(child.name == "report.json" if path.name == "browser-output" else + path.name in {p["source_cache"] for p in PRODUCTS.values()} and + (re.fullmatch(r"[a-f0-9]{64}", child.name) or child.name in (".volparossa-owner-v1", ".volparossa-index-v1"))) + total += len(raw_file(child, MAX_FORECAST)); files.append(child) + else: + total += len(raw_file(path, MAX_EXPECTED)); files.append(path) + require(total <= 4*1024*1024) + for path in files: path.unlink() + for path in directories: path.rmdir() + root.rmdir() + return dict(directory_removed=True, files_removed=True) + + +def validate_acquisition(value): + same({k: v for k, v in value.items() if k != "products"}, dict(schema=2, success=True, origin_https=True, + acquisition_outside_overlay=True, upstream_signature_attested=False, synthetic_weather=False, + synthetic_air_quality=False, independent_product_runs=True, decoder_sha256=B["DECODER_SHA"])) + require(type(value["products"]) is dict and set(value["products"]) == set(PRODUCTS)) + for key, item in value["products"].items(): + p = product(key) + require(type(item) is dict and set(item) == {"forecast_sha256", "forecast_bytes", "fields"} + and hex64(item["forecast_sha256"]) and type(item["forecast_bytes"]) is int and 0 < item["forecast_bytes"] <= MAX_FORECAST + and type(item["fields"]) is list and len(item["fields"]) == len(p["steps"]) * len(p["parameters"])) + coverage = [] + for field in item["fields"]: + require(set(field) == {"lead_hours", "parameter", "index_sha256", "field_sha256"} + and type(field["lead_hours"]) is int and field["lead_hours"] in p["steps"] + and field["parameter"] in p["parameters"] and hex64(field["index_sha256"]) and hex64(field["field_sha256"])) + coverage.append((field["lead_hours"], field["parameter"])) + require(set(coverage) == {(step, parameter) for step in p["steps"] for parameter in p["parameters"]}) + + +def closed_report(report): + require(type(report) is dict and set(report) <= {"schema", "passed", "phase", "source", "expected", "products", + "scope", "checks", "cleanup", "outer_cleanup", "failure"}) + same(report["schema"], 2); require(type(report["passed"]) is bool) + require(report["phase"] in {"inputs", "sandbox", "bridge_start", "provider", "browser_start", "hydration", "refresh", "shutdown", "complete"}) + if "failure" in report: require(report["failure"] in ("interrupted", "check_or_runtime_failed")) + source = report["source"] + require(set(source) == {"bundle_manifest_sha256", "driver_sha256", "preparer_sha256", "built_tree_sha256", "node_sha256", "bridge_modules"} + and all(hex64(v) for k, v in source.items() if k != "bridge_modules") + and set(source["bridge_modules"]) == BRIDGE_MODULES and all(hex64(v) for v in source["bridge_modules"].values())) + require(set(report["expected"]) == {"expected_sha256", "core_sha256", "display_sha256"} + and all(hex64(v) or k == "expected_sha256" and v is None for k, v in report["expected"].items())) + require(set(report["products"]) == set(PRODUCTS)) + for item in report["products"].values(): + require(set(item) == {"expected", "checks"} and set(item["expected"]) == + {"cold_receipt_sha256", "display_sha256", "forecast_sha256", "manifest_id"} + and all(hex64(v) for v in item["expected"].values())) + allowed = {"api_response_sha256", "initial_dom_display_sha256", "refresh_dom_display_sha256", + "initial_dom_matches_cold_publication", "refresh_dom_matches_cold_publication", "browser_api_requests", + "provider_requests_checked", "exact_native_times_and_values", "explicit_missing_capabilities", "no_external_resources", "source_attribution"} + require(set(item["checks"]) <= allowed) + for key, v in item["checks"].items(): + require(hex64(v) if key.endswith("sha256") else type(v) is int and 0 <= v <= 2 + if key in {"browser_api_requests", "provider_requests_checked"} else type(v) is bool) + require(set(report["scope"]) == {"actual_drizzli_ui", "actual_http_bridge", "real_core_adapter", "synthetic_weather", + "synthetic_air_quality", "separate_native_runs", "standalone_origin_proven", "standalone_overlay_proven", "browser_triggered_cold_fetch"} + and all(type(v) is bool for v in report["scope"].values())) + require(set(report["checks"]) <= {"no_external_resources", "provider_requests_checked", "browser_api_requests", + "initial_dom_display_sha256", "refresh_dom_display_sha256"}) + for key, v in report["checks"].items(): + require(hex64(v) if key.endswith("sha256") else type(v) is int and 0 <= v <= 4 + if key in {"provider_requests_checked", "browser_api_requests"} else type(v) is bool) + for name, allowed in (("cleanup", {"browser_exited", "bridge_exited", "private_state_removed", "normal_browser_stop", "normal_bridge_stop"}), + ("outer_cleanup", {"namespace_wrapper_exited", "private_state_removed", "client_namespace_unchanged"})): + if name in report: require(set(report[name]) <= allowed and all(type(v) is bool for v in report[name].values())) + return report + + +def retain(work): + path = work / "client-fixtures/weather-viewer/browser-output/report.json" + if not path.exists(): return dict(retained=False) + raw = V1["worker_report"](path) + closed_report(json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"])) + write_new(artifact(work, "original-report"), raw) + return dict(retained=True, sha256=hashlib.sha256(raw).hexdigest()) + + +def validate_publications(work, publication, nodes, key): + p = product(key); published = read(artifact(work, "publish", key)) + require(published["operation"] == "offline_content_publish" and published["network_publication"] is False + and published["publisher_key_hex"] == publication["publisher_key"] and published["content_type"] == "application/json" + and published["chunks"] == 1 and all(published[k] == publication[k] for k in + ("name", "revision", "bytes", "manifest_id", "expires_unix_seconds"))) + for node in nodes: + imported = read(artifact(work, node + "-import", key)); served = read(artifact(work, node + "-serve", key)) + require(imported["operation"] == "content_import" and imported["complete"] is True + and imported["manifest_id"] == publication["manifest_id"] and imported["content_bytes"] == publication["bytes"] + and imported["agent_cache"] == str(work / f"state-{node}" / p["cache"]) and imported["public_content"] is True + and all(imported[k] is False for k in ("ownership_changed", "network_transfer", "origin_authenticated")) + and served["serving"] is True and type(served["publications"]) is int + and served["publications"] == (4 if key == "weather" else 5) and served["replication_enabled"] is False) + + +def validate_cold_phase(work, manifest_sha, binary_sha): + require(hex64(binary_sha)) + selected = validate_selection(read(artifact(work, "input"))) + acquired = read(artifact(work, "acquisition")); validate_acquisition(acquired) + peers, layout = read(work / "a01-expected-peers.json"), read(work / "content-provider-layout.json") + control_gid = (work / "runtime-client/control/agent.sock").stat().st_gid + colds, caches = {}, {} + for key, p in PRODUCTS.items(): + publication = selected["products"][key]["publication"]; acquisition_item = acquired["products"][key] + require(acquisition_item["forecast_sha256"] == publication["sha256"] and acquisition_item["forecast_bytes"] == publication["bytes"]) + cold_result = read(artifact(work, "cold-consumer", key)); colds[key] = cold_result + same(cold_result["publication"], publication); same(cold_result["schema"], 2); same(cold_result["product"], key) + V1["validate_cold_receipt"](cold_result["receipt"], publication, work / "state-client" / p["cache"], + work / "client-fixtures/weather-viewer" / p["filename"]) + same(cold_result["cold_cache_requested"], True) + require(cold_result["core_sha256"] == binary_sha and hex64(cold_result["cold_receipt_sha256"]) + and type(cold_result["observed_unix_seconds"]) is int and 0 < cold_result["observed_unix_seconds"] < publication["expires_unix_seconds"]) + same(cold_result["boundary"], dict(user_uid=985, primary_control_gid=control_gid, supplementary_groups_empty=True, + all_capabilities_dropped=True, no_new_privileges=True, client_namespace=True, acquisition_parent_namespace=False)) + validate_publications(work, publication, layout["provider_nodes"], key) + V1["validate_packets"](cold_result, peers, layout, read(artifact(work, "selection", key)), + {role: read(artifact(work, "privacy-" + role, key)) for role in HTTPS["ROLES"]}, read(artifact(work, "control", key))) + before = read(artifact(work, "cache-before", key)); caches[key] = before + same(before, cache_state(key, work / "state-client" / p["cache"], before["user_uid"], before["group_gid"])) + require(before["user_uid"] != 985 and control_gid != before["group_gid"]) + same(read(artifact(work, "publisher-cleanup")), dict(directory_removed=True, files_removed=True)) + publisher = work / "client-fixtures/weather-publisher" + require(not publisher.exists() and not publisher.is_symlink()) + manifest = B["manifest"](raw_file(artifact(work, "manifest"), B["MAX_MANIFEST"], 0o400), manifest_sha) + same(manifest["version"], 2) + same(read(artifact(work, "tree-before")), dict(version=1, action="verify-tree", manifest_sha256=manifest_sha, + files=len(manifest["files"]), runtime_executed=False)) + return selected, acquired, colds, manifest, caches + + +def cold_summary(selected, acquired, colds, manifest_sha, binary_sha): + products = {} + for key in PRODUCTS: + publication = selected["products"][key]["publication"]; cold = colds[key] + products[key] = dict(forecast_sha256=publication["sha256"], forecast_bytes=publication["bytes"], + manifest_id=publication["manifest_id"], cold_receipt_sha256=cold["cold_receipt_sha256"], + acquired_fields=len(acquired["products"][key]["fields"]), protected_cold=dict(peer_bytes=cold["receipt"]["peer_bytes"], + providers_used=cold["receipt"]["providers_used"], origin_body_bytes=0, origin_range_requests=0, privacy_captures_validated=True)) + result = dict(schema=2, success=True, kind="volparossa-weather-products-protected-cold", bundle_manifest_sha256=manifest_sha, + core_sha256=binary_sha, decoder_sha256=B["DECODER_SHA"], products=products, + scope=dict(actual_acquisition=True, acquisition_outside_overlay=True, same_publications_joined=True, + independent_product_captures=True, protected_cold_transfer=True, actual_drizzli_ui=False, actual_http_bridge=False, + actual_ui_refresh=False, end_to_end_proven=False, browser_triggered_cold_fetch=False, + two_subflow_bulk_aggregation=False, upstream_signature_attested=False, production_service_complete=False), + cleanup=dict(publisher_sources_removed_before_fetch=True)) + validate_cold_export(result) + return result + + +def validate_cold_export(value): + require(set(value) == {"schema", "success", "kind", "bundle_manifest_sha256", "core_sha256", "decoder_sha256", "products", "scope", "cleanup"}) + same(value["schema"], 2); same(value["success"], True); same(value["kind"], "volparossa-weather-products-protected-cold") + require(all(hex64(value[k]) for k in ("bundle_manifest_sha256", "core_sha256", "decoder_sha256")) + and value["decoder_sha256"] == B["DECODER_SHA"] and set(value["products"]) == set(PRODUCTS)) + for key, item in value["products"].items(): + require(set(item) == {"forecast_sha256", "forecast_bytes", "manifest_id", "cold_receipt_sha256", "acquired_fields", "protected_cold"} + and all(hex64(item[k]) for k in ("forecast_sha256", "manifest_id", "cold_receipt_sha256")) + and type(item["forecast_bytes"]) is int and 0 < item["forecast_bytes"] <= MAX_FORECAST) + same(item["acquired_fields"], 12 if key == "weather" else 6) + transfer = item["protected_cold"] + require(type(transfer["providers_used"]) is int and transfer["providers_used"] in (1, 2)) + same(transfer, dict(peer_bytes=item["forecast_bytes"], providers_used=transfer["providers_used"], + origin_body_bytes=0, origin_range_requests=0, privacy_captures_validated=True)) + same(value["scope"], dict(actual_acquisition=True, acquisition_outside_overlay=True, same_publications_joined=True, + independent_product_captures=True, protected_cold_transfer=True, actual_drizzli_ui=False, actual_http_bridge=False, + actual_ui_refresh=False, end_to_end_proven=False, browser_triggered_cold_fetch=False, + two_subflow_bulk_aggregation=False, upstream_signature_attested=False, production_service_complete=False)) + same(value["cleanup"], dict(publisher_sources_removed_before_fetch=True)) + + +def cold_evidence(work, manifest_sha, binary_sha): + selected, acquired, colds, _manifest, _before = validate_cold_phase(work, manifest_sha, binary_sha) + return cold_summary(selected, acquired, colds, manifest_sha, binary_sha) + + +def evidence(work, manifest_sha, binary_sha): + selected, acquired, colds, manifest, before = validate_cold_phase(work, manifest_sha, binary_sha) + retained = read(artifact(work, "cold-evidence")); validate_cold_export(retained) + same(retained, cold_summary(selected, acquired, colds, manifest_sha, binary_sha)) + same(read(artifact(work, "input-cleanup")), dict(directory_removed=True, files_removed=True)) + require(all(not path.exists() and not path.is_symlink() for path in + (work / "client-fixtures/weather-publisher", work / "client-fixtures/weather-viewer"))) + for key, p in PRODUCTS.items(): + after = read(artifact(work, "cache-after", key)); same(before[key], after) + same(after, cache_state(key, work / "state-client" / p["cache"], after["user_uid"], after["group_gid"])) + for phase in ("before", "after"): + same(read(artifact(work, "tree-" + phase)), dict(version=1, action="verify-tree", manifest_sha256=manifest_sha, + files=len(manifest["files"]), runtime_executed=False)) + report = closed_report(read(artifact(work, "original-report"), 16384)) + same(read(artifact(work, "prepared")), dict(prepared=True, + expected_sha256=report["expected"]["expected_sha256"], display_sha256=report["expected"]["display_sha256"])) + require(report["passed"] is True and report["phase"] == "complete" and "failure" not in report) + same(report["scope"], dict(actual_drizzli_ui=True, actual_http_bridge=True, real_core_adapter=True, + synthetic_weather=False, synthetic_air_quality=False, separate_native_runs=True, + standalone_origin_proven=False, standalone_overlay_proven=False, browser_triggered_cold_fetch=False)) + require(report["source"]["bundle_manifest_sha256"] == manifest_sha and report["expected"]["core_sha256"] == binary_sha) + for field, relative in (("driver_sha256", "integrations/drizzli/smoke_core_browser.py"), + ("preparer_sha256", "integrations/drizzli/prepare-core-browser.mjs")): + require(report["source"][field] == manifest["files"][B["WEATHER"] + relative]["sha256"]) + for relative, digest in report["source"]["bridge_modules"].items(): require(digest == manifest["files"][B["WEATHER"] + relative]["sha256"]) + require(report["source"]["node_sha256"] == B["NODE_SHA"]) + static = {name[len(B["STATIC"]):]: item["sha256"] for name, item in manifest["files"].items() if name.startswith(B["STATIC"])} + require(report["source"]["built_tree_sha256"] == hashlib.sha256(json.dumps(static, sort_keys=True, separators=(",", ":")).encode()).hexdigest()) + for key in PRODUCTS: + item = report["products"][key]; publication = selected["products"][key]["publication"] + require(item["expected"]["cold_receipt_sha256"] == colds[key]["cold_receipt_sha256"] + and item["expected"]["forecast_sha256"] == publication["sha256"] + and item["expected"]["manifest_id"] == publication["manifest_id"]) + same(item["checks"], dict(api_response_sha256=item["checks"]["api_response_sha256"], + initial_dom_display_sha256=item["expected"]["display_sha256"], refresh_dom_display_sha256=item["expected"]["display_sha256"], + initial_dom_matches_cold_publication=True, refresh_dom_matches_cold_publication=True, browser_api_requests=2, + provider_requests_checked=1, exact_native_times_and_values=True, explicit_missing_capabilities=True, + no_external_resources=True, source_attribution=True)) + same(report["checks"], dict(no_external_resources=True, provider_requests_checked=2, browser_api_requests=4, + initial_dom_display_sha256=report["expected"]["display_sha256"], refresh_dom_display_sha256=report["expected"]["display_sha256"])) + same(report["cleanup"], dict(browser_exited=True, bridge_exited=True, private_state_removed=True, normal_browser_stop=True, normal_bridge_stop=True)) + same(report["outer_cleanup"], dict(namespace_wrapper_exited=True, private_state_removed=True, client_namespace_unchanged=True)) + result = dict(schema=2, success=True, kind="volparossa-weather-products-delivery", bundle_manifest_sha256=manifest_sha, + core_sha256=binary_sha, decoder_sha256=B["DECODER_SHA"], products=retained["products"], + browser_report_sha256=hashlib.sha256(raw_file(artifact(work, "original-report"))).hexdigest(), + display_sha256=report["expected"]["display_sha256"], + scope=dict(actual_acquisition=True, same_publications_joined=True, independent_product_captures=True, + actual_drizzli_ui=True, actual_http_bridge=True, actual_ui_refresh=True, end_to_end_proven=True, + browser_triggered_cold_fetch=False, browser_cache_only_proven=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False), + cleanup=dict(publisher_sources_removed_before_fetch=True, browser_private_state_removed=True, + viewer_removed=True, agent_cache_identity_preserved=True, agent_cache_private=True)) + validate_evidence_export(result) + return result + + +def validate_evidence_export(value): + require(set(value) == {"schema", "success", "kind", "bundle_manifest_sha256", "core_sha256", "decoder_sha256", "products", + "browser_report_sha256", "display_sha256", "scope", "cleanup"}) + same(value["kind"], "volparossa-weather-products-delivery") + require(hex64(value["browser_report_sha256"]) and hex64(value["display_sha256"])) + cold = {k: v for k, v in value.items() if k not in {"browser_report_sha256", "display_sha256", "scope", "cleanup"}} + cold.update(kind="volparossa-weather-products-protected-cold", scope=dict(actual_acquisition=True, + acquisition_outside_overlay=True, same_publications_joined=True, independent_product_captures=True, + protected_cold_transfer=True, actual_drizzli_ui=False, actual_http_bridge=False, actual_ui_refresh=False, + end_to_end_proven=False, browser_triggered_cold_fetch=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False), cleanup=dict(publisher_sources_removed_before_fetch=True)) + validate_cold_export(cold) + same(value["scope"], dict(actual_acquisition=True, same_publications_joined=True, independent_product_captures=True, + actual_drizzli_ui=True, actual_http_bridge=True, actual_ui_refresh=True, end_to_end_proven=True, + browser_triggered_cold_fetch=False, browser_cache_only_proven=False, two_subflow_bulk_aggregation=False, + upstream_signature_attested=False, production_service_complete=False)) + same(value["cleanup"], dict(publisher_sources_removed_before_fetch=True, browser_private_state_removed=True, + viewer_removed=True, agent_cache_identity_preserved=True, agent_cache_private=True)) + + +def validate_cache_export(value): + require(set(value) == {"schema", "products"}); same(value["schema"], 2) + require(set(value["products"]) == set(PRODUCTS)) + for item in value["products"].values(): + require(set(item) == {"device", "inode", "mode", "user_uid", "group_gid"}) + same(item["mode"], "0700") + require(all(type(v) is int and 0 <= v < 2**64 for k, v in item.items() if k != "mode")) + + +def validate_export(suffix, value): + if suffix == "original-report": closed_report(value) + elif suffix == "cold-evidence": validate_cold_export(value) + elif suffix == "evidence": validate_evidence_export(value) + elif suffix == "acquisition": validate_acquisition(value) + elif suffix in {"cache-before", "cache-after"}: validate_cache_export(value) + elif suffix == "input-cleanup": same(value, dict(directory_removed=True, files_removed=True)) + elif suffix in {"tree-before", "tree-after"}: + require(set(value) == {"version", "action", "manifest_sha256", "files", "runtime_executed"}) + same(value["version"], 1); same(value["action"], "verify-tree"); same(value["runtime_executed"], False) + require(hex64(value["manifest_sha256"]) and type(value["files"]) is int and 0 < value["files"] <= 2048) + else: raise ValueError("weather_products_export_rejected") + + +def copy_artifacts(work, output, uid, gid, only_cold=False): + if only_cold: require(artifact(work, "cold-evidence").exists()) + pending, count = [], 0 + for suffix in (("cold-evidence",) if only_cold else EXPORTS): + source = artifact(work, suffix) + if not source.exists(): continue + raw = raw_file(source, MAX_REPORT); value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + validate_export(suffix, value) + target = output / source.name + if target.exists() or target.is_symlink(): + require(target.resolve(strict=True) == target) + with os.fdopen(os.open(target, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as existing: + info = os.fstat(existing.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == uid and info.st_gid == gid + and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size == len(raw)) + require(existing.read(MAX_REPORT+1) == raw and B["identity"](os.fstat(existing.fileno())) == B["identity"](info)) + else: + B["new_path"](target); pending.append((target, raw)) + count += 1 + for target, raw in pending: + write_new(target, raw); os.chown(target, uid, gid) + return dict(files=count, raw_logs_exported=False) + + +def main(): + action, *args = sys.argv[1:] + if action == "init" and len(args) == 1: result = initialize(Path(args[0])) + elif action == "acquire" and len(args) == 6: result = acquisition(Path(args[0]), args[1], Path(args[2]), args[3], args[4], int(args[5])) + elif action == "select" and len(args) == 2: result = select(*map(Path, args)) + elif action == "cold" and len(args) == 8: result = cold(args[0], *map(Path, args[1:5]), args[5], args[6], int(args[7])) + elif action == "prepare" and len(args) == 6: result = prepare(*map(Path, args)) + elif action in {"publisher-cleanup", "user-cleanup"} and len(args) == 1: result = cleanup(Path(args[0]), action == "publisher-cleanup") + elif action == "cache-state" and len(args) == 4: result = cache_state(args[0], Path(args[1]), int(args[2]), int(args[3])) + elif action == "cache-summary" and len(args) == 2: result = cache_summary(Path(args[0]), args[1]) + elif action == "retain" and len(args) == 1: result = retain(Path(args[0])) + elif action == "cold-evidence" and len(args) == 3: result = cold_evidence(Path(args[0]), args[1], args[2]) + elif action == "evidence" and len(args) == 3: result = evidence(Path(args[0]), args[1], args[2]) + elif action in {"copy-cold-evidence", "copy-artifacts"} and len(args) == 4: + result = copy_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3]), only_cold=action == "copy-cold-evidence") + else: raise ValueError("weather_products_delivery_rejected") + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + + +if __name__ == "__main__": + try: main() + except (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError): + print('{"success":false,"failure":"weather_products_delivery_rejected"}') + raise SystemExit(1) diff --git a/tests/integration/weather-products-delivery.sh b/tests/integration/weather-products-delivery.sh new file mode 100644 index 000000000..4dd41be4f --- /dev/null +++ b/tests/integration/weather-products-delivery.sh @@ -0,0 +1,282 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only +# Version 2 only: two independently published and cold-fetched public products. +# Sourced after the enclosing runner verifies the selected input archive. +# Reuses the v1 guest/home/CLI utilities; sourcing this file performs no I/O. +# shellcheck disable=SC2154,SC2034 + +WEATHER_PRODUCTS_DEADLINE= + +weather_products_remaining() { + weather_requested_seconds=$1 + if [ -z "$WEATHER_PRODUCTS_DEADLINE" ]; then + printf '%s\n' "$weather_requested_seconds" + return 0 + fi + weather_seconds_left=$((WEATHER_PRODUCTS_DEADLINE - $(date +%s))) + [ "$weather_seconds_left" -gt 0 ] || return 1 + if [ "$weather_seconds_left" -lt "$weather_requested_seconds" ]; then + printf '%s\n' "$weather_seconds_left" + else + printf '%s\n' "$weather_requested_seconds" + fi +} + +weather_products_private() { + setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-products-delivery.py" "$@" +} + +weather_products_cli() { + weather_socket_node=$1 + shift + weather_command_limit=$(weather_products_remaining 120) || return 1 + timeout --signal=TERM --kill-after=5s "${weather_command_limit}s" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- "$binary_directory/volparossa" \ + --control-socket "$WORK/runtime-$weather_socket_node/control/agent.sock" "$@" +} + +weather_products_select() { + case $1 in + weather) + weather_product=weather + weather_product_path=weather + weather_product_file=forecast.json + weather_product_manifest=manifest.bin + weather_product_receipt=publish-receipt.json + weather_product_source_cache=source-cache + weather_product_name=disposable-public-community-weather + weather_publications=4 ;; + air_quality) + weather_product=air_quality + weather_product_path=air-quality + weather_product_file=air-quality.json + weather_product_manifest=air-quality-manifest.bin + weather_product_receipt=air-quality-publish-receipt.json + weather_product_source_cache=air-quality-source-cache + weather_product_name=disposable-public-community-air-quality + weather_publications=5 ;; + *) return 1 ;; + esac + weather_product_prefix=weather-products-delivery-$weather_product_path + weather_product_cache=$WORK/state-client/$weather_product_path-cache +} + +# These replace the v1 trap/export entrypoints only after verified v2 dispatch. +weather_client_delivery_cleanup() { + weather_cleanup_status=0 + if [ -f "$WORK/bin/weather-products-delivery.py" ] && [ -n "${site_control_gid:-}" ]; then + if [ ! -e "$WORK/weather-products-delivery-original-report.json" ] \ + && [ ! -L "$WORK/weather-products-delivery-original-report.json" ]; then + if [ -e "$WORK/client-fixtures/weather-viewer/browser-output/report.json" ] \ + || [ -L "$WORK/client-fixtures/weather-viewer/browser-output/report.json" ]; then + python3 -B "$WORK/bin/weather-products-delivery.py" retain "$WORK" \ + >"$WORK/weather-products-delivery-retained.json" || weather_cleanup_status=1 + fi + fi + weather_products_private publisher-cleanup "$WORK/client-fixtures/weather-publisher" \ + >"$WORK/weather-products-delivery-publisher-cleanup.json" || weather_cleanup_status=1 + weather_products_private user-cleanup "$WORK/client-fixtures/weather-viewer" \ + >"$WORK/weather-products-delivery-input-cleanup.json" || weather_cleanup_status=1 + fi + weather_client_home_cleanup || weather_cleanup_status=1 + return "$weather_cleanup_status" +} + +weather_client_delivery_copy_artifacts() { + [ -f "$WORK/bin/weather-products-delivery.py" ] || return 0 + python3 -B "$WORK/bin/weather-products-delivery.py" copy-artifacts "$WORK" \ + "$output_directory" "$OUTPUT_UID" "$OUTPUT_GID" >/dev/null +} + +weather_products_delivery_run() { + [ -n "$weather_client_bundle" ] && [ -n "$weather_client_manifest_sha256" ] \ + || fail WEATHER_PRODUCTS_INPUTS_MISSING + PHASE=weather-products-delivery-prepare + printf '%s\n' 'Weather v2 trial: acquire public ECMWF precipitation/weather and NOAA particulate fields in the disposable guest; publish separately to two existing peers, remove both originals and publisher identity, cold-fetch each product with independent captures, then check both products and refresh in the real isolated client. No host network or user-data changes.' + [ "$WORKER_UID" = 985 ] && [ "$site_control_gid" != "$AGENT_GID" ] \ + || fail WEATHER_PRODUCTS_USER_INVALID + weather_client_directory=$WORK/client-fixtures/weather-client-source + weather_publisher=$WORK/client-fixtures/weather-publisher + weather_viewer=$WORK/client-fixtures/weather-viewer + for weather_new in "$weather_client_directory" "$weather_publisher" "$weather_viewer" \ + "$WORK/state-client/weather-cache" "$WORK/state-client/air-quality-cache" \ + "$WORK/state-$provider_node_a/weather-cache" "$WORK/state-$provider_node_b/weather-cache" \ + "$WORK/state-$provider_node_a/air-quality-cache" "$WORK/state-$provider_node_b/air-quality-cache"; do + if [ -e "$weather_new" ] || [ -L "$weather_new" ]; then fail WEATHER_PRODUCTS_PATH_EXISTS; fi + done + for weather_script in weather-products-delivery.py weather-client-delivery.py weather-client-bundle.py \ + browser-filter-bundle.py content-provider-https-smoke.py content-network-smoke.py; do + install -o root -g root -m 0555 "$source_directory/tests/integration/$weather_script" \ + "$WORK/bin/$weather_script" || fail WEATHER_PRODUCTS_DRIVER_UNAVAILABLE + done + install -o root -g root -m 0600 "$weather_client_bundle" "$WORK/weather-client-inputs.tar" \ + || fail WEATHER_PRODUCTS_ARCHIVE_COPY_FAILED + python3 -B "$WORK/bin/weather-client-bundle.py" unpack "$WORK/weather-client-inputs.tar" \ + "$weather_client_directory" "$weather_client_manifest_sha256" \ + >"$WORK/weather-products-delivery-unpack.json" || fail WEATHER_PRODUCTS_UNPACK_FAILED + install -o root -g root -m 0400 "$weather_client_directory/INPUTS.json" \ + "$WORK/weather-products-delivery-manifest.json" || fail WEATHER_PRODUCTS_MANIFEST_RETENTION_FAILED + chown -R "$WORKER_UID:$site_control_gid" "$weather_client_directory" || fail WEATHER_PRODUCTS_OWNER_FAILED + install -d -o "$WORKER_UID" -g "$site_control_gid" -m 0700 "$weather_publisher" "$weather_viewer" \ + || fail WEATHER_PRODUCTS_PRIVATE_DIRECTORY_FAILED + weather_client_verify_tree >"$WORK/weather-products-delivery-tree-before.json" || fail WEATHER_PRODUCTS_SOURCE_CHANGED + weather_client_home_prepare || fail WEATHER_PRODUCTS_HOME_UNSAFE + weather_core_sha256=$(sha256sum "$binary_directory/volparossa" | awk '{print $1}') + weather_products_private init "$weather_publisher" >"$WORK/weather-products-delivery-assets.json" \ + || fail WEATHER_PRODUCTS_INPUT_FAILED + PHASE=weather-products-delivery-acquire + # Both bounded source acquisitions precede publication; they cannot consume + # the publication's lifetime. No origin retry is delegated to the browser. + # The helper owns separately-sessioned decoder children. Allow its bounded + # TERM/KILL/reap cleanup (at most 20s) to finish before the outer hard kill. + timeout --signal=TERM --kill-after=30s 740s setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-products-delivery.py" acquire \ + "$weather_client_directory" "$weather_client_manifest_sha256" "$weather_publisher" \ + "$site_parent_ns" "$site_client_ns" "$site_control_gid" \ + >"$WORK/weather-products-delivery-acquisition.json" 2>"$WORK/weather-products-delivery-acquisition.err" \ + || fail WEATHER_PRODUCTS_ACQUISITION_FAILED + # Budget all later commands together, not 900 seconds per operation. Each + # immutable publication still independently enforces its original expiry. + WEATHER_PRODUCTS_DEADLINE=$(($(date +%s) + 840)) + PHASE=weather-products-delivery-publish + weather_products_cli "$provider_node_a" init --identity "$weather_publisher/identity.key" \ + --passphrase-file "$weather_publisher/passphrase" >"$WORK/weather-products-delivery-init.log" \ + 2>"$WORK/weather-products-delivery-init.err" || fail WEATHER_PRODUCTS_IDENTITY_FAILED + weather_key= + for weather_kind in weather air_quality; do + weather_products_select "$weather_kind" || fail WEATHER_PRODUCTS_KIND_INVALID + weather_products_cli "$provider_node_a" content publish --input "$weather_publisher/$weather_product_file" \ + --name "$weather_product_name" --revision 1 --content-type application/json \ + --lifetime-seconds 900 --identity "$weather_publisher/identity.key" \ + --passphrase-file "$weather_publisher/passphrase" --cache "$weather_publisher/$weather_product_source_cache" \ + --manifest "$weather_publisher/$weather_product_manifest" >"$WORK/$weather_product_prefix-publish.json" \ + 2>"$WORK/$weather_product_prefix-publish.err" || fail WEATHER_PRODUCTS_PUBLISH_FAILED + weather_published_key=$(jq -er '.publisher_key_hex | select(test("^[0-9a-f]{64}$"))' \ + "$WORK/$weather_product_prefix-publish.json") || fail WEATHER_PRODUCTS_PUBLISHER_INVALID + if [ "$weather_published_key" = "$provider_publisher" ] || [ "$weather_published_key" = "$site_key" ] \ + || [ "$weather_published_key" = "$filter_key" ]; then fail WEATHER_PRODUCTS_PUBLISHER_REUSED; fi + if [ -n "$weather_key" ] && [ "$weather_key" != "$weather_published_key" ]; then fail WEATHER_PRODUCTS_PUBLISHER_CHANGED; fi + weather_key=$weather_published_key + install -o "$WORKER_UID" -g "$site_control_gid" -m 0400 "$WORK/$weather_product_prefix-publish.json" \ + "$weather_publisher/$weather_product_receipt" || fail WEATHER_PRODUCTS_PUBLISH_RECEIPT_UNAVAILABLE + for weather_node in "$provider_node_a" "$provider_node_b"; do + case $weather_node in + relay4) weather_address=49.165.5.1; weather_hostname=provider-a.volparossa.test ;; + relay5) weather_address=50.166.6.1; weather_hostname=provider-b.volparossa.test ;; + relay3) weather_address=48.164.4.1; weather_hostname=provider-c.volparossa.test ;; + *) fail WEATHER_PRODUCTS_PROVIDER_INVALID ;; + esac + weather_products_cli "$weather_node" content import --public-content \ + --manifest "$weather_publisher/$weather_product_manifest" --publisher-key "$weather_key" \ + --cache "$weather_publisher/$weather_product_source_cache" --agent-cache "$WORK/state-$weather_node/$weather_product_path-cache" \ + >"$WORK/$weather_product_prefix-$weather_node-import.json" \ + 2>"$WORK/$weather_product_prefix-$weather_node-import.err" || fail WEATHER_PRODUCTS_IMPORT_FAILED + weather_products_cli "$weather_node" content serve --name-lookup \ + --manifest "$weather_publisher/$weather_product_manifest" --publisher-key "$weather_key" \ + --cache "$WORK/state-$weather_node/$weather_product_path-cache" --bind "$weather_address:18080" \ + --advertised-hostname "$weather_hostname" >"$WORK/$weather_product_prefix-$weather_node-serve.json" \ + 2>"$WORK/$weather_product_prefix-$weather_node-serve.err" || fail WEATHER_PRODUCTS_SERVE_FAILED + jq -e --argjson publications "$weather_publications" \ + '.serving == true and .publications == $publications and .replication_enabled == false' \ + "$WORK/$weather_product_prefix-$weather_node-serve.json" >/dev/null || fail WEATHER_PRODUCTS_NOT_READY + if [ -L "$WORK/state-$weather_node/$weather_product_path-cache" ] \ + || [ "$(stat -Lc '%a:%u:%g' "$WORK/state-$weather_node/$weather_product_path-cache")" != "700:$AGENT_UID:$AGENT_GID" ]; then + fail WEATHER_PRODUCTS_PROVIDER_CACHE_CHANGED + fi + if nsenter --target "$provider_client_pid" --mount \ + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$WORK/state-$weather_node/$weather_product_path-cache"; then fail WEATHER_PRODUCTS_LOCAL_SHORTCUT; fi + if setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$WORK/state-$weather_node/$weather_product_path-cache"; then fail WEATHER_PRODUCTS_AGENT_CACHE_EXPOSED; fi + done + done + weather_products_private select "$weather_publisher" "$weather_viewer" \ + >"$WORK/weather-products-delivery-input.json" || fail WEATHER_PRODUCTS_SELECTION_FAILED + nsenter --target "$provider_client_pid" --mount \ + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$WORK/state-client/identity.key" || fail WEATHER_PRODUCTS_ISOLATION_PROBE_FAILED + if nsenter --target "$provider_client_pid" --mount \ + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$weather_viewer"; then fail WEATHER_PRODUCTS_USER_STATE_EXPOSED; fi + weather_products_private publisher-cleanup "$weather_publisher" \ + >"$WORK/weather-products-delivery-publisher-cleanup.json" || fail WEATHER_PRODUCTS_PUBLISHER_CLEANUP_FAILED + [ ! -e "$weather_publisher" ] && [ ! -L "$weather_publisher" ] || fail WEATHER_PRODUCTS_PUBLISHER_FILES_REMAIN + for weather_kind in weather air_quality; do + weather_products_select "$weather_kind" || fail WEATHER_PRODUCTS_KIND_INVALID + PHASE=weather-products-delivery-cold-$weather_product_path + start_privacy_observers "$weather_product_prefix-privacy" || fail WEATHER_PRODUCTS_CAPTURE_UNAVAILABLE + content_provider_start_control_observer "$weather_product_prefix-control" || fail WEATHER_PRODUCTS_CONTROL_UNAVAILABLE + weather_command_limit=$(weather_products_remaining 200) || fail WEATHER_PRODUCTS_DEADLINE + timeout --signal=TERM --kill-after=30s "${weather_command_limit}s" ip netns exec "$CLIENT" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-products-delivery.py" cold "$weather_product" \ + "$binary_directory/volparossa" "$WORK/runtime-client/control/agent.sock" "$weather_product_cache" "$weather_viewer" \ + "$site_parent_ns" "$site_client_ns" "$site_control_gid" \ + >"$WORK/$weather_product_prefix-cold-consumer.json" 2>"$WORK/$weather_product_prefix-cold.err" \ + || fail WEATHER_PRODUCTS_COLD_FAILED + benchmark_capture_paths "$weather_product_prefix" mptcp || fail WEATHER_PRODUCTS_PATHS_UNAVAILABLE + stop_privacy_observers || fail WEATHER_PRODUCTS_CAPTURE_INCOMPLETE + content_provider_stop_control_observer || fail WEATHER_PRODUCTS_CONTROL_INCOMPLETE + python3 -B "$WORK/bin/weather-products-delivery.py" cache-state "$weather_product" \ + "$weather_product_cache" "$AGENT_UID" "$AGENT_GID" >"$WORK/$weather_product_prefix-cache-before.json" \ + || fail WEATHER_PRODUCTS_CACHE_INVALID + done + PHASE=weather-products-delivery-cold-verify + python3 -B "$WORK/bin/weather-products-delivery.py" cache-summary "$WORK" before \ + >"$WORK/weather-products-delivery-cache-before.json" || fail WEATHER_PRODUCTS_CACHE_INVALID + python3 -B "$WORK/bin/weather-products-delivery.py" cold-evidence "$WORK" "$weather_client_manifest_sha256" \ + "$weather_core_sha256" >"$WORK/weather-products-delivery-cold-evidence.json" || fail WEATHER_PRODUCTS_COLD_EVIDENCE_INVALID + python3 -B "$WORK/bin/weather-products-delivery.py" copy-cold-evidence "$WORK" \ + "$output_directory" "$OUTPUT_UID" "$OUTPUT_GID" >/dev/null || fail WEATHER_PRODUCTS_COLD_RETENTION_FAILED + weather_command_limit=$(weather_products_remaining 40) || fail WEATHER_PRODUCTS_DEADLINE + timeout --signal=TERM --kill-after=30s "${weather_command_limit}s" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- python3 -B "$WORK/bin/weather-products-delivery.py" prepare "$weather_client_directory" "$weather_viewer" \ + "$binary_directory/volparossa" "$WORK/runtime-client/control/agent.sock" "$WORK/state-client/weather-cache" \ + "$WORK/state-client/air-quality-cache" >"$WORK/weather-products-delivery-prepared.json" || fail WEATHER_PRODUCTS_EXPECTED_FAILED + PHASE=weather-products-delivery-browser + weather_browser_status=0 + weather_command_limit=$(weather_products_remaining 270) || fail WEATHER_PRODUCTS_DEADLINE + timeout --signal=TERM --kill-after=30s "${weather_command_limit}s" ip netns exec "$CLIENT" setpriv \ + --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- env -i PATH=/usr/bin:/bin LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONDONTWRITEBYTECODE=1 \ + python3 -B "$weather_client_directory/volparossa-weather/integrations/drizzli/smoke_core_browser.py" \ + --execute --bundle-root "$weather_client_directory" --bundle-manifest-sha256 "$weather_client_manifest_sha256" \ + --config "$weather_viewer/config.json" --expected "$weather_viewer/expected.json" \ + --output "$weather_viewer/browser-output" --parent-netns "$site_parent_ns" --client-netns "$site_client_ns" \ + --control-gid "$site_control_gid" >"$WORK/weather-products-delivery-launch.json" \ + 2>"$WORK/weather-products-delivery-browser.err" || weather_browser_status=$? + PHASE=weather-products-delivery-verify + python3 -B "$WORK/bin/weather-products-delivery.py" retain "$WORK" \ + >"$WORK/weather-products-delivery-retained.json" || fail WEATHER_PRODUCTS_RECEIPT_RETENTION_FAILED + weather_client_delivery_cleanup || fail WEATHER_PRODUCTS_INPUT_CLEANUP_FAILED + [ "$weather_browser_status" -eq 0 ] || fail WEATHER_PRODUCTS_BROWSER_FAILED + weather_client_verify_tree >"$WORK/weather-products-delivery-tree-after.json" || fail WEATHER_PRODUCTS_SOURCE_CHANGED + for weather_kind in weather air_quality; do + weather_products_select "$weather_kind" || fail WEATHER_PRODUCTS_KIND_INVALID + python3 -B "$WORK/bin/weather-products-delivery.py" cache-state "$weather_product" \ + "$weather_product_cache" "$AGENT_UID" "$AGENT_GID" >"$WORK/$weather_product_prefix-cache-after.json" \ + || fail WEATHER_PRODUCTS_CACHE_INVALID + if setpriv --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$weather_product_cache"; then fail WEATHER_PRODUCTS_AGENT_CACHE_EXPOSED; fi + done + python3 -B "$WORK/bin/weather-products-delivery.py" cache-summary "$WORK" after \ + >"$WORK/weather-products-delivery-cache-after.json" || fail WEATHER_PRODUCTS_CACHE_INVALID + python3 -B "$WORK/bin/weather-products-delivery.py" evidence "$WORK" "$weather_client_manifest_sha256" \ + "$weather_core_sha256" >"$WORK/weather-products-delivery-evidence.json" || fail WEATHER_PRODUCTS_EVIDENCE_INVALID + PHASE=weather-products-delivery-complete +} From c8b57d94dd21e8db6a08f961228cccc530aa873d Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:20:14 +0200 Subject: [PATCH 18/49] test: initialize optional topology bundle fixtures --- .../integration/test-cooperative-code-vm-contract.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/integration/test-cooperative-code-vm-contract.py b/tests/integration/test-cooperative-code-vm-contract.py index 21d9a2c56..2b90cea2a 100644 --- a/tests/integration/test-cooperative-code-vm-contract.py +++ b/tests/integration/test-cooperative-code-vm-contract.py @@ -30,7 +30,9 @@ def selected_topology_argv(scenario): raise AssertionError("guest output redirection changed") selector = selector.replace(original, "/dev/null") script = "\n".join(("set -eu", 'scenario=$1', "expected_commit=" + "a" * 40, - "code_manifest_sha256=" + "b" * 64, "exec 3>&1", + "code_manifest_sha256=" + "b" * 64, + "browser_filter_manifest_sha256=none", "weather_client_manifest_sha256=none", + "exec 3>&1", "guest_phase() { :; }", "sudo() { printf '%s\\0' \"$@\" >&3; }", selector)) result = subprocess.run(["sh", "-c", script, "test", scenario], capture_output=True, timeout=5, check=True) if not result.stdout.endswith(b"\0"): @@ -45,6 +47,13 @@ def common_topology_args(scenario): class CooperativeCodeVm(unittest.TestCase): + def test_guest_selector_preserves_default_path_without_optional_bundles(self): + for scenario in ("alpha", "content-provider"): + with self.subTest(scenario=scenario): + self.assertEqual(selected_topology_argv(scenario), + ["-n", "--", "env", "./tests/integration/kvm-alpha-topology.sh", + *common_topology_args(scenario)]) + def test_guest_selector_preserves_exact_code_and_browser_commands(self): self.assertEqual(selected_topology_argv("agent-cooperative-code"), ["-n", "--", "./tests/integration/kvm-alpha-topology.sh", From adb9c149e776b6c213ee677eeaaf5ff3f900558a Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:04:22 +0200 Subject: [PATCH 19/49] fix(content): rediscover complementary providers within one deadline --- crates/volparossa-agent/src/content.rs | 64 +-- .../volparossa-agent/src/content/parallel.rs | 16 + .../src/content/rediscovery.rs | 240 +++++++++ .../src/content/rediscovery/tests.rs | 506 ++++++++++++++++++ docs/IMPLEMENTATION_STATUS.md | 74 ++- 5 files changed, 827 insertions(+), 73 deletions(-) create mode 100644 crates/volparossa-agent/src/content/rediscovery.rs create mode 100644 crates/volparossa-agent/src/content/rediscovery/tests.rs diff --git a/crates/volparossa-agent/src/content.rs b/crates/volparossa-agent/src/content.rs index d47ea3c65..53dc606fb 100644 --- a/crates/volparossa-agent/src/content.rs +++ b/crates/volparossa-agent/src/content.rs @@ -20,6 +20,7 @@ mod object_policy; mod parallel; mod private_storage; mod recent; +mod rediscovery; #[cfg(test)] mod refresh_tests; mod replication; @@ -759,68 +760,7 @@ impl ContentRuntime { policy: &volparossa_policy::VerifiedManifest, control_peer: libp2p::PeerId, ) -> Result<(Vec, u64), ContentError> { - if complete(manifest, store)? { - return Ok((Vec::new(), 0)); - } - let mut used = std::collections::BTreeSet::new(); - let mut attempted = std::collections::BTreeSet::new(); - let mut bytes = 0_u64; - if let Some(scope) = recent::RecentProviderScope::for_route(context, policy).await { - let recent = context - .content - .refresh_recent_provider_hints(scope, &context.discovery, Duration::from_secs(1)) - .await; - attempted.extend(recent.iter().map(|provider| provider.peer_id)); - let (providers, received) = - Self::pull_providers(context, manifest, store, policy, recent).await?; - used.extend(providers); - bytes = received; - if complete(manifest, store)? { - return Ok((used.into_iter().collect(), bytes)); - } - } - let discovered = context - .discovery - .discover_content_providers(control_peer, 16) - .await; - let state = context.state.read().await; - if !state.roles().client - || state - .active_policy(unix_millis()) - .as_ref() - .map(volparossa_policy::VerifiedManifest::policy_hash) - != Some(policy.policy_hash()) - { - return Err(ContentError::Policy); - } - drop(state); - let providers = match discovered { - Ok(providers) => providers, - Err(DiscoveryError::Invalid) => return Err(ContentError::Invalid), - Err(DiscoveryError::Invalidated) => return Err(ContentError::Policy), - Err( - DiscoveryError::Busy - | DiscoveryError::Closed - | DiscoveryError::Timeout - | DiscoveryError::Unavailable, - ) => { - // A later unavailable capability search cannot erase verified earlier bytes. - // Native callers still require complete reconstruction; HTTPS fills only gaps. - if bytes > 0 { - return Ok((used.into_iter().collect(), bytes)); - } - return Err(ContentError::Unavailable); - } - }; - let providers = providers - .into_iter() - .filter(|provider| !attempted.contains(&provider.peer_id)) - .collect(); - let (providers, received) = - Self::pull_providers(context, manifest, store, policy, providers).await?; - used.extend(providers); - bytes = bytes.checked_add(received).ok_or(ContentError::Invalid)?; - Ok((used.into_iter().collect(), bytes)) + rediscovery::pull(context, manifest, store, policy, control_peer).await } /// Reuse the same bounded provider set after a name-metadata round; no second discovery. diff --git a/crates/volparossa-agent/src/content/parallel.rs b/crates/volparossa-agent/src/content/parallel.rs index 5edbf70a0..33d86ff2d 100644 --- a/crates/volparossa-agent/src/content/parallel.rs +++ b/crates/volparossa-agent/src/content/parallel.rs @@ -102,6 +102,22 @@ pub(super) async fn pull_with_budget( let deadline = Instant::now() .checked_add(budget) .ok_or(ContentError::Invalid)?; + pull_until(context, manifest, store, policy, providers, deadline).await +} + +/// Preserve a caller's absolute session deadline across discovery and successive batches. +/// Partial verified progress survives exhaustion; no batch renews protected-stream lifetime. +pub(super) async fn pull_until( + context: &ControlContext, + manifest: &VerifiedManifest, + store: &mut ChunkStore, + policy: &volparossa_policy::VerifiedManifest, + providers: Vec, + deadline: Instant, +) -> Result<(Vec, u64), ContentError> { + if providers.len() > super::recent::OFFER_BATCH { + return Err(ContentError::Invalid); + } pull_inner( context, manifest, diff --git a/crates/volparossa-agent/src/content/rediscovery.rs b/crates/volparossa-agent/src/content/rediscovery.rs new file mode 100644 index 000000000..e2e2833a7 --- /dev/null +++ b/crates/volparossa-agent/src/content/rediscovery.rs @@ -0,0 +1,240 @@ +//! Bounded fresh capability snapshots for complementary native chunks. +//! +//! A completed DHT query is not a complete catalogue: a newly registered service may +//! become visible after the first snapshot. Keep one writer and original authority; +//! only previously unattempted providers may fill the remaining verified-cache gaps. + +use std::{collections::BTreeSet, future::Future, time::Duration}; + +use libp2p::PeerId; +use tokio::time::{Instant, timeout_at}; +use volparossa_content::{ChunkStore, VerifiedManifest, transfer::TransferLimits}; + +use super::{ + ContentError, check_publication_time, complete, parallel, recent::RecentProviderScope, +}; +use crate::{ + control::ControlContext, + discovery::{ContentDiscoveryError, DiscoveredContentProvider}, + unix_millis, +}; + +// One initial generic lookup and two bounded chances for late announcements. No sleeps, +// full-network catalogue, compulsory provider count, or per-round deadline renewal. +const MAX_ROUNDS: usize = 3; +const MAX_PROVIDERS: usize = volparossa_discovery::MAX_CONTENT_OFFERS; + +pub(super) async fn pull( + context: &ControlContext, + manifest: &VerifiedManifest, + store: &mut ChunkStore, + policy: &volparossa_policy::VerifiedManifest, + control_peer: PeerId, +) -> Result<(Vec, u64), ContentError> { + if complete(manifest, store)? { + return Ok((Vec::new(), 0)); + } + // The existing outer operation timeout remains in force, including route setup. + // All hints, discovery and batches below additionally share one transfer budget. + let deadline = Instant::now() + TransferLimits::default().session_timeout; + let scope = RecentProviderScope::for_route(context, policy) + .await + .filter(|scope| scope.control_peer == control_peer) + .ok_or(ContentError::Unavailable)?; + run( + &mut ProtectedSource { + context, + manifest, + store, + policy, + scope, + }, + deadline, + ) + .await +} + +// The coordinator below is shared by production and deterministic tests. Test sources +// control only discovery timing/authority; successful transfers use the actual chunk wire. +trait Source { + fn complete(&mut self) -> Result; + fn validate(&mut self) -> impl Future> + Send; + fn recent( + &mut self, + budget: Duration, + ) -> impl Future> + Send; + // Deliberately accepts only a count: never a manifest, object ID, name or URL. + fn discover( + &mut self, + maximum: usize, + ) -> impl Future, ContentDiscoveryError>> + Send; + fn transfer( + &mut self, + providers: Vec, + deadline: Instant, + ) -> impl Future, u64), ContentError>> + Send; +} + +struct ProtectedSource<'a> { + context: &'a ControlContext, + manifest: &'a VerifiedManifest, + store: &'a mut ChunkStore, + policy: &'a volparossa_policy::VerifiedManifest, + scope: RecentProviderScope, +} + +impl Source for ProtectedSource<'_> { + fn complete(&mut self) -> Result { + complete(self.manifest, self.store) + } + + async fn validate(&mut self) -> Result<(), ContentError> { + check_publication_time(self.manifest)?; + let state = self.context.state.read().await; + if !state.roles().client + || state + .active_policy(unix_millis()) + .as_ref() + .map(volparossa_policy::VerifiedManifest::policy_hash) + != Some(self.policy.policy_hash()) + { + return Err(ContentError::Policy); + } + drop(state); + if RecentProviderScope::for_route(self.context, self.policy).await != Some(self.scope) { + return Err(ContentError::Policy); + } + Ok(()) + } + + async fn recent(&mut self, budget: Duration) -> Vec { + self.context + .content + .refresh_recent_provider_hints(self.scope, &self.context.discovery, budget) + .await + } + + async fn discover( + &mut self, + maximum: usize, + ) -> Result, ContentDiscoveryError> { + self.context + .discovery + .discover_content_providers(self.scope.control_peer, maximum) + .await + } + + async fn transfer( + &mut self, + providers: Vec, + deadline: Instant, + ) -> Result<(Vec, u64), ContentError> { + Box::pin(parallel::pull_until( + self.context, + self.manifest, + self.store, + self.policy, + providers, + deadline, + )) + .await + } +} + +#[derive(Default)] +struct Progress { + attempted: BTreeSet, + used: BTreeSet, + bytes: u64, +} + +impl Progress { + async fn transfer( + &mut self, + source: &mut impl Source, + providers: Vec, + deadline: Instant, + ) -> Result<(), ContentError> { + if providers.len() > MAX_PROVIDERS { + return Err(ContentError::Invalid); + } + let providers = providers + .into_iter() + .filter(|provider| { + self.attempted.len() < MAX_PROVIDERS && self.attempted.insert(provider.peer_id) + }) + .collect::>(); + let remaining = deadline.saturating_duration_since(Instant::now()); + if providers.is_empty() || remaining.is_zero() { + return Ok(()); + } + let (used, bytes) = source.transfer(providers, deadline).await?; + source.validate().await?; + self.used.extend(used); + self.bytes = self.bytes.checked_add(bytes).ok_or(ContentError::Invalid)?; + Ok(()) + } + + fn receipt(self) -> (Vec, u64) { + (self.used.into_iter().collect(), self.bytes) + } +} + +async fn run( + source: &mut impl Source, + deadline: Instant, +) -> Result<(Vec, u64), ContentError> { + if source.complete()? { + return Ok((Vec::new(), 0)); + } + source.validate().await?; + let mut progress = Progress::default(); + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(ContentError::Unavailable); + } + let recent = timeout_at( + deadline, + source.recent(remaining.min(Duration::from_secs(1))), + ) + .await + .unwrap_or_default(); + source.validate().await?; + progress.transfer(source, recent, deadline).await?; + for _ in 0..MAX_ROUNDS { + source.validate().await?; + if source.complete()? { + return Ok(progress.receipt()); + } + if Instant::now() >= deadline || progress.attempted.len() >= MAX_PROVIDERS { + break; + } + let discovered = timeout_at(deadline, source.discover(MAX_PROVIDERS)) + .await + .unwrap_or(Err(ContentDiscoveryError::Timeout)); + source.validate().await?; + let providers = match discovered { + Ok(providers) => providers, + Err(ContentDiscoveryError::Invalid) => return Err(ContentError::Invalid), + Err(ContentDiscoveryError::Invalidated) => return Err(ContentError::Policy), + Err( + ContentDiscoveryError::Busy + | ContentDiscoveryError::Closed + | ContentDiscoveryError::Timeout + | ContentDiscoveryError::Unavailable, + ) => { + // Transient discovery errors never discard already authenticated progress. + // Native reconstruction still requires completeness; HTTPS may fill gaps. + if progress.bytes == 0 { + return Err(ContentError::Unavailable); + } + break; + } + }; + progress.transfer(source, providers, deadline).await?; + } + Ok(progress.receipt()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/volparossa-agent/src/content/rediscovery/tests.rs b/crates/volparossa-agent/src/content/rediscovery/tests.rs new file mode 100644 index 000000000..9a114b894 --- /dev/null +++ b/crates/volparossa-agent/src/content/rediscovery/tests.rs @@ -0,0 +1,506 @@ +//! Deterministic snapshot changes around real signed manifests, stores and v1 chunk streams. +//! These are coordinator/stream tests, not a distributed discovery or protected-overlay proof. + +use std::{ + collections::{BTreeMap, VecDeque}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::Poll, +}; + +use ed25519_dalek::SigningKey; +use tokio::{io::duplex, sync::Notify}; +use volparossa_content::{ + CHUNK_BYTES, CacheLimits, Metadata, Publication, Validity, + provider::{ + ProviderEndpoint, PublicationRegistry, SignedProviderOffer, pull_publication_worker, + serve_publication, + }, + publish, reassemble, + transfer::{TransferProgress, parallel::ParallelDownload}, +}; +use volparossa_discovery::ContentDiscoveryRequest; + +use super::*; +use crate::content::now; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Stall { + None, + Discovery, + Transfer, +} + +struct Held(Arc); + +impl Held { + fn new(count: &Arc) -> Self { + count.fetch_add(1, Ordering::SeqCst); + Self(Arc::clone(count)) + } +} + +impl Drop for Held { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +struct Fixture { + _directory: tempfile::TempDir, + manifest: VerifiedManifest, + cache: ChunkStore, + registries: BTreeMap, + providers: Vec, + recent: Vec, + snapshots: VecDeque, ContentDiscoveryError>>, + requests: Vec, + opened: Vec, + deadlines: Vec, + bytes: Vec, + invalid: bool, + invalidate_discovery: bool, + invalidate_transfer: bool, + fail_transfer: bool, + stall: Stall, + held: Arc, + entered: Arc, +} + +fn fixture(provider_count: usize) -> Fixture { + let directory = tempfile::tempdir().unwrap(); + let limits = CacheLimits { + max_bytes: (4 * CHUNK_BYTES) as u64, + max_entries: 4, + min_free_bytes: 0, + }; + let mut publisher = ChunkStore::create(&directory.path().join("publisher"), limits).unwrap(); + let cache = ChunkStore::create(&directory.path().join("receiver"), limits).unwrap(); + let bytes = [31_u8, 47, 91] + .into_iter() + .flat_map(|value| vec![value; CHUNK_BYTES]) + .collect::>(); + let signer = SigningKey::from_bytes(&[181; 32]); + let validity = Validity { + created: now(), + expires: now() + 60, + }; + let envelope = publish( + &mut bytes.as_slice(), + Publication { + length: bytes.len() as u64, + metadata: Metadata { + name: "private-looking-object-not-in-discovery".into(), + revision: 1, + content_type: "application/octet-stream".into(), + }, + validity, + }, + &signer, + &mut publisher, + ) + .unwrap(); + let manifest = envelope.verify(&signer.verifying_key(), now()).unwrap(); + let mut providers = Vec::new(); + let mut registries = BTreeMap::new(); + for index in 0..provider_count { + let key = SigningKey::from_bytes(&[u8::try_from(index + 1).unwrap(); 32]); + let public = + libp2p::identity::ed25519::PublicKey::try_from_bytes(key.verifying_key().as_bytes()) + .unwrap(); + let peer_id = libp2p::identity::PublicKey::from(public).to_peer_id(); + let offer = SignedProviderOffer::sign( + &key, + ProviderEndpoint::new("provider.volparossa.test", 18443).unwrap(), + validity, + ) + .unwrap() + .verify(&key.verifying_key(), now()) + .unwrap(); + providers.push(DiscoveredContentProvider { peer_id, offer }); + let root = directory.path().join(format!("provider-{index}")); + let mut store = ChunkStore::create(&root, limits).unwrap(); + for (chunk_index, chunk) in manifest.chunks().iter().enumerate() { + // A has only the first chunk; B has the remaining chunks AND the first. + // Exact transferred byte counts therefore detect refetching a verified chunk. + if (index == 0 && chunk_index == 0) || index == 1 { + store + .put_verified(*chunk.id(), &publisher.get(chunk.id()).unwrap().unwrap()) + .unwrap(); + } + } + drop(store); + let mut registry = PublicationRegistry::new(); + registry + .register(manifest.clone(), root, limits, now()) + .unwrap(); + registries.insert(peer_id, registry); + } + Fixture { + _directory: directory, + manifest, + cache, + registries, + providers, + recent: Vec::new(), + snapshots: VecDeque::new(), + requests: Vec::new(), + opened: Vec::new(), + deadlines: Vec::new(), + bytes, + invalid: false, + invalidate_discovery: false, + invalidate_transfer: false, + fail_transfer: false, + stall: Stall::None, + held: Arc::new(AtomicUsize::new(0)), + entered: Arc::new(Notify::new()), + } +} + +impl Source for Fixture { + fn complete(&mut self) -> Result { + complete(&self.manifest, &mut self.cache) + } + + async fn validate(&mut self) -> Result<(), ContentError> { + check_publication_time(&self.manifest)?; + if self.invalid { + Err(ContentError::Policy) + } else { + Ok(()) + } + } + + async fn recent(&mut self, _budget: Duration) -> Vec { + std::mem::take(&mut self.recent) + } + + async fn discover( + &mut self, + maximum: usize, + ) -> Result, ContentDiscoveryError> { + self.requests + .push(ContentDiscoveryRequest::new(maximum).unwrap()); + if self.stall == Stall::Discovery { + let _held = Held::new(&self.held); + self.entered.notify_one(); + std::future::pending::<()>().await; + } + self.invalid |= self.invalidate_discovery; + self.snapshots.pop_front().unwrap_or(Ok(Vec::new())) + } + + async fn transfer( + &mut self, + providers: Vec, + deadline: Instant, + ) -> Result<(Vec, u64), ContentError> { + self.deadlines.push(deadline); + self.opened + .extend(providers.iter().map(|provider| provider.peer_id)); + if self.fail_transfer { + return Err(ContentError::Invalid); + } + let limits = TransferLimits { + session_timeout: deadline.saturating_duration_since(Instant::now()), + exchange_timeout: Duration::from_secs(2), + max_requests: 3, + max_bytes: self.bytes.len() as u64, + }; + let (download, workers) = ParallelDownload::new_adaptive( + &self.manifest, + providers.len(), + &mut self.cache, + limits, + providers.len(), + ) + .unwrap(); + let mut progress = vec![TransferProgress::default(); providers.len()]; + let mut streams = workers + .into_iter() + .zip(&providers) + .map(|(mut worker, provider)| { + let registry = &self.registries[&provider.peer_id]; + let stalled = self.stall == Stall::Transfer; + let held = &self.held; + let entered = &self.entered; + // Owned futures, real selector/chunk protocol and real cache writer. No tasks + // outlive the coordinator when the requester cancels or its deadline expires. + Some(Box::pin(async move { + let _held = Held::new(held); + let (mut client, mut server) = duplex(4096); + let serve = async { + if stalled { + entered.notify_one(); + std::future::pending::<()>().await; + } + serve_publication(&mut server, registry, limits).await + }; + let (received, sent) = + tokio::join!(pull_publication_worker(&mut client, &mut worker), serve); + received.unwrap(); + sent.unwrap(); + })) + }) + .collect::>(); + let joined = async { + let all_streams = std::future::poll_fn(|cx| { + for slot in &mut streams { + if slot + .as_mut() + .is_some_and(|work| work.as_mut().poll(cx).is_ready()) + { + *slot = None; + } + } + if streams.iter().all(Option::is_none) { + Poll::Ready(()) + } else { + Poll::Pending + } + }); + let (written, ()) = tokio::join!( + download.run_with_allowance(&mut self.cache, &mut progress, || providers.len()), + all_streams + ); + written.map_err(|_| ContentError::Unavailable) + }; + timeout_at(deadline, joined) + .await + .map_err(|_| ContentError::Unavailable)??; + self.invalid |= self.invalidate_transfer; + let bytes = progress.iter().map(|p| p.bytes).sum(); + let used = providers + .iter() + .zip(progress) + .filter(|(_, p)| p.bytes > 0) + .map(|(provider, _)| provider.peer_id.to_string()) + .collect(); + Ok((used, bytes)) + } +} + +fn deadline() -> Instant { + Instant::now() + Duration::from_secs(8) +} + +#[tokio::test] +async fn late_complementary_snapshot_reconstructs_without_refetching_or_renewing_deadline() { + let mut source = fixture(2); + let first = source.providers[0].clone(); + let second = source.providers[1].clone(); + source.snapshots.extend([ + Ok(vec![first.clone()]), + Ok(vec![first.clone(), second.clone()]), + ]); + let until = deadline(); + let (providers, bytes) = run(&mut source, until).await.unwrap(); + assert_eq!( + bytes, + source.bytes.len() as u64, + "each verified chunk counted only once" + ); + assert_eq!(providers.len(), 2); + assert_eq!(source.opened, [first.peer_id, second.peer_id]); + assert_eq!(source.deadlines, [until, until]); + assert_eq!(source.requests.len(), 2); + let mut output = Vec::new(); + assert_eq!( + reassemble( + &source.manifest, + &mut [&mut source.cache], + now(), + &mut output + ) + .unwrap(), + source.bytes.len() as u64 + ); + assert_eq!(output, source.bytes); + let (providers, bytes) = run(&mut source, until).await.unwrap(); + assert!(providers.is_empty()); + assert_eq!(bytes, 0); + assert_eq!(source.requests.len(), 2, "complete cache needs no network"); + assert_eq!(source.opened.len(), 2); +} + +#[tokio::test] +async fn complete_single_provider_needs_neither_a_second_provider_nor_rediscovery() { + let mut source = fixture(2); + source + .snapshots + .push_back(Ok(vec![source.providers[1].clone()])); + let (providers, bytes) = run(&mut source, deadline()).await.unwrap(); + assert_eq!(providers.len(), 1); + assert_eq!(bytes, source.bytes.len() as u64); + assert_eq!(source.requests.len(), 1); + assert_eq!(source.opened.len(), 1); +} + +#[tokio::test] +async fn duplicate_recent_hints_and_snapshots_do_not_repeat_provider_work() { + let mut source = fixture(2); + let first = source.providers[0].clone(); + let second = source.providers[1].clone(); + source.recent = vec![first.clone(), first.clone()]; + source.snapshots.extend([ + Ok(vec![first.clone(), first.clone()]), + Ok(vec![first.clone(), second.clone(), second.clone()]), + ]); + let (_, bytes) = run(&mut source, deadline()).await.unwrap(); + assert_eq!(bytes, source.bytes.len() as u64); + assert_eq!(source.opened, [first.peer_id, second.peer_id]); +} + +#[tokio::test] +async fn repeated_same_and_empty_snapshots_are_bounded_and_not_full_object_success() { + for duplicate in [false, true] { + let mut source = fixture(2); + for _ in 0..MAX_ROUNDS + 1 { + source.snapshots.push_back(Ok(if duplicate { + vec![source.providers[0].clone()] + } else { + Vec::new() + })); + } + let (_, bytes) = run(&mut source, deadline()).await.unwrap(); + assert_eq!(source.requests.len(), MAX_ROUNDS); + assert_eq!(source.opened.len(), usize::from(duplicate)); + assert_eq!(bytes, if duplicate { CHUNK_BYTES as u64 } else { 0 }); + assert!(!source.complete().unwrap()); + assert!( + reassemble( + &source.manifest, + &mut [&mut source.cache], + now(), + &mut std::io::sink() + ) + .is_err() + ); + for request in &source.requests { + assert_eq!(request.maximum_offers(), MAX_PROVIDERS); + assert!(request.target_peers().unwrap().is_empty()); + assert_eq!(request.nonce().len(), 32); + } + assert!( + source + .requests + .windows(2) + .all(|r| r[0].nonce() != r[1].nonce()) + ); + } +} + +#[tokio::test] +async fn total_unique_attempts_include_hints_and_cannot_exceed_one_offer_batch() { + let mut source = fixture(MAX_PROVIDERS + 3); + // Empty providers only, so completion cannot mask the total-attempt bound. + source.recent = source.providers[2..4].to_vec(); + source + .snapshots + .push_back(Ok(source.providers[3..MAX_PROVIDERS + 3].to_vec())); + let (_, bytes) = run(&mut source, deadline()).await.unwrap(); + assert_eq!(bytes, 0); + assert_eq!(source.opened.len(), MAX_PROVIDERS); + assert_eq!( + source.opened.iter().copied().collect::>().len(), + MAX_PROVIDERS + ); + assert_eq!(source.requests.len(), 1); +} + +#[tokio::test] +async fn authority_and_discovery_errors_remain_fatal_or_preserve_only_verified_partial_progress() { + for error in [ + ContentDiscoveryError::Invalid, + ContentDiscoveryError::Invalidated, + ContentDiscoveryError::Busy, + ContentDiscoveryError::Closed, + ContentDiscoveryError::Timeout, + ContentDiscoveryError::Unavailable, + ] { + for partial in [false, true] { + let mut source = fixture(2); + if partial { + source.recent.push(source.providers[0].clone()); + } + source.snapshots.push_back(Err(error)); + let result = run(&mut source, deadline()).await; + match error { + ContentDiscoveryError::Invalid => { + assert!(matches!(result, Err(ContentError::Invalid))) + } + ContentDiscoveryError::Invalidated => { + assert!(matches!(result, Err(ContentError::Policy))) + } + _ if partial => assert_eq!(result.unwrap().1, CHUNK_BYTES as u64), + _ => assert!(matches!(result, Err(ContentError::Unavailable))), + } + assert_eq!(source.requests.len(), 1); + assert_eq!( + source + .cache + .get(source.manifest.chunks()[0].id()) + .unwrap() + .is_some(), + partial + ); + } + } + for after_transfer in [false, true] { + let mut source = fixture(2); + source.invalidate_discovery = !after_transfer; + source.invalidate_transfer = after_transfer; + source.snapshots.push_back(Ok(source.providers.clone())); + assert!(matches!( + run(&mut source, deadline()).await, + Err(ContentError::Policy) + )); + assert_eq!(source.opened.len(), if after_transfer { 2 } else { 0 }); + } + let mut source = fixture(2); + source.fail_transfer = true; + source.snapshots.push_back(Ok(source.providers.clone())); + assert!(matches!( + run(&mut source, deadline()).await, + Err(ContentError::Invalid) + )); +} + +#[tokio::test] +async fn deadlines_and_requester_cancellation_drop_owned_discovery_and_stream_futures() { + for stall in [Stall::Discovery, Stall::Transfer] { + let mut source = fixture(2); + source.stall = stall; + source.snapshots.push_back(Ok(source.providers.clone())); + let held = Arc::clone(&source.held); + let entered = Arc::clone(&source.entered); + { + let work = run(&mut source, deadline()); + tokio::pin!(work); + tokio::select! { result = &mut work => panic!("must remain pending: {result:?}"), () = entered.notified() => {} } + assert!(held.load(Ordering::SeqCst) > 0); + } + assert_eq!( + held.load(Ordering::SeqCst), + 0, + "requester drop owns cancellation" + ); + let mut source = fixture(2); + source.stall = stall; + source.snapshots.push_back(Ok(source.providers.clone())); + let result = run(&mut source, Instant::now() + Duration::from_millis(30)).await; + assert!(matches!(result, Err(ContentError::Unavailable))); + assert_eq!(source.held.load(Ordering::SeqCst), 0); + assert_eq!(source.requests.len(), 1); + } + let mut source = fixture(2); + assert!(matches!( + run(&mut source, Instant::now()).await, + Err(ContentError::Unavailable) + )); + assert!(source.requests.is_empty()); + assert!(source.opened.is_empty()); +} diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 6dea14568..0f04af53a 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,16 +29,16 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify the adaptive-provider correction in the real disposable topology, preserving the joined temperature/wind-to-Drizz.li proof, traffic checks and cleanup. | Core candidate `44c50563` passes targeted tests, but the corrected live run remains outstanding. Trial 03 stopped before VM startup at the unchanged 4 GiB guest plus 2 GiB host-reserve gate; it produced no new network evidence. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The combined UI/bridge passes with synthetic data. The two-product joined driver and exact input bundle now pass targeted checks; the next step is the real disposable trial when the unchanged memory gate permits. Real peer delivery remains unproved; existing temperature/wind evidence does not cover it. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Recover missing chunks from late-discovered complementary providers within one bounded operation, then verify the real topology including the earlier adaptive-provider correction. | The local run on `c8b57d94` reached the base fetch but discovered only one of two complementary stores. The separate CI run failed earlier, before discovering the required Exit. Neither reached adaptive transfer or Weather delivery. Keep these failures distinct; do not replace the required Exit or weaken completeness checks. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The combined UI/bridge passes with synthetic data; the joined driver and immutable input bundle pass targeted checks. Real two-product delivery depends on the shared discovery repair and a new exact-source trial. Existing temperature/wind proof and the separate NOAA source trial do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather` | Build and exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | Reuse the provider contracts and core retrieval path; client adaptation, a reproducible client build and its own runtime proof remain required. Do not create a second network scheduler or source-acquisition implementation. | -Parallel work may continue on the Drizz.li field adapter and Overmorrow client -boundaries while the core trial waits for sufficient memory. Keep the trial's -inputs frozen, give shared interface changes one owner and integrate dependent -consumer changes together. Do not launch competing heavy builds or lower the -host reserve to make a trial start. A resource wait delays that proof, not all -implementation work. +Prioritize the shared discovery failure before another unchanged live run or a +second client integration. The Drizz.li/Weather candidate and frozen input bundle +remain intact; Overmorrow's client work is still required. Keep shared interface +changes under one owner and integrate dependent consumers together. Any new VM +trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; +do not run competing heavy builds or lower that reserve. After each milestone, or when a substantive blocker changes, reassess the next shared dependency against the existing evidence and application worktrees. @@ -404,6 +404,12 @@ The Weather air-quality candidate now has a separate NOAA GEFS-Aerosols PM2.5/PM10 contract, exact-range source adapter and shared core-cache consumer. A real CLI/decoder trial verifies two native instantaneous forecast points (steps 3 and 6) using 2,841,174 field bytes; it does not derive AQI or observations. +A separate [native forecast receipt](https://github.com/VOLPAROSSA/volparossa-weather/blob/f5accb1c674aa2bd2eaec00e36ccbc4c393b51a0/third_party/noaa-gefs-future-smoke.json) +records steps 3, 6, 12, 24 and 48 from the 2026-10-11 00:00 UTC run, acquired +at 18:13 UTC that day. Steps 24 and 48 were still in the future at acquisition. +Ten native fields total 6,357,433 bytes and produce an 8,773-byte forecast. +This verifies source acquisition and the production decoder, not forecast +accuracy, observations, peer delivery or browser display. Ninety-one Weather tests pass, including subprocess-based cache/CLI boundaries, query/expiry checks and cancellation. The particulate core-cache tests use a synthetic core process; real particulate overlay delivery remains unproved. @@ -473,9 +479,55 @@ and archive SHA-256 `f1509e305236e37afecfc7287e517c99a6714487f036af4b25db5770a0bc5493`. Capture, full runtime-input verification, read-only unpack and tree verification pass; the frozen v1 archive remains unchanged and accepted. This is fixture and -input evidence, not a completed live two-product trial. The unchanged 4 GiB guest -plus 2 GiB host reserve is still required; no VM or new origin acquisition was -started for this preparation. +input evidence, not a completed live two-product trial. + +The subsequent local run on `c8b57d94dd21e8db6a08f961228cccc530aa873d` +passed the unchanged memory gate, built the core in the disposable VM, and +established the protected route. It failed at the earlier base content fetch +with `CONTENT_PROVIDER_RUNTIME_FETCH_FAILED`, before Weather acquisition. +Both stores were registered locally, but the one discovery snapshot recorded +only one verified provider offer. Only that provider, holding five of nine chunks, +returned application data; the complementary four-chunk provider received no +application request. The failed fetch retained no completed transfer receipt. +A completed discovery query is not a complete provider catalogue. The evidence +establishes the incomplete snapshot; a publication/convergence race is the +likely cause, not a separately proven DHT diagnosis. + +The failed local report is retained at +`.git/ci-evidence/weather-products-delivery-01.TH6yxN/output/content-provider-smoke.json` +with SHA-256 `3a4a9c7243e1b77508a5f5d5a578bb75f3b1ecf36505aef4525258526eb15a1a`. +Cleanup reports zero owned objects and unchanged guest-root network state outside +the test namespaces. The runner and QEMU exited. This report contains no Weather +transfer, UI, adaptive-provider or full-alpha success. + +The separate [CI run 38163595466](https://github.com/VOLPAROSSA/volparossa/actions/runs/38163595466) +on the same source failed earlier with `CONTENT_PROVIDER_INVENTORY_UNAVAILABLE`: +the relay advertisements and second Exit appeared, but the required primary Exit +did not. Its artifact retains successful cleanup and unchanged guest-root state. +Current logs do not distinguish a missing primary publication from later +provider-index propagation. Neither a longer readiness wait nor substituting +the second Exit is an established fix. The previous pre-VM fixture failure +in [run 38162824767](https://github.com/VOLPAROSSA/volparossa/actions/runs/38162824767) +is fixed by `c8b57d94`; that correction does not resolve either live failure. + +The candidate now retries completed capability snapshots when verified chunks +are missing: at most three generic discovery rounds and sixteen unique provider +candidates, including recent hints. All batches share one absolute 300-second +transfer budget inside the unchanged 600-second operation limit. Previously +verified chunks remain in the same cache, and publication, policy, control-relay +and route-context authority are checked between rounds. There is no new origin +or direct-peer fallback. Native fetch and explicit HTTPS `PeersFirst` use this +coordinator; other source-selection strategies are unchanged. + +Seven targeted coordinator/stream tests pass, including late complementary +discovery, exact reconstruction without duplicate bytes, the complete-cache and +single-provider fast paths, bounds, errors, invalidation, deadlines and +cancellation. These tests use signed manifests and real chunk streams, with +controlled discovery snapshots. Five existing parallel-transfer tests and four +recent-provider tests also pass, including partial progress at the transfer +deadline and cleanup before fallback. These checks do not establish live DHT convergence or +resolve the separate CI primary-Exit failure. A new exact-source topology trial +remains required before either live failure is described as fixed. ### Citizen procedure simulation From 5f82f619cca8c4e6e621d01bcc3269e41a499ba8 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:25:31 +0200 Subject: [PATCH 20/49] test: require future weather coverage in joined delivery proof --- docs/IMPLEMENTATION_STATUS.md | 13 +- .../test-weather-products-delivery.py | 175 ++++++++++++++++-- .../integration/weather-products-delivery.py | 65 ++++++- 3 files changed, 225 insertions(+), 28 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 0f04af53a..0db568e28 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -459,8 +459,17 @@ forecasts and the publisher identity before either cold fetch. Each product requires its own drained packet/control captures, selected protected route, byte-matched receipt and private-cache checks. The browser then checks both provider responses, initial panels and refresh against those cold receipts. -Native NOAA timestamps remain independent of ECMWF; these selected model points -are not presented as current observations or guaranteed future coverage. +Native NOAA timestamps remain independent of ECMWF. The driver now requests +IFS leads 3, 9, 24 and 48 hours and NOAA leads 3, 6, 12, 24 and 48 hours: +sixteen weather fields and ten particulate fields. It checks exact native +validity times and requires future coverage during acquisition, browser +preparation and final proof creation. Earlier forecast points are not relabelled +as current observations. This correction leaves the frozen Weather inputs and +time budgets unchanged; future coverage in the joined live trial remains to be +demonstrated. Seventeen targeted driver tests, eight shell-contract tests and +fourteen unchanged v1-delivery tests pass. They cover clock boundaries, native +validity, acquisition timing, cold-byte and browser-expectation binding, and +historical evidence validation without consulting the current clock. A shared 840-second post-acquisition budget bounds publication, transfers and browser work, without extending either signed 900-second publication or source diff --git a/tests/integration/test-weather-products-delivery.py b/tests/integration/test-weather-products-delivery.py index a466d4c59..8fff69422 100644 --- a/tests/integration/test-weather-products-delivery.py +++ b/tests/integration/test-weather-products-delivery.py @@ -16,6 +16,38 @@ A = runpy.run_path(str(HERE / "weather-products-delivery.py")) OLD = runpy.run_path(str(HERE / "test-weather-client-delivery.py")) SHA = "a" * 64 +CLOCK = A["stamp"]("2026-10-11T14:00:00.000Z") + + +def iso(milliseconds): + return (datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + + datetime.timedelta(milliseconds=milliseconds)).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def forecast(key, model_run=None, acquired_at=None): + p = A["PRODUCTS"][key] + model_run = model_run or selected()["products"][key]["query"]["run"] + def field(parameter): + return dict(parameter=parameter, decoder_sha256=A["B"]["DECODER_SHA"], index_sha256=SHA, field_sha256=SHA) + samples = [] + for step in p["steps"]: + samples.append(dict(lead_hours=step, valid_at=iso(A["stamp"](model_run) + step * 3600000), + **(dict(evidence=field("2t"), wind_10m=dict(evidence=dict(eastward=field("10u"), northward=field("10v"))), + precipitation=dict(evidence=field("tp"))) if key == "weather" else + dict(evidence=dict(pm2_5=field("PMTF"), pm10=field("PMTC")))))) + return dict(schema=p["schema"], kind="forecast", source=dict(provider=p["provider"], acquisition="origin-https", + model=p["model"], model_run_at=model_run, acquired_at=acquired_at or iso(CLOCK)), + area=dict(latitude=52.52, longitude=13.41, sharing="public-community"), samples=samples) + + +def prepare_fixture(viewer): + selection, forecasts = selected(), {} + for key, p in A["PRODUCTS"].items(): + forecasts[key] = forecast(key) + raw = A["B"]["encode"](forecasts[key]); A["write_new"](viewer / p["filename"], raw) + selection["products"][key]["publication"].update(bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + save(viewer / "selection.json", selection, 0o400) + return selection, forecasts def selected(): @@ -116,6 +148,10 @@ def joined_fixture(work): class Products(unittest.TestCase): + def setUp(self): + clock = patch.dict(A["prepare"].__globals__, observed_ms=lambda: CLOCK) + clock.start(); self.addCleanup(clock.stop) + def test_acquisition_commands_include_precipitation_and_independent_noaa_cycle_with_one_deadline(self): with tempfile.TemporaryDirectory() as temporary: base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) @@ -126,18 +162,7 @@ def execute(command, cwd, maximum, timeout): calls.append((command, timeout)) if command[-1] == "-V": return b"ecCodes Version 2.44.0\n" key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" - p = A["PRODUCTS"][key] - def field(parameter): - return dict(parameter=parameter, decoder_sha256=A["B"]["DECODER_SHA"], index_sha256=SHA, field_sha256=SHA) - samples = [] - for step in p["steps"]: - samples.append(dict(lead_hours=step, **(dict(evidence=field("2t"), - wind_10m=dict(evidence=dict(eastward=field("10u"), northward=field("10v"))), - precipitation=dict(evidence=field("tp"))) if key == "weather" else - dict(evidence=dict(pm2_5=field("PMTF"), pm10=field("PMTC")))))) - return json.dumps(dict(schema=p["schema"], source=dict(provider=p["provider"], acquisition="origin-https", - model=p["model"], model_run_at=command[command.index("--run")+1]), - area=dict(latitude=52.52, longitude=13.41, sharing="public-community"), samples=samples)).encode() + return json.dumps(forecast(key, command[command.index("--run")+1])).encode() clock = datetime.datetime(2026, 10, 11, 14, tzinfo=datetime.timezone.utc).timestamp() with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None), \ patch.dict(A["B"], verify_tree=lambda *args: None, manifest=lambda *args: {"version": 2}), \ @@ -151,15 +176,81 @@ def field(parameter): self.assertEqual(aq[aq.index("--run")+1], "2026-10-11T00:00:00.000Z") # Same cycle time is permissible when independently selected; # unlike their native samples, the times need not differ by force. - self.assertEqual(weather[weather.index("--steps")+1], "3,9,24") - self.assertEqual(aq[aq.index("--steps")+1], "3,6,12") + self.assertEqual(weather[weather.index("--steps")+1], "3,9,24,48") + self.assertEqual(aq[aq.index("--steps")+1], "3,6,12,24,48") self.assertEqual([call[1] for call in calls], [10, 360, 340]) self.assertTrue((root / "forecast.json").exists()); self.assertTrue((root / "air-quality.json").exists()) + def test_acquisition_preserves_future_native_coverage_across_midnight_and_noon(self): + cases = [ + ("2026-10-10T23:59:59.999Z", "2026-10-10T00:00:00.000Z", "2026-10-10T00:00:00.000Z"), + ("2026-10-11T00:00:00.000Z", "2026-10-10T12:00:00.000Z", "2026-10-10T00:00:00.000Z"), + ("2026-10-11T11:59:59.999Z", "2026-10-10T12:00:00.000Z", "2026-10-10T00:00:00.000Z"), + ("2026-10-11T12:00:00.000Z", "2026-10-11T00:00:00.000Z", "2026-10-11T00:00:00.000Z"), + ] + for when, weather_run, aq_run in cases: + with self.subTest(when=when), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) + source = base / "inputs"; source.mkdir(mode=0o700) + A["initialize"](root); A["write_new"](source / "INPUTS.json", b"fixture", 0o400) + started, received = A["stamp"](when), [] + def execute(command, cwd, maximum, timeout): + if command[-1] == "-V": return b"ecCodes Version 2.44.0\n" + key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" + value = forecast(key, command[command.index("--run")+1], iso(started + 2000)) + received.append(value); return json.dumps(value).encode() + with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None, + observed_ms=lambda: started + 2000), \ + patch.dict(A["B"], verify_tree=lambda *args: None, manifest=lambda *args: {"version": 2}), \ + patch.object(A["time"], "time", return_value=started / 1000), \ + patch.object(A["time"], "monotonic", side_effect=[100, 120, 480]): + A["acquisition"](source, SHA, root, "net:[1]", "net:[2]", 1000) + self.assertEqual([v["source"]["model_run_at"] for v in received], [weather_run, aq_run]) + for value in received: + self.assertGreater(A["stamp"](value["samples"][-1]["valid_at"]), started + 2000) + self.assertLess(A["stamp"](value["samples"][0]["valid_at"]), started + 2000) + + def test_utc_timestamps_require_real_dates_and_preserve_exact_milliseconds(self): + self.assertEqual(A["stamp"]("2026-10-11T14:00:00.001Z"), CLOCK + 1) + for value in (None, True, "2026-02-30T00:00:00.000Z", "2026-10-11T24:00:00.000Z", + "2026-10-11T14:00:00Z", "2026-10-11T14:00:00.000+00:00", "2026-10-11T14:00:00.0000Z"): + with self.subTest(value=value), self.assertRaises(ValueError): A["stamp"](value) + for key, value in (("weather", "2026-02-30T00:00:00.000Z"), ("weather", "2026-10-11T06:00:00.000Z"), + ("air_quality", "2026-10-11T12:00:00.000Z")): + bad = selected(); bad["products"][key]["query"]["run"] = value + with self.subTest(key=key, value=value), self.assertRaises(ValueError): A["validate_selection"](bad) + + def test_forecast_times_bind_acquisition_window_native_leads_and_future_coverage(self): + for key in A["PRODUCTS"]: + good = forecast(key); model_run = good["source"]["model_run_at"] + A["forecast_times"](good, key, model_run, CLOCK, (CLOCK, CLOCK)) + mutations = [(("kind",), "observation"), (("source", "model_run_at"), iso(CLOCK)), + (("source", "acquired_at"), iso(A["stamp"](model_run)-1)), + (("source", "acquired_at"), iso(CLOCK+1)), + (("samples", 0, "valid_at"), iso(A["stamp"](good["samples"][0]["valid_at"])+1)), + (("samples", 0, "lead_hours"), True), + (("samples", -1, "valid_at"), iso(CLOCK+1))] + for keys, changed in mutations: + value = copy.deepcopy(good); target = value + for part in keys[:-1]: target = target[part] + target[keys[-1]] = changed + with self.subTest(key=key, mutation=keys), self.assertRaises(ValueError): + A["forecast_times"](value, key, model_run, CLOCK) + backdated = copy.deepcopy(good); backdated["source"]["acquired_at"] = iso(CLOCK-1) + with self.assertRaises(ValueError): A["forecast_times"](backdated, key, model_run, CLOCK, (CLOCK, CLOCK)) + with self.assertRaises(ValueError): A["forecast_times"](good, key, model_run, CLOCK, (CLOCK+1, CLOCK)) + last = A["stamp"](good["samples"][-1]["valid_at"]) + A["forecast_times"](good, key, model_run, last-1) + for observed in (last, last+1): + with self.subTest(key=key, observed=observed), self.assertRaises(ValueError): + A["forecast_times"](good, key, model_run, observed) + query = A["fixed_query"](key, model_run) + with self.assertRaises(ValueError): A["future_query"](key, query, A["stamp"](model_run)-1) + def test_preparer_uses_both_receipts_and_separate_caches(self): with tempfile.TemporaryDirectory() as temporary: base = Path(temporary); viewer = base / "weather-viewer"; viewer.mkdir(mode=0o700) - selection = selected(); save(viewer / "selection.json", selection, 0o400) + selection, forecasts = prepare_fixture(viewer) def execute(command, cwd, maximum, timeout): self.assertEqual(command[command.index("--air-quality")+1], str(viewer / "air-quality.json")) self.assertEqual(command[command.index("--air-quality-receipt")+1], str(viewer / "air-quality-cold-receipt.json")) @@ -168,7 +259,8 @@ def execute(command, cwd, maximum, timeout): self.assertEqual(config["core"]["scratchParent"], config["airQuality"]["core"]["scratchParent"]) self.assertTrue(config["core"]["query"]["includePrecipitation"]) expected = dict(schema=2, core_sha256=SHA, display_sha256=SHA, - products={key: {"publication": value["publication"]} for key, value in selection["products"].items()}) + products={key: dict(publication=value["publication"], forecast=forecasts[key]) + for key, value in selection["products"].items()}) raw = A["B"]["encode"](expected); A["write_new"](viewer / "expected.json", raw, 0o400) return json.dumps(dict(prepared=True, expected_sha256=hashlib.sha256(raw).hexdigest(), display_sha256=SHA)).encode() with patch.dict(A["prepare"].__globals__, run=execute), patch.dict(A["B"], fingerprint=lambda *args: {"sha256": SHA}): @@ -176,6 +268,32 @@ def execute(command, cwd, maximum, timeout): base / "state-client/weather-cache", base / "state-client/air-quality-cache") self.assertTrue(result["prepared"]) + def test_preparer_rejects_unbound_cold_bytes_expected_forecast_and_exhausted_future(self): + for key in A["PRODUCTS"]: + for fault in ("length", "hash", "expected", "future"): + with self.subTest(key=key, fault=fault), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); viewer = base / "weather-viewer"; viewer.mkdir(mode=0o700) + selection, forecasts = prepare_fixture(viewer) + if fault in ("length", "hash"): + path = viewer / A["PRODUCTS"][key]["filename"] + raw = A["B"]["encode"](forecasts[key]) + altered = raw + b" " if fault == "length" else raw.replace(b"2026-10-10", b"2026-10-09", 1) + self.assertNotEqual(raw, altered) + if fault == "hash": self.assertEqual(len(raw), len(altered)) + path.unlink(); A["write_new"](path, altered) + if fault == "expected": forecasts[key]["source"]["acquired_at"] = iso(CLOCK-1) + observed = A["stamp"](forecasts[key]["samples"][-1]["valid_at"]) if fault == "future" else CLOCK + def execute(command, cwd, maximum, timeout): + expected = dict(schema=2, core_sha256=SHA, display_sha256=SHA, + products={name: dict(publication=value["publication"], forecast=forecasts[name]) + for name, value in selection["products"].items()}) + raw = A["B"]["encode"](expected); A["write_new"](viewer / "expected.json", raw, 0o400) + return json.dumps(dict(prepared=True, expected_sha256=hashlib.sha256(raw).hexdigest(), display_sha256=SHA)).encode() + with patch.dict(A["prepare"].__globals__, run=execute, observed_ms=lambda: observed), \ + patch.dict(A["B"], fingerprint=lambda *args: {"sha256": SHA}), self.assertRaises(ValueError): + A["prepare"](base / "source", viewer, base / "binary", base / "agent.sock", + base / "state-client/weather-cache", base / "state-client/air-quality-cache") + def test_partial_browser_report_is_retainable_but_never_success(self): failed = report(); failed.update(passed=False, phase="provider", checks={}, cleanup={}) failed["scope"].update(actual_drizzli_ui=False, real_core_adapter=False, separate_native_runs=False) @@ -197,8 +315,9 @@ def test_exact_selection_keeps_separate_native_runs_and_no_cross_product_alias(s bad = copy.deepcopy(value); bad["products"][key][group][field] = changed with self.subTest(field=field), self.assertRaises(ValueError): A["validate_selection"](bad) - def test_acquisition_requires_twelve_weather_and_six_native_particulate_fields(self): + def test_acquisition_requires_sixteen_weather_and_ten_native_particulate_fields(self): good = acquisition(); A["validate_acquisition"](good) + self.assertEqual([len(good["products"][key]["fields"]) for key in A["PRODUCTS"]], [16, 10]) for key in A["PRODUCTS"]: for field in ("missing", "duplicate", "wrong_parameter", "wrong_step"): value = copy.deepcopy(good); fields = value["products"][key]["fields"] @@ -244,8 +363,8 @@ def test_full_join_rejects_product_specific_origin_privacy_and_ui_failures(self) proof = A["evidence"](work, digest, SHA); A["validate_evidence_export"](proof) self.assertTrue(proof["scope"]["end_to_end_proven"]) self.assertFalse(proof["scope"]["browser_triggered_cold_fetch"]) - self.assertEqual(proof["products"]["weather"]["acquired_fields"], 12) - self.assertEqual(proof["products"]["air_quality"]["acquired_fields"], 6) + self.assertEqual(proof["products"]["weather"]["acquired_fields"], 16) + self.assertEqual(proof["products"]["air_quality"]["acquired_fields"], 10) def test_weather_only_capture_or_receipt_never_proves_air_quality(self): for suffix in ("cold-consumer", "selection", "privacy-client", "privacy-exit", "control", "cache-before"): @@ -255,6 +374,24 @@ def test_weather_only_capture_or_receipt_never_proves_air_quality(self): with patch.dict(A["cold_evidence"].__globals__, cache_state=lambda key, *args: caches[key]): with self.assertRaises((OSError, ValueError)): A["cold_evidence"](work, digest, SHA) + def test_final_future_check_works_after_viewer_cleanup_but_export_has_no_wall_clock_dependency(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, caches = joined_fixture(work) + self.assertFalse((work / "client-fixtures/weather-viewer").exists()) + last = min(A["stamp"](v["query"]["run"]) + 48 * 3600000 for v in selected()["products"].values()) + with patch.dict(A["evidence"].__globals__, cache_state=lambda key, *args: caches[key], observed_ms=lambda: last-1): + proof = A["evidence"](work, digest, SHA) + for observed in (last, last+1, 0): + with patch.dict(A["evidence"].__globals__, cache_state=lambda key, *args: caches[key], observed_ms=lambda: observed): + with self.subTest(observed=observed), self.assertRaises(ValueError): A["evidence"](work, digest, SHA) + def unexpected_clock(): raise AssertionError("historical export must not read wall clock") + with patch.dict(A["evidence"].__globals__, observed_ms=unexpected_clock), \ + patch.object(A["time"], "time", side_effect=unexpected_clock), \ + patch.object(A["time"], "time_ns", side_effect=unexpected_clock): + A["validate_evidence_export"](proof) + A["validate_acquisition"](acquisition()) + A["validate_cold_export"](A["read"](A["artifact"](work, "cold-evidence"))) + def test_cold_evidence_survives_missing_ui_without_claiming_end_to_end(self): with tempfile.TemporaryDirectory() as temporary: work = Path(temporary); digest, caches = joined_fixture(work) diff --git a/tests/integration/weather-products-delivery.py b/tests/integration/weather-products-delivery.py index f9e97e2c3..8e29ca9e0 100644 --- a/tests/integration/weather-products-delivery.py +++ b/tests/integration/weather-products-delivery.py @@ -31,12 +31,12 @@ "weather": dict(suffix="weather", name="disposable-public-community-weather", filename="forecast.json", manifest="manifest.bin", published="publish-receipt.json", cache="weather-cache", source_cache="source-cache", receipt="cold-receipt.json", script="forecast.mjs", schema="volparossa.forecast.v1", provider="ecmwf", - model="ifs", steps=[3, 9, 24], parameters=["2t", "10u", "10v", "tp"]), + model="ifs", steps=[3, 9, 24, 48], parameters=["2t", "10u", "10v", "tp"]), "air_quality": dict(suffix="air-quality", name="disposable-public-community-air-quality", filename="air-quality.json", manifest="air-quality-manifest.bin", published="air-quality-publish-receipt.json", cache="air-quality-cache", source_cache="air-quality-source-cache", receipt="air-quality-cold-receipt.json", script="air-quality.mjs", schema="volparossa.air-quality.forecast.v1", provider="noaa", model="gefs-aerosols", - steps=[3, 6, 12], parameters=["PMTF", "PMTC"]), + steps=[3, 6, 12, 24, 48], parameters=["PMTF", "PMTC"]), } BRIDGE_MODULES = {"scripts/serve-provider.mjs", "src/provider-http.mjs", "src/provider-core.mjs", "src/core-cache.mjs", "src/air-quality.mjs", "src/air-quality-provider.mjs", "src/air-quality-core.mjs"} @@ -67,6 +67,39 @@ def fixed_query(key, model_run): return result +def stamp(value): + require(type(value) is str and re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", value)) + moment = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return (moment - datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)) // datetime.timedelta(milliseconds=1) + + +def observed_ms(): + return time.time_ns() // 1000000 + + +def future_query(key, query, observed_at): + same(query, fixed_query(key, query["run"])) + run_at = stamp(query["run"]) + require(type(observed_at) is int and run_at <= observed_at < run_at + 48 * 3600000) + + +def forecast_times(value, key, model_run, observed_at, acquisition_window=None): + require(value["kind"] == "forecast" and value["source"]["model_run_at"] == model_run) + run_at, acquired_at = stamp(model_run), stamp(value["source"]["acquired_at"]) + require(type(observed_at) is int and run_at <= acquired_at <= observed_at) + samples = value["samples"] + same([sample["lead_hours"] for sample in samples], product(key)["steps"]) + for sample in samples: + require(type(sample["lead_hours"]) is int + and stamp(sample["valid_at"]) == run_at + sample["lead_hours"] * 3600000) + # Past native samples remain forecasts, never observations. The final lead + # must still be in the future at this actual acquisition/preparation check. + require(stamp(samples[-1]["valid_at"]) > observed_at) + if acquisition_window is not None: + started_at, finished_at = acquisition_window + require(started_at <= acquired_at <= finished_at == observed_at) + + def acquisition(source, manifest_sha, root, parent, client, group): private(root) require(root.name == "weather-publisher" and set(p.name for p in root.iterdir()) == {"passphrase", "scratch"}) @@ -77,8 +110,9 @@ def acquisition(source, manifest_sha, root, parent, client, group): require(run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10).strip() == b"ecCodes Version 2.44.0") now = int(time.time()) # Select independently completed cycles. NOAA's 00UTC cycle with an explicit - # twelve-hour availability margin is not - # rewritten to match IFS and its native 3/6/12h samples are not interpolated. + # twelve-hour availability margin is not rewritten to match IFS. Native + # samples are not interpolated; lead 48 preserves future coverage across + # midnight/noon boundaries without pretending the earlier leads are current. cycles = dict(weather=now // 43200 * 43200 - 43200, air_quality=(now - 43200) // 86400 * 86400) products = {} for key, p in PRODUCTS.items(): @@ -90,13 +124,15 @@ def acquisition(source, manifest_sha, root, parent, client, group): "--decoder-sha256", B["DECODER_SHA"], "--scratch-parent", str(root / "scratch")] if key == "weather": command += ["--model", "ifs", "--include-wind", "--include-precipitation"] remaining = deadline - time.monotonic(); require(remaining > 0) + started_at = observed_ms() raw = run(command, root, MAX_FORECAST, min(360, remaining)) + finished_at = observed_ms() value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) require(value["schema"] == p["schema"] and value["source"]["provider"] == p["provider"] and value["source"]["acquisition"] == "origin-https" and value["source"]["model"] == p["model"] and value["source"]["model_run_at"] == model_run) same(value["area"], dict(latitude=52.52, longitude=13.41, sharing="public-community")) - same([sample["lead_hours"] for sample in value["samples"]], p["steps"]) + forecast_times(value, key, model_run, finished_at, (started_at, finished_at)) require(not list((root / "scratch").iterdir())) fields = [] for sample in value["samples"]: @@ -135,6 +171,7 @@ def validate_selection(value): same(query, fixed_query(key, query["run"])) require(type(query["run"]) is str and re.fullmatch(r"\d{4}-\d{2}-\d{2}T(?:00|12):00:00\.000Z" if key == "weather" else r"\d{4}-\d{2}-\d{2}T00:00:00\.000Z", query["run"])) + stamp(query["run"]) require(value["products"]["weather"]["publication"]["manifest_id"] != value["products"]["air_quality"]["publication"]["manifest_id"]) return value @@ -213,7 +250,14 @@ def core_config(key, cache): and expected["core_sha256"] == core_sha and set(expected["products"]) == set(PRODUCTS) and hashlib.sha256(expected_raw).hexdigest() == result["expected_sha256"] and expected["display_sha256"] == result["display_sha256"]) - for key in PRODUCTS: same(expected["products"][key]["publication"], selected["products"][key]["publication"]) + for key, p in PRODUCTS.items(): + publication = selected["products"][key]["publication"] + same(expected["products"][key]["publication"], publication) + cold_raw = raw_file(viewer / p["filename"], MAX_FORECAST, 0o600) + require(len(cold_raw) == publication["bytes"] and hashlib.sha256(cold_raw).hexdigest() == publication["sha256"]) + forecast = json.loads(cold_raw, object_pairs_hook=B["B"]["unique_pairs"]) + same(expected["products"][key]["forecast"], forecast) + forecast_times(forecast, key, selected["products"][key]["query"]["run"], observed_ms()) return result @@ -403,7 +447,8 @@ def validate_cold_export(value): require(set(item) == {"forecast_sha256", "forecast_bytes", "manifest_id", "cold_receipt_sha256", "acquired_fields", "protected_cold"} and all(hex64(item[k]) for k in ("forecast_sha256", "manifest_id", "cold_receipt_sha256")) and type(item["forecast_bytes"]) is int and 0 < item["forecast_bytes"] <= MAX_FORECAST) - same(item["acquired_fields"], 12 if key == "weather" else 6) + p = product(key) + same(item["acquired_fields"], len(p["steps"]) * len(p["parameters"])) transfer = item["protected_cold"] require(type(transfer["providers_used"]) is int and transfer["providers_used"] in (1, 2)) same(transfer, dict(peer_bytes=item["forecast_bytes"], providers_used=transfer["providers_used"], @@ -462,6 +507,12 @@ def evidence(work, manifest_sha, binary_sha): initial_dom_display_sha256=report["expected"]["display_sha256"], refresh_dom_display_sha256=report["expected"]["display_sha256"])) same(report["cleanup"], dict(browser_exited=True, bridge_exited=True, private_state_removed=True, normal_browser_stop=True, normal_bridge_stop=True)) same(report["outer_cleanup"], dict(namespace_wrapper_exited=True, private_state_removed=True, client_namespace_unchanged=True)) + # Raw viewer inputs have already been removed. Acquisition and preparation + # bound their native times to these queries and the browser's exact hashes. + # Check future coverage when creating the joined proof, never when merely + # validating/exporting this historical evidence later. + observed_at = observed_ms() + for key in PRODUCTS: future_query(key, selected["products"][key]["query"], observed_at) result = dict(schema=2, success=True, kind="volparossa-weather-products-delivery", bundle_manifest_sha256=manifest_sha, core_sha256=binary_sha, decoder_sha256=B["DECODER_SHA"], products=retained["products"], browser_report_sha256=hashlib.sha256(raw_file(artifact(work, "original-report"))).hexdigest(), From cc5f355843dd35f7b30a162759cbe37b89904c9d Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:53:00 +0200 Subject: [PATCH 21/49] test: preserve archive owner during weather profile dispatch --- docs/IMPLEMENTATION_STATUS.md | 40 ++++++++-- .../integration/test-weather-client-bundle.py | 21 ++++++ .../test-weather-products-delivery-shell.py | 74 ++++++++++++++++++- tests/integration/weather-client-delivery.sh | 22 +++++- 4 files changed, 147 insertions(+), 10 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 0db568e28..9d4024bdc 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,13 +29,15 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Recover missing chunks from late-discovered complementary providers within one bounded operation, then verify the real topology including the earlier adaptive-provider correction. | The local run on `c8b57d94` reached the base fetch but discovered only one of two complementary stores. The separate CI run failed earlier, before discovering the required Exit. Neither reached adaptive transfer or Weather delivery. Keep these failures distinct; do not replace the required Exit or weaken completeness checks. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The combined UI/bridge passes with synthetic data; the joined driver and immutable input bundle pass targeted checks. Real two-product delivery depends on the shared discovery repair and a new exact-source trial. Existing temperature/wind proof and the separate NOAA source trial do not cover this delivery. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The latest local run on `5f82f619` completes the base two-provider fetch with measured overlap, using one discovery query. It does not exercise late rediscovery or reach adaptive transfer. The separate CI primary-Exit discovery failure remains unresolved. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run stops before acquisition at an archive-owner mismatch during profile dispatch. The corrected invocation passes 43 focused tests; the joined path still needs a new exact-source trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather` | Build and exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | Reuse the provider contracts and core retrieval path; client adaptation, a reproducible client build and its own runtime proof remain required. Do not create a second network scheduler or source-acquisition implementation. | -Prioritize the shared discovery failure before another unchanged live run or a -second client integration. The Drizz.li/Weather candidate and frozen input bundle -remain intact; Overmorrow's client work is still required. Keep shared interface +Run the corrected archive-owner dispatch in a new exact-source trial, retaining +the separate discovery and adaptive-transfer checks. This follows the latest +trial's progress beyond the earlier base-fetch failure, without claiming that +discovery convergence is solved. The Drizz.li/Weather candidate and frozen input +bundle remain intact; Overmorrow's client work is still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. @@ -538,6 +540,34 @@ deadline and cleanup before fallback. These checks do not establish live DHT con resolve the separate CI primary-Exit failure. A new exact-source topology trial remains required before either live failure is described as fixed. +The next local run on `5f82f619cca8c4e6e621d01bcc3269e41a499ba8` builds +successfully and completes the base native fetch: 2,097,275 bytes in nine chunks +from the two complementary providers, with no origin bytes. Independent checks +of the retained receipts, selected route, control traffic and drained packet +captures pass. The two bulk-payload windows overlap by 643,330,579 ns. Both +offers arrived in one discovery query, so this run does not prove late-provider +recovery, per-subflow bulk aggregation or a speed improvement. + +The run then stops before Weather acquisition with `WEATHER_CLIENT_PROFILE_INVALID`, +recorded under the preceding `content-provider-filter-cold` phase. Preflight +verified the 0600 archive as its VM owner, but later profile dispatch called the +strict reader as root, which rejects that non-root owner. This is a test-driver +ownership-context error, not evidence of a malformed frozen bundle. No Weather +or adaptive-transfer success is claimed. The report remains at +`.git/ci-evidence/weather-products-delivery-02.w9hxuJ/output/content-provider-smoke.json`, +SHA-256 `62adcc7bc20002ead79e5cd1a75fb8d5022f9dabe9dc93a7814343febd6916b9`. +Cleanup retains zero owned objects and unchanged guest-root network state. +The development host's DNS and IPv4/IPv6 route hashes also match their pre-run +values; a host-firewall comparison was unavailable without additional privilege. + +Profile dispatch now mirrors preflight: root-owned archives are checked as root, +and VM-user-owned archives as that user. Other owners and failed checks remain +rejected; parser rules, archive bytes and downstream root-owned unpacking are +unchanged. The profile check also has its own reported phase. All 43 focused +checks pass (22 bundle, nine products shell and 12 legacy shell tests), including +the reproduced owner mismatch and closed failure paths; ShellCheck passes too. +This correction still requires a new joined live trial. + ### Citizen procedure simulation Citizen [PR1](https://github.com/VOLPAROSSA/volparossa-citizen/pull/1) is merged diff --git a/tests/integration/test-weather-client-bundle.py b/tests/integration/test-weather-client-bundle.py index 4050ee27b..bcef56dae 100644 --- a/tests/integration/test-weather-client-bundle.py +++ b/tests/integration/test-weather-client-bundle.py @@ -103,6 +103,27 @@ def test_runtime_version_action_checks_whole_archive_without_executing_or_changi self.assertEqual(rejected.getvalue(), "") archive.unlink() + def test_runtime_profile_keeps_invoking_owner_boundary(self): + if os.geteuid() == 0: + self.skipTest("ownership regression uses an unprivileged test-owned archive") + for version in (1, 2): + self.files = self.profile_files(version) + archive, expected = self.archive(version=version) + before = archive.read_bytes() + owner = archive.stat().st_uid + # VM inputs belong to vpci: reading those as root must not bypass + # the existing parser's owner check. No real uid/chown is changed. + with self.subTest(version=version), patch.object(W["os"], "geteuid", return_value=0): + with self.assertRaises(ValueError): + W["checked_archive"](archive, expected, report_version=True) + with patch.object(W["os"], "geteuid", return_value=owner): + profile = W["checked_archive"](archive, expected, report_version=True) + self.assertEqual(profile["bundle_version"], version) + self.assertEqual(archive.read_bytes(), before) + self.assertEqual(archive.stat().st_uid, owner) + self.assertEqual(archive.stat().st_mode & 0o777, 0o600) + archive.unlink() + def test_v2_roundtrip_is_readonly_and_never_executes_captured_inputs(self): self.files = self.profile_files(2) archive, expected = self.archive(version=2) diff --git a/tests/integration/test-weather-products-delivery-shell.py b/tests/integration/test-weather-products-delivery-shell.py index 1c19e3669..80b0d3cd8 100644 --- a/tests/integration/test-weather-products-delivery-shell.py +++ b/tests/integration/test-weather-products-delivery-shell.py @@ -104,15 +104,85 @@ def test_verified_closed_dispatch_before_any_v1_work(self): source_directory=$WORK weather_client_bundle=inert weather_client_manifest_sha256=inert +stat() { printf '0\\n'; } python3() { printf '%s\n' 'JSON'; } fail() { exit 77; } run_selected -'''.replace("JSON", value), source="run_selected() {\n" + head + '\nprintf "v1\\n"\n}\n') +'''.replace("JSON", value), source=source + "\nrun_selected() {\n" + head + '\nprintf "v1\\n"\n}\n') self.assertEqual(result.returncode, 0 if type(version) is int and version in (1, 2) else 77) if result.returncode == 0: self.assertEqual(result.stdout, f"v{version}\n") - self.assertLess(head.index("runtime-version"), head.index(".bundle_version")) + self.assertLess(head.index("weather_client_runtime_profile"), head.index(".bundle_version")) self.assertLess(head.index(".bundle_version"), head.index(' . "$source_directory')) + def test_profile_owner_boundary_and_helper_failure_are_fail_closed(self): + source = LEGACY.read_text() + head = source.split("weather_client_delivery_run() {", 1)[1].split(" PHASE=weather-client-delivery-prepare", 1)[0] + helper_root = self.root / "tests/integration" + helper_root.mkdir(parents=True) + (helper_root / "weather-products-delivery.sh").write_text( + 'weather_products_delivery_run() { printf "v2\\n"; }\n') + scenarios = (("root", 0, 0, 0), ("vpci", 1000, 0, 0), ("other", 985, 0, 77), + ("stat-failure", 1000, 0, 77), ("missing-account", 1000, 0, 77), + ("runuser-failure", 1000, 0, 77), ("helper-failure", 0, 1, 77), + ("helper-failure", 1000, 1, 77)) + for version in (1, 2): + for scenario, owner, helper_status, expected_status in scenarios: + with self.subTest(version=version, scenario=scenario, owner=owner): + result = self.call(''' +source_directory=$WORK +weather_client_bundle=$WORK/inert.tar +weather_client_manifest_sha256=inert-digest +PHASE=content-provider-filter-cold +test_scenario=SCENARIO +test_owner=OWNER +test_effective_uid=0 +stat() { + [ "$*" = "-Lc %u $weather_client_bundle" ] || exit 90 + [ "$test_scenario" != stat-failure ] || return 1 + printf '%s\n' "$test_owner" +} +id() { + [ "$*" = '-u vpci' ] || exit 91 + [ "$test_scenario" != missing-account ] || return 1 + printf '1000\n' +} +python3() { + [ "$*" = "-B $source_directory/tests/integration/weather-client-bundle.py runtime-version $weather_client_bundle $weather_client_manifest_sha256" ] || exit 92 + printf 'python:%s\n' "$test_effective_uid" >> "$WORK/profile-calls" + # The real parser accepts root-owned or invoking-owner files, not a + # vpci-owned input merely because the invoking process has root access. + [ "$test_owner" = 0 ] || [ "$test_owner" = "$test_effective_uid" ] || return 93 + printf '{"bundle_version":VERSION}\n' + return HELPER_STATUS +} +runuser() { + [ "$1 $2 $3" = '-u vpci --' ] || exit 94 + printf 'runuser\n' >> "$WORK/profile-calls" + [ "$test_scenario" != runuser-failure ] || return 1 + shift 3 + test_effective_uid=1000 + "$@" +} +fail() { printf '%s %s\n' "$PHASE" "$1" >&2; exit 77; } +run_selected +'''.replace("SCENARIO", scenario).replace("OWNER", str(owner)) + .replace("VERSION", str(version)).replace("HELPER_STATUS", str(helper_status)), + source=source + "\nrun_selected() {\n" + head + '\nprintf "v1\\n"\n}\n') + self.assertEqual(result.returncode, expected_status, result.stdout + result.stderr) + if expected_status: + self.assertEqual(result.stdout, "") + self.assertEqual(result.stderr.strip(), + "weather-client-delivery-profile WEATHER_CLIENT_PROFILE_INVALID") + else: + self.assertEqual(result.stdout, f"v{version}\n") + calls = self.root / "profile-calls" + observed = calls.read_text().splitlines() if calls.exists() else [] + expected = [] if scenario in ("other", "stat-failure", "missing-account") else ( + ["runuser"] if scenario == "runuser-failure" else + ["python:0"] if owner == 0 else ["runuser", "python:1000"]) + self.assertEqual(observed, expected) + if calls.exists(): calls.unlink() + def test_publication_cleanup_independent_captures_and_retained_proof_order(self): source = SOURCE.read_text() run = source.split("weather_products_delivery_run() {", 1)[1] diff --git a/tests/integration/weather-client-delivery.sh b/tests/integration/weather-client-delivery.sh index b22265a7a..1e67b2ba9 100644 --- a/tests/integration/weather-client-delivery.sh +++ b/tests/integration/weather-client-delivery.sh @@ -92,15 +92,31 @@ weather_client_verify_tree() { "$weather_client_directory" "$weather_client_manifest_sha256" } +weather_client_runtime_profile() { + # Match the topology preflight: the guest input may still belong to vpci. + # Root readability is not parser ownership; do not relax its owner check or + # retry a failed unprivileged validation with broader authority. + weather_profile_uid=$(stat -Lc '%u' "$weather_client_bundle") || return 1 + if [ "$weather_profile_uid" -eq 0 ]; then + python3 -B "$source_directory/tests/integration/weather-client-bundle.py" \ + runtime-version "$weather_client_bundle" "$weather_client_manifest_sha256" + elif weather_profile_vpci_uid=$(id -u vpci) \ + && [ "$weather_profile_uid" -eq "$weather_profile_vpci_uid" ]; then + runuser -u vpci -- python3 -B "$source_directory/tests/integration/weather-client-bundle.py" \ + runtime-version "$weather_client_bundle" "$weather_client_manifest_sha256" + else + return 1 + fi +} + weather_client_delivery_run() { + PHASE=weather-client-delivery-profile if [ -z "${weather_client_bundle:-}" ] || [ -z "${weather_client_manifest_sha256:-}" ]; then fail WEATHER_CLIENT_INPUTS_MISSING fi # Select only after the complete immutable archive has been validated. The # legacy profile must keep its original driver and evidence contract. - weather_profile=$(python3 -B "$source_directory/tests/integration/weather-client-bundle.py" \ - runtime-version "$weather_client_bundle" "$weather_client_manifest_sha256") \ - || fail WEATHER_CLIENT_PROFILE_INVALID + weather_profile=$(weather_client_runtime_profile) || fail WEATHER_CLIENT_PROFILE_INVALID weather_profile_version=$(printf '%s\n' "$weather_profile" | \ jq -er '.bundle_version | select(type == "number" and (. == 1 or . == 2))') \ || fail WEATHER_CLIENT_PROFILE_INVALID From a1e77b4e1f33553a87f8a3ccdfbe7069a6cae403 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:28:39 +0200 Subject: [PATCH 22/49] test: retain closed Weather acquisition failure evidence --- docs/IMPLEMENTATION_STATUS.md | 57 ++++- tests/integration/kvm-alpha-topology.sh | 10 +- tests/integration/run-alpha-topology-vm.sh | 73 +++++- .../integration/test-weather-client-runner.py | 147 ++++++++++- .../test-weather-products-delivery-shell.py | 65 +++++ .../test-weather-products-delivery.py | 136 ++++++++++ .../integration/weather-products-delivery.py | 232 ++++++++++++++---- .../integration/weather-products-delivery.sh | 23 +- 8 files changed, 676 insertions(+), 67 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 9d4024bdc..6597eaca1 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,14 +29,14 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The latest local run on `5f82f619` completes the base two-provider fetch with measured overlap, using one discovery query. It does not exercise late rediscovery or reach adaptive transfer. The separate CI primary-Exit discovery failure remains unresolved. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run stops before acquisition at an archive-owner mismatch during profile dispatch. The corrected invocation passes 43 focused tests; the joined path still needs a new exact-source trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The latest local run on `cc5f3558` completes the base two-provider fetch with measured overlap, using one discovery query. It does not exercise late rediscovery or reach adaptive transfer. The separate CI primary-Exit discovery failure remains unresolved. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather` | Build and exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | Reuse the provider contracts and core retrieval path; client adaptation, a reproducible client build and its own runtime proof remain required. Do not create a second network scheduler or source-acquisition implementation. | -Run the corrected archive-owner dispatch in a new exact-source trial, retaining -the separate discovery and adaptive-transfer checks. This follows the latest -trial's progress beyond the earlier base-fetch failure, without claiming that -discovery convergence is solved. The Drizz.li/Weather candidate and frozen input +Respect the observed source rate limit instead of repeatedly relaunching the +joined trial or routing requests through other peers. Continue the separate +late-provider recovery proof, which needs no weather downloads; the successful +base fetch does not prove late discovery convergence. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's client work is still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; @@ -566,7 +566,50 @@ rejected; parser rules, archive bytes and downstream root-owned unpacking are unchanged. The profile check also has its own reported phase. All 43 focused checks pass (22 bundle, nine products shell and 12 legacy shell tests), including the reproduced owner mismatch and closed failure paths; ShellCheck passes too. -This correction still requires a new joined live trial. +The subsequent joined trial verifies that correction, as described below. + +The next local run on `cc5f355843dd35f7b30a162759cbe37b89904c9d` passes +profile dispatch, source-tree verification and publisher initialization, then +fails at `weather-products-delivery-acquire` with +`WEATHER_PRODUCTS_ACQUISITION_FAILED`. Before that failure, the base protected +fetch again reconstructs 2,097,275 bytes in nine chunks from two complementary +providers. Independent route, control and packet-capture checks pass; bulk-payload +windows overlap by 456,007,223 ns. Both offers arrive in one discovery query, +so late rediscovery, adaptive transfer and Weather delivery remain unproved. + +The failure report is retained at +`.git/ci-evidence/weather-products-delivery-03.TTYTlI/output/content-provider-smoke.json`, +SHA-256 `011636021ae8d6d55db208b5991b16c34807f6c427744bbbe70e36639e96ac62`. +Cleanup reports zero owned objects and unchanged guest-root network state; +the VM and temporary overlay are removed. Development-host DNS and IPv4/IPv6 +route hashes match the pre-run values. This does not add a development-host +firewall comparison. + +The retained report does not identify the acquisition substep that failed. +The helper emits only a generic failure, and the success-only atomic exporter +rejects that acquisition record before copying any Weather receipts. The +topology and emergency collectors then lose otherwise valid Weather diagnostics. +The correction separates closed failure diagnostics from atomic success evidence +and retains independently validated tree/cleanup receipts without exporting raw +process logs. Nine fixed acquisition stages identify the failing product and +step; the shell retains command exit status, including missing or invalid +diagnostics. Both collectors require a complete, validated failure export and +preserve failure status. The existing runner still discards subprocess stderr, +so stage diagnostics do not identify native upstream error codes. Seventy-eight +focused tests pass: 23 products helper, 11 products shell, 18 runner, 14 legacy +helper and 12 legacy shell tests. Independent review passes, shell syntax is valid +and comparison against the committed baseline finds no new ShellCheck findings. + +A separate native source probe uses the same verified frozen inputs, IFS run +`2026-10-11T00:00:00.000Z`, requested fields and leads. It stops before decoding +with `EXACT_RANGE_REQUIRED`. One bounded header check at `2026-10-11T20:23:26.986Z` +validates the index but receives HTTP 429 for the first 654,404-byte field range; +the response body is cancelled without consumption. `Retry-After` was not +captured, so its value is unknown. The closed diagnostic remains in +`.git/ci-evidence/weather-ecmwf-source-diagnostic.apV8R0/http-header-diagnostic.json`, +SHA-256 `504fae8c9c230ba502ef568acfa5f4c5cc78a1441832b6ebfda0fd75a968235a`. +This establishes source rate limiting during the probe, not the lost VM error's +exact cause. No further source retry or joined VM was launched after that check. ### Citizen procedure simulation diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 650a45ece..e4768ea9a 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1620,8 +1620,11 @@ capture_host_state() { } copy_artifacts() { + artifact_copy_status=0 if [ -n "$weather_client_bundle" ] && command -v weather_client_delivery_copy_artifacts >/dev/null 2>&1; then - weather_client_delivery_copy_artifacts || return 1 + # Keep the failed closed export visible to final acceptance, but retain + # independent safe diagnostics even when a Weather artifact is invalid. + weather_client_delivery_copy_artifacts || artifact_copy_status=1 fi if [ -n "$browser_filter_bundle" ] && command -v browser_filter_delivery_copy_artifacts >/dev/null 2>&1; then browser_filter_delivery_copy_artifacts || return 1 @@ -1632,7 +1635,7 @@ copy_artifacts() { [ ! -f "$WORK/$artifact" ] || [ -L "$WORK/$artifact" ] || \ install -o "$OUTPUT_UID" -g "$OUTPUT_GID" -m 0600 "$WORK/$artifact" "$output_directory/$artifact" || return 1 done - return 0 + return "$artifact_copy_status" fi for artifact in \ benchmark-selection-draws.jsonl a02-selected-paths.json \ @@ -1751,9 +1754,10 @@ copy_artifacts() { host-state-before.json host-state-after.json a15-evidence.json; do if [ -f "$WORK/$artifact" ] && [ ! -L "$WORK/$artifact" ]; then install -o "$OUTPUT_UID" -g "$OUTPUT_GID" -m 0600 \ - "$WORK/$artifact" "$output_directory/$artifact" + "$WORK/$artifact" "$output_directory/$artifact" || artifact_copy_status=1 fi done + return "$artifact_copy_status" } # shellcheck source=tests/integration/benchmark-selection.sh diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 45fafa43a..49f33e4c4 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1107,6 +1107,8 @@ WEATHER_CLIENT_HELPERS = ("weather-client-delivery.py", "weather-client-bundle.p "browser-filter-bundle.py", "content-provider-https-smoke.py", "content-network-smoke.py") WEATHER_PRODUCTS_NAMES = {name.replace("weather-client-delivery-", "weather-products-delivery-", 1) for name in WEATHER_CLIENT_NAMES} +WEATHER_PRODUCTS_FAILURE_NAMES = {f"weather-products-delivery-{suffix}.json" for suffix in ( + "acquisition-failure", "export-status", "tree-before", "input-cleanup")} WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", *WEATHER_CLIENT_HELPERS) @@ -1153,18 +1155,85 @@ def weather_client_candidates(root, label, helper, target): "weather-products-delivery.py": WEATHER_PRODUCTS_NAMES}.get(helper.name) if names is None: return [] + action = "copy-artifacts" + if helper.name == "weather-products-delivery.py": + try: + (root / "weather-products-delivery-acquisition-failure.json").lstat() + except FileNotFoundError: + pass + except OSError: + return [] + else: + # Marker trust is checked by the immutable helper, never inferred + # from existence; unsafe markers have no success-export fallback. + action = "copy-failure-artifacts" validated = Path(tempfile.mkdtemp(prefix="weather-client-validated.", dir=target)) + export_status = 0 try: - subprocess.run(["/usr/bin/python3", "-I", "-B", str(helper), "copy-artifacts", str(root), + subprocess.run(["/usr/bin/python3", "-I", "-B", str(helper), action, str(root), str(validated), str(os.getuid()), str(os.getgid())], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd="/", env={"PATH": "/usr/bin:/bin", "LC_ALL": "C"}, timeout=3, check=True) + except subprocess.CalledProcessError as error: + if action != "copy-failure-artifacts" or error.returncode != 1: + return [] + export_status = 1 except (OSError, subprocess.SubprocessError): return [] + if action == "copy-failure-artifacts": + names = weather_failure_exports(validated, export_status) return [(validated / name, f"{label}/{name}") for name in sorted(names) if (validated / name).is_file() and not (validated / name).is_symlink()] +def weather_failure_exports(validated, returncode): + """Keep only a completed dedicated failure export, never a crashed partial write.""" + status_name = "weather-products-delivery-export-status.json" + + def unique(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate status key") + value[key] = item + return value + + def safe_file(path): + info = path.lstat() + return (stat.S_ISREG(info.st_mode) and info.st_nlink == 1 + and info.st_uid == os.getuid() and info.st_gid == os.getgid() + and stat.S_IMODE(info.st_mode) == 0o600 and 0 < info.st_size <= 65536) + + try: + status_path = validated / status_name + if not safe_file(status_path): + return set() + observed = read_tail(status_path, 65536) + if observed is None or observed[1] > 65536: + return set() + value = json.loads(observed[0], object_pairs_hook=unique) + if (type(value) is not dict or set(value) != {"schema", "kind", "complete", "success", "exported", "rejected"} + or type(value["schema"]) is not int or value["schema"] != 2 + or value["kind"] != "weather-products-failure-export" + or value["complete"] is not True or value["success"] is not (returncode == 0)): + return set() + allowed = {"acquisition-failure", "tree-before", "input-cleanup"} + exported, rejected = value["exported"], value["rejected"] + if (type(exported) is not list or type(rejected) is not list + or not all(type(item) is str for item in exported + rejected) + or len(exported) != len(set(exported)) or len(rejected) != len(set(rejected)) + or not set(exported) <= allowed or not set(rejected) <= allowed | {"acquisition"} + or set(exported) & set(rejected) or "acquisition-failure" not in exported + or value["success"] is not (not rejected)): + return set() + names = {f"weather-products-delivery-{suffix}.json" for suffix in exported} | {status_name} + if not names <= WEATHER_PRODUCTS_FAILURE_NAMES or not all(safe_file(validated / name) for name in names): + return set() + return names + except (OSError, ValueError, TypeError): + return set() + + def browser_filter_candidates(root, label, helper, target): # Reuse the closed recursive JSON validator, never import the worker's # Browser tree or copy its logs. A failed/missing validator yields no files. @@ -1462,7 +1531,7 @@ def collect(home, opt, revision, scenario, guest_status, if observed is None: continue data, original_size = observed - if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES and original_size > len(data): + if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES | WEATHER_PRODUCTS_FAILURE_NAMES and original_size > len(data): continue # Preserve complete validated original bytes, never a JSON tail. if scenario == "browser-network" and path.name in browser_names and original_size > len(data): # A truncated JSON tail is not reconstructible structured evidence. diff --git a/tests/integration/test-weather-client-runner.py b/tests/integration/test-weather-client-runner.py index 109af7614..fb6dd9be4 100644 --- a/tests/integration/test-weather-client-runner.py +++ b/tests/integration/test-weather-client-runner.py @@ -140,6 +140,53 @@ def test_control_observer_accepts_only_the_one_new_weather_prefix(self): capture_output=True, timeout=5) self.assertEqual(result.returncode, expected, result.stderr) + def test_failed_weather_export_retains_safe_general_artifacts_and_failure_status(self): + source = (HERE / "kvm-alpha-topology.sh").read_text() + body = source.split("\ncopy_artifacts() {\n", 1)[1].split("\n}\n", 1)[0] + function = "copy_artifacts() {\n" + body + "\n}\n" + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + work = root / "work" + work.mkdir() + for name in ("host-state-before.json", "host-state-after.json", "a15-evidence.json"): + (work / name).write_bytes(b'{"safe":true}\n') + (work / "status-client.txt").write_bytes(b"connected: false\n") + for name in ("weather-products-delivery-acquisition.err", "forecast.json", "config.json"): + (work / name).write_bytes(b"PRIVATE CANARY") + (work / "logs-client.txt").symlink_to(work / "config.json") + for scenario in ("content-provider", "browser-network"): + for export_status in (0, 1): + with self.subTest(scenario=scenario, export_status=export_status): + output = root / f"output-{scenario}-{export_status}" + output.mkdir() + script = "set -eu\n" + function + ''' +WORK=$1 +output_directory=$2 +OUTPUT_UID=$3 +OUTPUT_GID=$4 +scenario=$5 +weather_client_bundle=/selected +browser_filter_bundle= +weather_client_delivery_copy_artifacts() { return "$6"; } +if copy_artifacts; then actual=0; else actual=$?; fi +printf '%s\\n' "$actual" +''' + # A shell function has its own positional arguments. + script = script.replace('return "$6"', f"return {export_status}") + result = subprocess.run(["/bin/sh", "-c", script, "fixture", str(work), str(output), + str(os.getuid()), str(os.getgid()), scenario], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.decode().strip(), str(export_status)) + expected = {"host-state-before.json", "host-state-after.json", "a15-evidence.json"} + if scenario == "content-provider": expected.add("status-client.txt") + self.assertEqual({path.name for path in output.iterdir()}, expected) + for path in output.iterdir(): + self.assertEqual(path.read_bytes(), (work / path.name).read_bytes()) + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) + self.assertEqual((path.stat().st_uid, path.stat().st_gid), (os.getuid(), os.getgid())) + self.assertNotIn(b"PRIVATE CANARY", path.read_bytes()) + class WeatherEmergencyExport(unittest.TestCase): @classmethod @@ -209,6 +256,56 @@ def inert_copy(command, **kwargs): with patch.object(self.api["subprocess"], "run", side_effect=AssertionError("must not execute")): self.assertEqual(self.api["weather_client_candidates"](root, "work-1", root / "unknown.py", target), []) + def test_products_failure_candidates_require_completed_closed_export(self): + for outcome in ("success", "rejected", "missing-status", "bad-status", "float-schema", "unsafe-status", "missing-file", + "duplicate-status-key", "signal", "timeout", "oserror", "invalid-marker"): + with self.subTest(outcome=outcome), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + target = root / "output" + target.mkdir() + marker = root / "weather-products-delivery-acquisition-failure.json" + # The immutable helper must validate even a dangling symlink; + # selection must never silently fall back to the success path. + marker.symlink_to("missing-marker") + + def inert_copy(command, **kwargs): + self.assertEqual(command[4:6], ["copy-failure-artifacts", str(root)]) + self.assertEqual(kwargs["stderr"], subprocess.DEVNULL) + copied = Path(command[6]) + if outcome != "invalid-marker": + exported = ["acquisition-failure", "tree-before", "input-cleanup"] + for suffix in exported: + path = copied / f"weather-products-delivery-{suffix}.json" + path.write_text('{"inert":true}') + path.chmod(0o600) + (copied / "forecast.json").write_text("PRIVATE CANARY") + (copied / "weather-products-delivery-acquisition.err").write_text("PRIVATE CANARY") + rejected = ["acquisition"] if outcome == "rejected" else [] + status = dict(schema=2, kind="weather-products-failure-export", complete=True, + success=not rejected, exported=exported, rejected=rejected) + if outcome == "bad-status": status["private"] = "PRIVATE CANARY" + if outcome == "float-schema": status["schema"] = 2.0 + if outcome != "missing-status": + path = copied / "weather-products-delivery-export-status.json" + raw = json.dumps(status) + if outcome == "duplicate-status-key": raw = '{"schema":2,' + raw[1:] + path.write_text(raw) + path.chmod(0o644 if outcome == "unsafe-status" else 0o600) + if outcome == "missing-file": + (copied / "weather-products-delivery-input-cleanup.json").unlink() + if outcome == "timeout": raise subprocess.TimeoutExpired(command, 3) + if outcome == "oserror": raise OSError("inert") + if outcome in {"rejected", "missing-status", "invalid-marker", "signal"}: + raise subprocess.CalledProcessError(-15 if outcome == "signal" else 1, command) + + with patch.object(self.api["subprocess"], "run", inert_copy): + candidates = self.api["weather_client_candidates"](root, "work-1", + root / "weather-products-delivery.py", target) + expected = ({"work-1/" + name for name in self.api["WEATHER_PRODUCTS_FAILURE_NAMES"]} + if outcome in {"success", "rejected"} else set()) + self.assertEqual({name for _, name in candidates}, expected) + self.assertFalse(any(b"PRIVATE CANARY" in path.read_bytes() for path, _ in candidates)) + def acquisition(self): return dict(schema=1, success=True, origin_https=True, acquisition_outside_overlay=True, upstream_signature_attested=False, synthetic_weather=False, @@ -227,10 +324,11 @@ def products_acquisition(self): index_sha256=SHA, field_sha256=SHA) for step in steps for parameter in parameters]) for name, steps, parameters in ( - ("weather", (3, 9, 24), ("2t", "10u", "10v", "tp")), - ("air_quality", (3, 6, 12), ("PMTF", "PMTC")))}) + ("weather", (3, 9, 24, 48), ("2t", "10u", "10v", "tp")), + ("air_quality", (3, 6, 12, 24, 48), ("PMTF", "PMTC")))}) - def collect(self, raw, limit=None, published=None, late_invalid=False, products=False): + def collect(self, raw, limit=None, published=None, late_invalid=False, products=False, + failure_marker=None, invalid_cleanup=False): with tempfile.TemporaryDirectory() as temporary: prefix = "weather-products-delivery" if products else "weather-client-delivery" base = Path(temporary).resolve() @@ -247,6 +345,16 @@ def collect(self, raw, limit=None, published=None, late_invalid=False, products= source = work / (prefix + "-acquisition.json") source.write_bytes(raw) source.chmod(0o600) + if failure_marker is not None: + values = {"acquisition-failure": failure_marker, + "tree-before": dict(version=1, action="verify-tree", manifest_sha256=SHA, + files=193, runtime_executed=False), + "input-cleanup": ({"private": "PRIVATE CANARY"} if invalid_cleanup else + dict(directory_removed=True, files_removed=True))} + for suffix, value in values.items(): + path = work / f"{prefix}-{suffix}.json" + path.write_text(json.dumps(value)) + path.chmod(0o600) if late_invalid: invalid = work / (prefix + "-evidence.json") invalid.write_text('{"unknown":"PRIVATE CANARY"}') @@ -283,6 +391,39 @@ def test_products_valid_receipt_survives_exactly_without_inventing_joined_succes self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) self.assertFalse(any("weather-client-delivery" in name for name in files)) + def test_products_dedicated_failure_exports_survive_real_helper_and_invalid_sibling(self): + diagnostic = dict(schema=2, kind="weather-products-acquisition-diagnostic", success=False, + stage="weather-run", product="weather", failure="check-or-execution-failed") + marker = dict(schema=2, kind="weather-products-acquisition-failure", success=False, + command_exit_status=1, diagnostic_state="valid", diagnostic=diagnostic) + cases = ((marker, False), (marker, True), + (dict(marker, command_exit_status=124, diagnostic_state="missing", diagnostic=None), False)) + for value, invalid_cleanup in cases: + with self.subTest(state=value["diagnostic_state"], invalid_cleanup=invalid_cleanup): + files = self.collect(json.dumps(diagnostic).encode(), products=True, + failure_marker=value, invalid_cleanup=invalid_cleanup) + prefix = "work-1/weather-products-delivery-" + self.assertEqual(json.loads(files[prefix + "acquisition-failure.json"]), value) + self.assertIn(prefix + "tree-before.json", files) + self.assertEqual(prefix + "input-cleanup.json" in files, not invalid_cleanup) + status = json.loads(files[prefix + "export-status.json"]) + expected_rejected = (["acquisition"] if value["diagnostic_state"] != "valid" else []) + if invalid_cleanup: expected_rejected.append("input-cleanup") + self.assertEqual(status["rejected"], expected_rejected) + self.assertEqual(status["success"], not expected_rejected) + self.assertTrue(status["complete"]) + self.assertNotIn(prefix + "acquisition.json", files) + self.assertFalse(json.loads(files["vm-incomplete.json"])["success"]) + self.assertFalse(any(b"PRIVATE CANARY" in data for data in files.values())) + malformed = dict(marker, private="PRIVATE CANARY") + files = self.collect(json.dumps(diagnostic).encode(), products=True, failure_marker=malformed) + self.assertFalse(any("weather-products-delivery" in name for name in files)) + files = self.collect(json.dumps(diagnostic).encode(), products=True, failure_marker=marker, limit=64) + self.assertEqual({name for name in files if "weather-products-delivery" in name}, + {"work-1/weather-products-delivery-input-cleanup.json"}) + self.assertEqual(json.loads(files["work-1/weather-products-delivery-input-cleanup.json"]), + dict(directory_removed=True, files_removed=True)) + def test_products_export_rejects_unknown_private_duplicate_and_mixed_source_data(self): valid = self.products_acquisition() wrong_field = self.products_acquisition() diff --git a/tests/integration/test-weather-products-delivery-shell.py b/tests/integration/test-weather-products-delivery-shell.py index 80b0d3cd8..8885484b4 100644 --- a/tests/integration/test-weather-products-delivery-shell.py +++ b/tests/integration/test-weather-products-delivery-shell.py @@ -89,6 +89,71 @@ def test_already_retained_report_not_replaced(self): ''')) self.assertEqual(retained.read_bytes(), b"retained") + def test_acquisition_failure_is_retained_before_fail_without_entering_publication(self): + source = SOURCE.read_text() + fragment = " PHASE=weather-products-delivery-acquire\n" + source.split( + " PHASE=weather-products-delivery-acquire\n", 1)[1].split( + " WEATHER_PRODUCTS_DEADLINE=", 1)[0] + for status, retain_status in ((1, 0), (124, 0), (137, 0), (1, 1), (0, 0)): + with self.subTest(status=status, retain_status=retain_status): + result = self.call(''' +umask 077 +weather_client_directory=$WORK/source +weather_client_manifest_sha256=inert +weather_publisher=$WORK/publisher +WORKER_UID=985 +site_parent_ns=net:[1] +site_client_ns=net:[2] +timeout() { printf '{"success":false}\n'; return STATUS; } +python3() { + [ "$*" = "-B $WORK/bin/weather-products-delivery.py retain-acquisition-failure $WORK STATUS" ] || exit 90 + printf 'retained:STATUS\n' + printf 'STATUS\n' > "$WORK/retained-status" + return RETAIN_STATUS +} +fail() { printf '%s %s\n' "$PHASE" "$1" >&2; exit 77; } +'''.replace("RETAIN_STATUS", str(retain_status)).replace("STATUS", str(status)) + + fragment + '\nprintf "publication-next\\n"\n') + if status: + expected = "WEATHER_PRODUCTS_ACQUISITION_DIAGNOSTIC_FAILED" if retain_status else "WEATHER_PRODUCTS_ACQUISITION_FAILED" + self.assertEqual(result.returncode, 77, result.stderr) + self.assertEqual(result.stderr.strip(), "weather-products-delivery-acquire " + expected) + self.assertEqual(result.stdout, "") + self.assertEqual((self.root / "retained-status").read_text().strip(), str(status)) + marker = self.root / "weather-products-delivery-acquisition-failure.json" + self.assertEqual(marker.read_text(), f"retained:{status}\n") + self.assertEqual(marker.stat().st_mode & 0o777, 0o600) + marker.unlink() + (self.root / "retained-status").unlink() + else: + self.ok(result) + self.assertEqual(result.stdout, "publication-next\n") + self.assertFalse((self.root / "retained-status").exists()) + + def test_failure_marker_selects_only_dedicated_closed_export(self): + (self.root / "bin").mkdir() + (self.root / "bin/weather-products-delivery.py").touch() + marker = self.root / "weather-products-delivery-acquisition-failure.json" + for kind in ("absent", "regular", "symlink"): + with self.subTest(kind=kind): + if kind == "regular": marker.write_text("untrusted fixture marker") + elif kind == "symlink": marker.symlink_to("missing-target") + result = self.call(''' +output_directory=$WORK/output +OUTPUT_UID=1000 +OUTPUT_GID=1000 +python3() { + printf '%s\n' "$*" > "$WORK/export-call" + return 1 +} +if weather_client_delivery_copy_artifacts; then exit 90; fi +''') + self.ok(result) + action = "copy-artifacts" if kind == "absent" else "copy-failure-artifacts" + self.assertEqual((self.root / "export-call").read_text().strip(), + f"-B {self.root}/bin/weather-products-delivery.py {action} {self.root} {self.root}/output 1000 1000") + if kind != "absent": marker.unlink() + def test_verified_closed_dispatch_before_any_v1_work(self): source = LEGACY.read_text() head = source.split("weather_client_delivery_run() {", 1)[1].split(" PHASE=weather-client-delivery-prepare", 1)[0] diff --git a/tests/integration/test-weather-products-delivery.py b/tests/integration/test-weather-products-delivery.py index 8fff69422..7fe17f61b 100644 --- a/tests/integration/test-weather-products-delivery.py +++ b/tests/integration/test-weather-products-delivery.py @@ -8,6 +8,8 @@ import os from pathlib import Path import runpy +import subprocess +import sys import tempfile import unittest from unittest.mock import patch @@ -72,6 +74,20 @@ def acquisition(): for step in p["steps"] for parameter in p["parameters"]]) for key, p in A["PRODUCTS"].items()}) +def diagnostic(stage="weather-run"): + return A["AcquisitionFailure"](stage, ValueError("private coordinates, path, URL and raw stderr")).diagnostic + + +def failure_fixture(work, raw=None): + save(A["artifact"](work, "tree-before"), dict(version=1, action="verify-tree", manifest_sha256=SHA, + files=193, runtime_executed=False)) + save(A["artifact"](work, "input-cleanup"), dict(directory_removed=True, files_removed=True)) + A["write_new"](A["artifact"](work, "acquisition"), raw if raw is not None else A["B"]["encode"](diagnostic())) + marker = A["retain_acquisition_failure"](work, 1) + save(A["artifact"](work, "acquisition-failure"), marker) + return marker + + def report(): legacy = OLD["report"](); expected = legacy["expected"] result = dict(schema=2, passed=True, phase="complete", source=legacy["source"], @@ -152,6 +168,126 @@ def setUp(self): clock = patch.dict(A["prepare"].__globals__, observed_ms=lambda: CLOCK) clock.start(); self.addCleanup(clock.stop) + def test_acquisition_diagnoses_every_stage_without_error_details(self): + for failed_stage in A["ACQUISITION_STAGES"]: + with self.subTest(stage=failed_stage), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) + source = base / "inputs"; source.mkdir(mode=0o700) + A["initialize"](root); A["write_new"](source / "INPUTS.json", b"fixture", 0o400) + def gate(stage): + if failed_stage == stage: raise ValueError("SECRET URL coordinates and path") + def execute(command, cwd, maximum, timeout): + if command[-1] == "-V": + gate("decoder"); return b"ecCodes Version 2.44.0\n" + key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" + gate(A["PRODUCTS"][key]["suffix"] + "-run") + if failed_stage == A["PRODUCTS"][key]["suffix"] + "-validation": return b'{"SECRET":' + return json.dumps(forecast(key, command[command.index("--run")+1])).encode() + with patch.dict(A["acquisition"].__globals__, run=execute, + private=lambda *args, **kwargs: gate("preconditions"), + boundary=lambda *args: gate("boundary"), + validate_acquisition=lambda *args: gate("final-validation")), \ + patch.dict(A["B"], verify_tree=lambda *args: gate("tree"), manifest=lambda *args: {"version": 2}), \ + patch.object(A["time"], "time", return_value=CLOCK // 1000): + with self.assertRaises(A["AcquisitionFailure"]) as raised: + A["acquisition"](source, SHA, root, "net:[1]", "net:[2]", 1000) + value = raised.exception.diagnostic + A["validate_acquisition_diagnostic"](value) + self.assertEqual(value["stage"], failed_stage) + self.assertNotIn("SECRET", json.dumps(value)) + self.assertNotIn(str(base), json.dumps(value)) + + def test_acquisition_failure_classification_does_not_leak_subprocess_arguments(self): + cases = [(FileNotFoundError("SECRET path"), "missing-input"), + (PermissionError("SECRET path"), "permission-denied"), (OSError("SECRET"), "io-failed"), + (subprocess.TimeoutExpired(["SECRET", "URL"], 1, output=b"SECRET"), "timeout"), + (subprocess.CalledProcessError(7, ["SECRET"], stderr=b"SECRET"), "process-failed"), + (ValueError("SECRET"), "check-or-execution-failed")] + for error, expected in cases: + with self.subTest(expected=expected): + value = A["AcquisitionFailure"]("air-quality-run", error).diagnostic + A["validate_acquisition_diagnostic"](value) + self.assertEqual(value["failure"], expected); self.assertNotIn("SECRET", json.dumps(value)) + with tempfile.TemporaryDirectory() as temporary: + result = subprocess.run([sys.executable, str(HERE / "weather-products-delivery.py"), "acquire", + temporary, SHA, str(Path(temporary) / "missing-private-path"), "net:[1]", "net:[2]", "1000"], + capture_output=True, check=False) + self.assertEqual(result.returncode, 1) + value = json.loads(result.stdout); A["validate_acquisition_diagnostic"](value) + self.assertEqual(value["stage"], "preconditions"); self.assertEqual(result.stderr, b"") + self.assertNotIn(temporary.encode(), result.stdout) + + def test_retained_acquisition_failures_are_closed_and_not_success_claims(self): + cases = [(None, "missing"), (b"", "missing"), (b'{"secret":', "invalid"), + (A["B"]["encode"](acquisition()), "invalid"), + (A["B"]["encode"](dict(diagnostic(), stderr="SECRET")), "invalid"), + (A["B"]["encode"](dict(diagnostic(), product="air_quality")), "invalid"), + (A["B"]["encode"](diagnostic()), "valid")] + for raw, state in cases: + with self.subTest(state=state, raw=raw), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary) + if raw is not None: A["write_new"](A["artifact"](work, "acquisition"), raw) + value = A["retain_acquisition_failure"](work, 124) + A["validate_acquisition_failure"](value) + self.assertEqual(value["diagnostic_state"], state) + self.assertEqual(value["command_exit_status"], 124) + self.assertNotIn("SECRET", json.dumps(value)) + with self.assertRaises(ValueError): A["validate_acquisition"](value) + for status in (0, True, -1, 256): + with self.assertRaises(ValueError): A["retain_acquisition_failure"](work, status) + mutations = [dict(diagnostic(), success=True), dict(diagnostic(), coordinates=[52.52, 13.41]), + dict(diagnostic(), stage="private URL"), dict(diagnostic(), failure="raw stderr")] + for value in mutations: + with self.assertRaises(ValueError): A["validate_acquisition_diagnostic"](value) + + def test_dedicated_failure_export_retains_valid_siblings_and_is_idempotent(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); failure_fixture(work) + output = work / "output"; output.mkdir() + A["write_new"](work / "raw-private.log", b"SECRET") + for _ in range(2): + result = A["copy_failure_artifacts"](work, output, os.getuid(), os.getgid()) + A["validate_failure_export_status"](result); self.assertTrue(result["success"]) + self.assertEqual(result["exported"], list(A["FAILURE_EXPORTS"])) + self.assertEqual({path.name for path in output.iterdir()}, + {A["artifact"](output, suffix).name for suffix in (*A["FAILURE_EXPORTS"], "export-status")}) + self.assertFalse(A["artifact"](output, "acquisition").exists()) + + def test_failure_export_retains_safe_siblings_but_exits_nonzero_for_rejected_evidence(self): + for bad in ("acquisition", "tree-before", "input-cleanup"): + with self.subTest(bad=bad), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); failure_fixture(work, b"" if bad == "acquisition" else None) + if bad != "acquisition": + source = A["artifact"](work, bad); source.unlink(); save(source, dict(raw="SECRET")) + output = work / "output"; output.mkdir() + result = subprocess.run([sys.executable, str(HERE / "weather-products-delivery.py"), + "copy-failure-artifacts", str(work), str(output), str(os.getuid()), str(os.getgid())], + capture_output=True, check=False) + self.assertEqual(result.returncode, 1); self.assertEqual(result.stderr, b"") + status = A["read"](A["artifact"](output, "export-status")) + A["validate_failure_export_status"](status) + self.assertFalse(status["success"]); self.assertEqual(status["rejected"], [bad]) + self.assertEqual(json.loads(result.stdout), status) + for suffix in status["exported"]: self.assertTrue(A["artifact"](output, suffix).exists()) + for path in output.iterdir(): self.assertNotIn(b"SECRET", path.read_bytes()) + self.assertFalse(A["artifact"](output, bad).exists()) + + def test_failure_export_rejects_unsafe_or_forged_marker_before_writing(self): + for bad in ("missing", "symlink", "mode", "success", "raw-field", "status", "duplicate"): + with self.subTest(bad=bad), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); value = failure_fixture(work); marker = A["artifact"](work, "acquisition-failure") + marker.unlink() + if bad == "symlink": marker.symlink_to(A["artifact"](work, "acquisition")) + elif bad != "missing": + if bad == "success": value["success"] = True + elif bad == "raw-field": value["raw"] = "SECRET" + elif bad == "status": value["command_exit_status"] = 0 + if bad == "duplicate": A["write_new"](marker, b'{"schema":2,"schema":2}') + else: save(marker, value, 0o644 if bad == "mode" else 0o600) + output = work / "output"; output.mkdir() + with self.assertRaises((ValueError, OSError)): A["copy_failure_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertEqual(list(output.iterdir()), []) + def test_acquisition_commands_include_precipitation_and_independent_noaa_cycle_with_one_deadline(self): with tempfile.TemporaryDirectory() as temporary: base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) diff --git a/tests/integration/weather-products-delivery.py b/tests/integration/weather-products-delivery.py index 8e29ca9e0..870ee21e7 100644 --- a/tests/integration/weather-products-delivery.py +++ b/tests/integration/weather-products-delivery.py @@ -42,6 +42,66 @@ "src/air-quality.mjs", "src/air-quality-provider.mjs", "src/air-quality-core.mjs"} EXPORTS = ("cold-evidence", "evidence", "acquisition", "original-report", "tree-before", "tree-after", "cache-before", "cache-after", "input-cleanup") +ACQUISITION_STAGES = {"preconditions": None, "boundary": None, "tree": None, "decoder": None, + "weather-run": "weather", "weather-validation": "weather", "air-quality-run": "air_quality", + "air-quality-validation": "air_quality", "final-validation": None} +FAILURE_CLASSES = {"missing-input", "permission-denied", "io-failed", "timeout", "process-failed", + "check-or-execution-failed"} +FAILURE_EXPORTS = ("acquisition-failure", "tree-before", "input-cleanup") +SAFE_ERRORS = (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError) + + +class AcquisitionFailure(ValueError): + """Carry only a closed classification, never a subprocess error or argv.""" + def __init__(self, stage, error): + failure = ("missing-input" if isinstance(error, FileNotFoundError) else + "permission-denied" if isinstance(error, PermissionError) else + "io-failed" if isinstance(error, OSError) else + "timeout" if isinstance(error, subprocess.TimeoutExpired) else + "process-failed" if isinstance(error, subprocess.CalledProcessError) else + "check-or-execution-failed") + self.diagnostic = dict(schema=2, kind="weather-products-acquisition-diagnostic", success=False, + stage=stage, product=ACQUISITION_STAGES[stage], failure=failure) + super().__init__("weather_products_acquisition_failed") + + +def validate_acquisition_diagnostic(value): + require(type(value) is dict and set(value) == {"schema", "kind", "success", "stage", "product", "failure"}) + same(value["schema"], 2); same(value["kind"], "weather-products-acquisition-diagnostic"); same(value["success"], False) + require(type(value["stage"]) is str and value["stage"] in ACQUISITION_STAGES + and type(value["failure"]) is str and value["failure"] in FAILURE_CLASSES) + same(value["product"], ACQUISITION_STAGES[value["stage"]]) + + +def validate_acquisition_failure(value): + require(type(value) is dict and set(value) == {"schema", "kind", "success", "command_exit_status", "diagnostic_state", "diagnostic"}) + same(value["schema"], 2); same(value["kind"], "weather-products-acquisition-failure"); same(value["success"], False) + require(type(value["command_exit_status"]) is int and 1 <= value["command_exit_status"] <= 255 + and type(value["diagnostic_state"]) is str and value["diagnostic_state"] in {"valid", "missing", "invalid"}) + if value["diagnostic_state"] == "valid": validate_acquisition_diagnostic(value["diagnostic"]) + else: same(value["diagnostic"], None) + + +def retain_acquisition_failure(work, status): + # The root shell invokes this only after acquire exits nonzero. A killed + # worker may leave empty stdout; never turn that into guessed source failure. + require(type(status) is int and 1 <= status <= 255) + source = artifact(work, "acquisition") + state, diagnostic = "missing", None + try: + if source.exists() or source.is_symlink(): + info = private(source, directory=False) + raw = raw_file(source, MAX_REPORT, 0o600) if info.st_size else b"" + if raw: + candidate = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + validate_acquisition_diagnostic(candidate) + state, diagnostic = "valid", candidate + except SAFE_ERRORS: + state = "invalid" + result = dict(schema=2, kind="weather-products-acquisition-failure", success=False, + command_exit_status=status, diagnostic_state=state, diagnostic=diagnostic) + validate_acquisition_failure(result) + return result def product(key): @@ -101,55 +161,65 @@ def forecast_times(value, key, model_run, observed_at, acquisition_window=None): def acquisition(source, manifest_sha, root, parent, client, group): - private(root) - require(root.name == "weather-publisher" and set(p.name for p in root.iterdir()) == {"passphrase", "scratch"}) - boundary(parent, client, group, True) - B["verify_tree"](source, manifest_sha) - require(B["manifest"](raw_file(source / "INPUTS.json", B["MAX_MANIFEST"], 0o400), manifest_sha)["version"] == 2) - deadline = time.monotonic() + 720 - require(run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10).strip() == b"ecCodes Version 2.44.0") - now = int(time.time()) - # Select independently completed cycles. NOAA's 00UTC cycle with an explicit - # twelve-hour availability margin is not rewritten to match IFS. Native - # samples are not interpolated; lead 48 preserves future coverage across - # midnight/noon boundaries without pretending the earlier leads are current. - cycles = dict(weather=now // 43200 * 43200 - 43200, air_quality=(now - 43200) // 86400 * 86400) - products = {} - for key, p in PRODUCTS.items(): - model_run = datetime.datetime.fromtimestamp(cycles[key], datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") - command = [str(source / "runtime/node"), "--max-old-space-size=256", - str(source / B["WEATHER"] / "scripts" / p["script"]), "origin", "--latitude", "52.52", "--longitude", "13.41", - "--run", model_run, "--steps", ",".join(map(str, p["steps"])), "--community-area", - "--execute", "--accept-data-license", "--decoder", str(source / "decoder/grib_ls"), - "--decoder-sha256", B["DECODER_SHA"], "--scratch-parent", str(root / "scratch")] - if key == "weather": command += ["--model", "ifs", "--include-wind", "--include-precipitation"] - remaining = deadline - time.monotonic(); require(remaining > 0) - started_at = observed_ms() - raw = run(command, root, MAX_FORECAST, min(360, remaining)) - finished_at = observed_ms() - value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) - require(value["schema"] == p["schema"] and value["source"]["provider"] == p["provider"] - and value["source"]["acquisition"] == "origin-https" and value["source"]["model"] == p["model"] - and value["source"]["model_run_at"] == model_run) - same(value["area"], dict(latitude=52.52, longitude=13.41, sharing="public-community")) - forecast_times(value, key, model_run, finished_at, (started_at, finished_at)) - require(not list((root / "scratch").iterdir())) - fields = [] - for sample in value["samples"]: - evidence = (sample["evidence"], *sample["wind_10m"]["evidence"].values(), sample["precipitation"]["evidence"]) \ - if key == "weather" else tuple(sample["evidence"].values()) - for field in evidence: - require(field["decoder_sha256"] == B["DECODER_SHA"] and field["parameter"] in p["parameters"] - and hex64(field["index_sha256"]) and hex64(field["field_sha256"])) - fields.append(dict(lead_hours=sample["lead_hours"], parameter=field["parameter"], - index_sha256=field["index_sha256"], field_sha256=field["field_sha256"])) - products[key] = dict(forecast_sha256=hashlib.sha256(raw).hexdigest(), forecast_bytes=len(raw), fields=fields) - write_new(root / p["filename"], raw) - result = dict(schema=2, success=True, origin_https=True, acquisition_outside_overlay=True, - upstream_signature_attested=False, synthetic_weather=False, synthetic_air_quality=False, - independent_product_runs=True, decoder_sha256=B["DECODER_SHA"], products=products) - validate_acquisition(result) - return result + stage = "preconditions" + try: + private(root) + require(root.name == "weather-publisher" and set(p.name for p in root.iterdir()) == {"passphrase", "scratch"}) + stage = "boundary" + boundary(parent, client, group, True) + stage = "tree" + B["verify_tree"](source, manifest_sha) + require(B["manifest"](raw_file(source / "INPUTS.json", B["MAX_MANIFEST"], 0o400), manifest_sha)["version"] == 2) + deadline = time.monotonic() + 720 + stage = "decoder" + require(run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10).strip() == b"ecCodes Version 2.44.0") + now = int(time.time()) + # Select independently completed cycles. NOAA's 00UTC cycle with an explicit + # twelve-hour availability margin is not rewritten to match IFS. Native + # samples are not interpolated; lead 48 preserves future coverage across + # midnight/noon boundaries without pretending the earlier leads are current. + cycles = dict(weather=now // 43200 * 43200 - 43200, air_quality=(now - 43200) // 86400 * 86400) + products = {} + for key, p in PRODUCTS.items(): + stage = p["suffix"] + "-run" + model_run = datetime.datetime.fromtimestamp(cycles[key], datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + command = [str(source / "runtime/node"), "--max-old-space-size=256", + str(source / B["WEATHER"] / "scripts" / p["script"]), "origin", "--latitude", "52.52", "--longitude", "13.41", + "--run", model_run, "--steps", ",".join(map(str, p["steps"])), "--community-area", + "--execute", "--accept-data-license", "--decoder", str(source / "decoder/grib_ls"), + "--decoder-sha256", B["DECODER_SHA"], "--scratch-parent", str(root / "scratch")] + if key == "weather": command += ["--model", "ifs", "--include-wind", "--include-precipitation"] + remaining = deadline - time.monotonic(); require(remaining > 0) + started_at = observed_ms() + raw = run(command, root, MAX_FORECAST, min(360, remaining)) + finished_at = observed_ms() + stage = p["suffix"] + "-validation" + value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + require(value["schema"] == p["schema"] and value["source"]["provider"] == p["provider"] + and value["source"]["acquisition"] == "origin-https" and value["source"]["model"] == p["model"] + and value["source"]["model_run_at"] == model_run) + same(value["area"], dict(latitude=52.52, longitude=13.41, sharing="public-community")) + forecast_times(value, key, model_run, finished_at, (started_at, finished_at)) + require(not list((root / "scratch").iterdir())) + fields = [] + for sample in value["samples"]: + evidence = (sample["evidence"], *sample["wind_10m"]["evidence"].values(), sample["precipitation"]["evidence"]) \ + if key == "weather" else tuple(sample["evidence"].values()) + for field in evidence: + require(field["decoder_sha256"] == B["DECODER_SHA"] and field["parameter"] in p["parameters"] + and hex64(field["index_sha256"]) and hex64(field["field_sha256"])) + fields.append(dict(lead_hours=sample["lead_hours"], parameter=field["parameter"], + index_sha256=field["index_sha256"], field_sha256=field["field_sha256"])) + products[key] = dict(forecast_sha256=hashlib.sha256(raw).hexdigest(), forecast_bytes=len(raw), fields=fields) + write_new(root / p["filename"], raw) + stage = "final-validation" + result = dict(schema=2, success=True, origin_https=True, acquisition_outside_overlay=True, + upstream_signature_attested=False, synthetic_weather=False, synthetic_air_quality=False, + independent_product_runs=True, decoder_sha256=B["DECODER_SHA"], products=products) + validate_acquisition(result) + return result + except SAFE_ERRORS as error: + raise AcquisitionFailure(stage, error) from None def validate_publication(value, key): @@ -594,10 +664,66 @@ def copy_artifacts(work, output, uid, gid, only_cold=False): return dict(files=count, raw_logs_exported=False) +def validate_failure_export_status(value): + require(type(value) is dict and set(value) == {"schema", "kind", "complete", "success", "exported", "rejected"}) + same(value["schema"], 2); same(value["kind"], "weather-products-failure-export"); same(value["complete"], True) + for field, allowed in (("exported", set(FAILURE_EXPORTS)), ("rejected", set(FAILURE_EXPORTS) | {"acquisition"})): + items = value[field] + require(type(items) is list and len(items) <= len(allowed) + and all(type(item) is str and item in allowed for item in items) and len(items) == len(set(items))) + require("acquisition-failure" in value["exported"] and not set(value["exported"]) & set(value["rejected"])) + same(value["success"], not value["rejected"]) + + +def copy_failure_file(target, raw, uid, gid): + if target.exists() or target.is_symlink(): + require(target.resolve(strict=True) == target) + with os.fdopen(os.open(target, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK), "rb") as existing: + info = os.fstat(existing.fileno()) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == uid and info.st_gid == gid + and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size == len(raw)) + require(existing.read(MAX_REPORT+1) == raw and B["identity"](os.fstat(existing.fileno())) == B["identity"](info)) + else: + write_new(target, raw); os.chown(target, uid, gid) + + +def copy_failure_artifacts(work, output, uid, gid): + # This action is explicitly selected only after acquisition failed. Its + # trusted-shell marker is mandatory; malformed success output is never an + # alternative success export. Keep copy_artifacts fully atomic as before. + marker = artifact(work, "acquisition-failure") + private(marker, directory=False) + raw = raw_file(marker, MAX_REPORT, 0o600) + failure = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + validate_acquisition_failure(failure) + copy_failure_file(artifact(output, "acquisition-failure"), raw, uid, gid) + exported, rejected = ["acquisition-failure"], [] + if failure["diagnostic_state"] != "valid": rejected.append("acquisition") + for suffix in FAILURE_EXPORTS[1:]: + source = artifact(work, suffix) + if not source.exists() and not source.is_symlink(): continue + try: + raw = raw_file(source, MAX_REPORT, 0o600) + value = json.loads(raw, object_pairs_hook=B["B"]["unique_pairs"]) + validate_export(suffix, value) + copy_failure_file(artifact(output, suffix), raw, uid, gid) + exported.append(suffix) + except SAFE_ERRORS: + rejected.append(suffix) + result = dict(schema=2, kind="weather-products-failure-export", complete=True, + success=not rejected, exported=exported, rejected=rejected) + validate_failure_export_status(result) + # Written last, allowing the emergency collector to distinguish a complete + # partial-safe export (exit 1) from a crash/timeout while copying files. + copy_failure_file(artifact(output, "export-status"), B["encode"](result), uid, gid) + return result + + def main(): action, *args = sys.argv[1:] if action == "init" and len(args) == 1: result = initialize(Path(args[0])) elif action == "acquire" and len(args) == 6: result = acquisition(Path(args[0]), args[1], Path(args[2]), args[3], args[4], int(args[5])) + elif action == "retain-acquisition-failure" and len(args) == 2: result = retain_acquisition_failure(Path(args[0]), int(args[1])) elif action == "select" and len(args) == 2: result = select(*map(Path, args)) elif action == "cold" and len(args) == 8: result = cold(args[0], *map(Path, args[1:5]), args[5], args[6], int(args[7])) elif action == "prepare" and len(args) == 6: result = prepare(*map(Path, args)) @@ -609,12 +735,18 @@ def main(): elif action == "evidence" and len(args) == 3: result = evidence(Path(args[0]), args[1], args[2]) elif action in {"copy-cold-evidence", "copy-artifacts"} and len(args) == 4: result = copy_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3]), only_cold=action == "copy-cold-evidence") + elif action == "copy-failure-artifacts" and len(args) == 4: + result = copy_failure_artifacts(Path(args[0]), Path(args[1]), int(args[2]), int(args[3])) else: raise ValueError("weather_products_delivery_rejected") print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + if action == "copy-failure-artifacts" and not result["success"]: raise SystemExit(1) if __name__ == "__main__": try: main() - except (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError): + except AcquisitionFailure as error: + print(json.dumps(error.diagnostic, sort_keys=True, separators=(",", ":"))) + raise SystemExit(1) + except SAFE_ERRORS: print('{"success":false,"failure":"weather_products_delivery_rejected"}') raise SystemExit(1) diff --git a/tests/integration/weather-products-delivery.sh b/tests/integration/weather-products-delivery.sh index 4dd41be4f..e4b0b56fd 100644 --- a/tests/integration/weather-products-delivery.sh +++ b/tests/integration/weather-products-delivery.sh @@ -88,7 +88,14 @@ weather_client_delivery_cleanup() { weather_client_delivery_copy_artifacts() { [ -f "$WORK/bin/weather-products-delivery.py" ] || return 0 - python3 -B "$WORK/bin/weather-products-delivery.py" copy-artifacts "$WORK" \ + weather_export_action=copy-artifacts + if [ -e "$WORK/weather-products-delivery-acquisition-failure.json" ] \ + || [ -L "$WORK/weather-products-delivery-acquisition-failure.json" ]; then + weather_export_action=copy-failure-artifacts + fi + # The trusted helper validates the marker. An unsafe failure marker must not + # fall back to the independent, atomic success-evidence exporter. + python3 -B "$WORK/bin/weather-products-delivery.py" "$weather_export_action" "$WORK" \ "$output_directory" "$OUTPUT_UID" "$OUTPUT_GID" >/dev/null } @@ -133,6 +140,7 @@ weather_products_delivery_run() { # the publication's lifetime. No origin retry is delegated to the browser. # The helper owns separately-sessioned decoder children. Allow its bounded # TERM/KILL/reap cleanup (at most 20s) to finish before the outer hard kill. + weather_acquisition_status=0 timeout --signal=TERM --kill-after=30s 740s setpriv \ --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ @@ -140,7 +148,18 @@ weather_products_delivery_run() { "$weather_client_directory" "$weather_client_manifest_sha256" "$weather_publisher" \ "$site_parent_ns" "$site_client_ns" "$site_control_gid" \ >"$WORK/weather-products-delivery-acquisition.json" 2>"$WORK/weather-products-delivery-acquisition.err" \ - || fail WEATHER_PRODUCTS_ACQUISITION_FAILED + || weather_acquisition_status=$? + if [ "$weather_acquisition_status" -ne 0 ]; then + if [ -e "$WORK/weather-products-delivery-acquisition-failure.json" ] \ + || [ -L "$WORK/weather-products-delivery-acquisition-failure.json" ]; then + fail WEATHER_PRODUCTS_ACQUISITION_DIAGNOSTIC_FAILED + fi + python3 -B "$WORK/bin/weather-products-delivery.py" retain-acquisition-failure \ + "$WORK" "$weather_acquisition_status" \ + >"$WORK/weather-products-delivery-acquisition-failure.json" \ + || fail WEATHER_PRODUCTS_ACQUISITION_DIAGNOSTIC_FAILED + fail WEATHER_PRODUCTS_ACQUISITION_FAILED + fi # Budget all later commands together, not 900 seconds per operation. Each # immutable publication still independently enforces its original expiry. WEATHER_PRODUCTS_DEADLINE=$(($(date +%s) + 840)) From 903db153c3a035571686375cb1697069dadd711c Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:02:40 +0200 Subject: [PATCH 23/49] test: add bounded late-provider recovery topology proof --- .github/workflows/alpha-topology.yml | 2 + .../examples/content-acceptance-fixture.rs | 83 +++- docs/IMPLEMENTATION_STATUS.md | 13 +- .../content-provider-late-recovery.py | 370 ++++++++++++++++++ .../content-provider-late-recovery.sh | 130 ++++++ tests/integration/content-provider-smoke.py | 44 ++- tests/integration/content-provider-smoke.sh | 9 +- tests/integration/kvm-alpha-topology.sh | 17 +- tests/integration/run-alpha-topology-vm.sh | 15 +- .../test-content-provider-late-recovery.py | 235 +++++++++++ .../test-content-provider-smoke.py | 154 +++++++- 11 files changed, 1050 insertions(+), 22 deletions(-) create mode 100644 tests/integration/content-provider-late-recovery.py create mode 100644 tests/integration/content-provider-late-recovery.sh create mode 100644 tests/integration/test-content-provider-late-recovery.py diff --git a/.github/workflows/alpha-topology.yml b/.github/workflows/alpha-topology.yml index 47acb1fce..fe049dd88 100644 --- a/.github/workflows/alpha-topology.yml +++ b/.github/workflows/alpha-topology.yml @@ -180,6 +180,8 @@ jobs: sh -n tests/integration/content-provider-filter-smoke.sh python3 -B tests/integration/test-content-provider-filter-smoke.py python3 -B tests/integration/test-content-provider-smoke.py + sh -n tests/integration/content-provider-late-recovery.sh + python3 -B tests/integration/test-content-provider-late-recovery.py sh -n tests/integration/content-provider-https-smoke.sh python3 -B tests/integration/test-content-provider-https-smoke.py sh -n tests/integration/content-mailbox-smoke.sh diff --git a/crates/volparossa-content/examples/content-acceptance-fixture.rs b/crates/volparossa-content/examples/content-acceptance-fixture.rs index d008e657e..4f4d59e67 100644 --- a/crates/volparossa-content/examples/content-acceptance-fixture.rs +++ b/crates/volparossa-content/examples/content-acceptance-fixture.rs @@ -39,7 +39,10 @@ async fn main() -> Result<()> { match args.as_slice() { [mode, root] if mode == "seed" => seed(Path::new(root), None), [mode, root, first, second] if mode == "seed-providers" => { - seed_replicas(Path::new(root), None, Path::new(first), Path::new(second)) + seed_replicas(Path::new(root), None, Path::new(first), Path::new(second), false) + } + [mode, root, first, second] if mode == "seed-late-providers" => { + seed_replicas(Path::new(root), None, Path::new(first), Path::new(second), true) } [mode, root, first, second, third] if mode == "seed-adaptive-providers" => { adaptive::seed(Path::new(root), [Path::new(first), Path::new(second), Path::new(third)]) @@ -73,7 +76,7 @@ async fn main() -> Result<()> { ) .await } - _ => Err("usage: seed ROOT | seed-providers ROOT CACHE_A CACHE_B | seed-adaptive-providers ROOT CACHE_A CACHE_B CACHE_C | seed-replication ROOT CACHE_P CACHE_Q | seed-private ROOT RECIPIENT_PUBLIC_HEX | recipient-init CLIENT_ROOT REPORT | serve ROOT a|b LISTEN REPORT | fetch CLIENT_ROOT MANIFEST PUBLISHER_HEX CONNECT REPORT | open-message CLIENT_ROOT MANIFEST PUBLISHER_HEX REPORT".into()), + _ => Err("usage: seed ROOT | seed-providers ROOT CACHE_A CACHE_B | seed-late-providers ROOT CACHE_A CACHE_B | seed-adaptive-providers ROOT CACHE_A CACHE_B CACHE_C | seed-replication ROOT CACHE_P CACHE_Q | seed-private ROOT RECIPIENT_PUBLIC_HEX | recipient-init CLIENT_ROOT REPORT | serve ROOT a|b LISTEN REPORT | fetch CLIENT_ROOT MANIFEST PUBLISHER_HEX CONNECT REPORT | open-message CLIENT_ROOT MANIFEST PUBLISHER_HEX REPORT".into()), } } @@ -95,6 +98,7 @@ fn seed(root: &Path, recipient: Option<[u8; 32]>) -> Result<()> { recipient, &root.join("replica-a"), &root.join("replica-b"), + false, ) } @@ -105,6 +109,7 @@ fn seed_replicas( recipient: Option<[u8; 32]>, first: &Path, second: &Path, + late: bool, ) -> Result<()> { fs::DirBuilder::new().mode(0o700).create(root)?; let origin = tempfile::tempdir_in(root)?; @@ -118,9 +123,12 @@ fn seed_replicas( // Distinct, deterministic public fixture bytes: every full chunk has a different hash. let mut content = Vec::with_capacity(8 * CHUNK_BYTES + 123); for index in 0_u8..8 { - content.extend(vec![b'A' + index; CHUNK_BYTES]); + content.extend(vec![ + if late { 0x10 + index } else { b'A' + index }; + CHUNK_BYTES + ]); } - content.extend(vec![b'Z'; 123]); + content.extend(vec![if late { 0x19 } else { b'Z' }; 123]); let validity = Validity { created: now, expires: now + 3600, @@ -132,7 +140,12 @@ fn seed_replicas( &mut content.as_slice(), Publication { metadata: Metadata { - name: "disposable-native-network-publication".into(), + name: if late { + "disposable-late-native-publication" + } else { + "disposable-native-network-publication" + } + .into(), revision: 1, content_type: "application/octet-stream".into(), }, @@ -169,8 +182,8 @@ fn seed_replicas( return Err("publisher directory survived removal".into()); } write_new(&root.join("manifest.bin"), &signed.encode())?; - let report = json!({ - "report_kind": "volparossa-content-seed", + let mut report = json!({ + "report_kind": if late { "volparossa-content-late-provider-seed" } else { "volparossa-content-seed" }, "publisher_hex": hex::encode(trusted_publisher.as_bytes()), "manifest_id": hex::encode(manifest.manifest_id()), "object_sha256": expected_hash.to_string(), @@ -184,6 +197,11 @@ fn seed_replicas( "publisher_private_key_persisted": false, "recipient_encrypted": recipient.is_some(), }); + if late { + report["name"] = json!("disposable-late-native-publication"); + report["created_unix_seconds"] = json!(now); + report["expires_unix_seconds"] = json!(now + 3600); + } write_report(&root.join("publication.json"), &report) } @@ -557,7 +575,7 @@ mod tests { let root = temporary.path().join("publication"); let first = temporary.path().join("provider-a-cache"); let second = temporary.path().join("provider-b-cache"); - seed_replicas(&root, None, &first, &second)?; + seed_replicas(&root, None, &first, &second, false)?; let report: Value = serde_json::from_slice(&read_bounded(&root.join("publication.json"), 4096)?)?; assert_eq!(report["publisher_removed"], true); @@ -588,4 +606,53 @@ mod tests { ); Ok(()) } + + #[test] + fn late_provider_seed_has_fresh_chunks_and_complementary_final_stores() -> Result<()> { + let temporary = tempfile::tempdir()?; + let root = temporary.path().join("late-publication"); + let first_path = temporary.path().join("late-a"); + let second_path = temporary.path().join("late-b"); + seed_replicas(&root, None, &first_path, &second_path, true)?; + let report: Value = + serde_json::from_slice(&read_bounded(&root.join("publication.json"), 4096)?)?; + assert_eq!( + report["report_kind"], + "volparossa-content-late-provider-seed" + ); + assert_eq!(report["publisher_removed"], true); + assert_eq!(report["publisher_private_key_persisted"], false); + assert_eq!(report["replica_a_chunks"], 5); + assert_eq!(report["replica_b_chunks"], 4); + assert_eq!(fs::read_dir(&root)?.count(), 2); + let verified = manifest( + &root.join("manifest.bin"), + report["publisher_hex"].as_str().ok_or("publisher absent")?, + )?; + let mut first = ChunkStore::open(&first_path, limits())?; + let mut second = ChunkStore::open(&second_path, limits())?; + for (index, chunk) in verified.chunks().iter().enumerate() { + assert_eq!(first.get(chunk.id())?.is_some(), index % 2 == 0); + assert_eq!(second.get(chunk.id())?.is_some(), index % 2 == 1); + for byte in b'A'..b'A' + 60 { + assert_ne!(*chunk.id(), ChunkId::digest(&vec![byte; CHUNK_BYTES])); + } + } + let mut restored = Vec::new(); + reassemble( + &verified, + &mut [&mut first, &mut second], + unix_seconds()?, + &mut restored, + )?; + assert_eq!(restored.len(), 2_097_275); + assert_eq!( + report["object_sha256"], + ChunkId::digest(&restored).to_string() + ); + assert_eq!(restored[0], 0x10); + assert_eq!(restored.last(), Some(&0x19)); + assert!(seed_replicas(&root, None, &first_path, &second_path, true).is_err()); + Ok(()) + } } diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 6597eaca1..ac3b632bd 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,7 +29,7 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The latest local run on `cc5f3558` completes the base two-provider fetch with measured overlap, using one discovery query. It does not exercise late rediscovery or reach adaptive transfer. The separate CI primary-Exit discovery failure remains unresolved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The separate late-provider fixture is implemented and passes its focused tests; its real discovery timing is unproven. The latest local run on `cc5f3558` completes only the base two-provider fetch with measured overlap and one discovery query. The separate CI primary-Exit discovery failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather` | Build and exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | Reuse the provider contracts and core retrieval path; client adaptation, a reproducible client build and its own runtime proof remain required. Do not create a second network scheduler or source-acquisition implementation. | @@ -42,6 +42,17 @@ changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. +The late-provider fixture uses a fresh publication, route and consumer cache. +It registers the second complementary provider only after observing the first +completed, single-offer discovery. A temporary provider-side gate is bounded to +six seconds, below the unchanged ten-second connection timeout, and is removed +before child-process cleanup. Seven coordinator/evidence tests, two Rust seed +tests and nineteen aggregate-report tests pass, alongside ten cancellation +checks. Current reports require this separate result; historical version-one +reports remain explicitly readable. These checks validate the fixture, not live +late-provider discovery or improved transfer speed. The existing concurrent +transfer and privacy requirements remain mandatory. + After each milestone, or when a substantive blocker changes, reassess the next shared dependency against the existing evidence and application worktrees. Protected compute for Code/Browser, private storage for Chat/Image/Cloud, diff --git a/tests/integration/content-provider-late-recovery.py b/tests/integration/content-provider-late-recovery.py new file mode 100644 index 000000000..5bb609cb5 --- /dev/null +++ b/tests/integration/content-provider-late-recovery.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Separate late-provider availability proof; never a concurrency/speed substitute.""" +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import select +import signal +import socket +import stat +import subprocess +import sys +import time + +BASE = runpy.run_path(str(Path(__file__).with_name("content-provider-https-smoke.py"))) +require, read, ROLES = BASE["require"], BASE["read"], BASE["ROLES"] +PREFIX = "content-provider-late-recovery" +NODES = ("relay4", "relay5") +TABLE = "vpa_content_late_gate" +LIMIT = 262144 +GATE_NS = 6_000_000_000 +BYTES = 2097275 +SHA = hashlib.sha256(b"".join(bytes([16+i])*262144 for i in range(8)) + bytes([25])*123).hexdigest() +EVENTS = ("CONTENT_DISCOVERY_COMPLETED", "CONTENT_DISCOVERY_RECEIVED_OFFERS", "CONTENT_DISCOVERY_RECEIVED_EMPTY", + "CONTENT_PROVIDER_QUERY_STARTED", "CONTENT_PROVIDER_OFFER_VERIFIED") + + +def path(work, suffix, extension="json"): + return work / f"{PREFIX}-{suffix}.{extension}" + + +def write(target, value): + raw = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + b"\n" + require(len(raw) <= LIMIT, "evidence too large") + with target.open("xb") as stream: + os.fchmod(stream.fileno(), 0o600); stream.write(raw) + + +def private_read(target): + require(target.resolve() == target and not target.is_symlink(), "unsafe private record") + info = target.stat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o600 and info.st_size <= LIMIT, "wrong private record owner") + return read(target) + + +def guest(work): + require(os.geteuid() == 0 and socket.gethostname() == "volparossa-alpha" + and work.is_absolute() and work.resolve() == work and work.parent == Path("/opt") + and work.name.startswith("va.") and work.is_dir() and work.stat().st_uid == 0, "disposable guest required") + result = subprocess.run(["systemd-detect-virt", "--vm"], capture_output=True, timeout=3, check=False) + require(result.returncode == 0 and result.stdout.strip() == b"kvm", "KVM guest required") + + +def namespace(work, expected=None): + record = private_read(work / "helper-record-relay4.json") + name = record["network_namespace"] + require(record["node"] == "relay4" and record["unit"] == "volparossa-alpha-helper@relay4.service" + and re.fullmatch(r"va-[0-9a-f]{8}-r4", name), "wrong provider namespace") + info = os.stat(f"/run/netns/{name}") + identity = f"{info.st_dev}:{info.st_ino}" + require(identity == record["network_namespace_identity"] + and (expected is None or expected == dict(name=name, identity=identity)), "namespace replaced") + return dict(name=name, identity=identity) + + +def command(args, deadline, input_data=None): + remaining = (deadline-time.monotonic_ns()) / 1e9 + require(remaining > 0, "fixture deadline exceeded") + result = subprocess.run(args, input=input_data, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=min(remaining, 2), check=False) + require(result.returncode == 0 and len(result.stdout) <= LIMIT, "bounded fixture command failed") + return result.stdout + + +def gate_rules(): + return (f"table inet {TABLE} {{ chain input {{ type filter hook input priority -30; policy accept; " + 'iifname "r4x" ip saddr 46.162.3.1 ip daddr 49.165.5.1 tcp dport 18080 counter drop; } }\n').encode() + + +def validate_gate(value): + entries = value["nftables"] + require(len(entries) == 4, "unexpected gate objects") + table = [item["table"] for item in entries if "table" in item] + chains = [item["chain"] for item in entries if "chain" in item] + rules = [item["rule"] for item in entries if "rule" in item] + require(len(table) == len(chains) == len(rules) == 1 + and table[0]["family"] == "inet" and table[0]["name"] == TABLE + and chains[0]["family"] == "inet" and chains[0]["table"] == TABLE and chains[0]["name"] == "input" + and chains[0]["type"] == "filter" and chains[0]["hook"] == "input" + and chains[0]["prio"] == -30 and chains[0]["policy"] == "accept" + and rules[0]["family"] == "inet" and rules[0]["table"] == TABLE and rules[0]["chain"] == "input", + "gate escaped exact input table") + expressions = rules[0]["expr"] + expected = [dict(match=dict(op="==", left=dict(meta=dict(key="iifname")), right="r4x")), + *[dict(match=dict(op="==", left=dict(payload=dict(protocol="ip", field=field)), right=address)) + for field, address in (("saddr", "46.162.3.1"), ("daddr", "49.165.5.1"))], + dict(match=dict(op="==", left=dict(payload=dict(protocol="tcp", field="dport")), right=18080))] + require(len(expressions) == 6 and expressions[:4] == expected and expressions[5] == {"drop": None} + and set(expressions[4]) == {"counter"} + and set(expressions[4]["counter"]) == {"packets", "bytes"} + and all(type(value) is int and value >= 0 for value in expressions[4]["counter"].values()), "wrong gate rule") + + +def remove_gate(work, guest_checked=False): + record = path(work, "gate", "state") + if not record.exists() and not record.is_symlink(): return + if not guest_checked: guest(work) + state = private_read(record); ns = namespace(work, state["namespace"]) + base = ["ip", "netns", "exec", ns["name"], "nft"] + # Listing all table names lets absence be distinguished from command failure. + deadline = time.monotonic_ns() + 2_000_000_000 + tables = json.loads(command(base + ["-j", "list", "tables"], deadline)) + present = any(row.get("table", {}).get("family") == "inet" and row.get("table", {}).get("name") == TABLE + for row in tables["nftables"]) + if present: + snapshot = json.loads(command(base + ["-j", "list", "table", "inet", TABLE], deadline)) + validate_gate(snapshot) + command(base + ["delete", "table", "inet", TABLE], deadline) + + +def process_identity(pid): + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return dict(pid=pid, start_ticks=int(fields[19])) + + +def stop_fetch(work): + record = path(work, "fetch", "state") + if not record.exists() and not record.is_symlink(): return + guest(work); identity = private_read(record) + require(set(identity) == {"pid", "start_ticks"} and all(type(v) is int and v > 1 for v in identity.values()), + "wrong fetch process record") + try: descriptor = os.pidfd_open(identity["pid"]) + except ProcessLookupError: return + try: + require(process_identity(identity["pid"]) == identity, "fetch PID reused") + signal.pidfd_send_signal(descriptor, signal.SIGTERM) + poll = select.poll(); poll.register(descriptor, select.POLLIN) + if not poll.poll(1000): + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + require(poll.poll(1000), "owned fetch did not terminate") + except ProcessLookupError: pass + finally: os.close(descriptor) + + +def cleanup(work): + # Gate release must not wait for child termination near the connect deadline. + try: remove_gate(work) + finally: stop_fetch(work) + + +def event_rows(raw, baseline): + require(len(raw) <= LIMIT, "event snapshot too large") + rows = [] + lines = raw.decode("ascii").splitlines() + require(len(lines) <= 400, "event ring bound exceeded") + stamps = [] + for line in lines: + fields = line.split() + if not fields or not fields[0].isdigit(): continue + timestamp = int(fields[0]); stamps.append(timestamp) + if timestamp <= baseline or len(fields) < 3 or not fields[2].startswith("event="): continue + code = fields[2][6:] + if code in EVENTS: rows.append(dict(unix_ms=timestamp, event=code)) + require(not (len(lines) == 400 and stamps and min(stamps) > baseline), "event ring lost interval") + return rows + + +def count(rows, code): return sum(row["event"] == code for row in rows) + + +def first_snapshot(snapshot): + return (count(snapshot["client"], EVENTS[0]) == count(snapshot["client"], EVENTS[1]) == 1 + and count(snapshot["client"], EVENTS[2]) == 0 + and count(snapshot["broker"], EVENTS[3]) == count(snapshot["broker"], EVENTS[4]) == 1) + + +def run(work, binary, provider_namespace): + guest(work); ns = namespace(work) + require(ns["name"] == provider_namespace and binary.resolve() == binary and binary.is_file(), "wrong executable/namespace") + layout, peers, publication = (read(path(work, name)) for name in ("layout", "peers", "publication")) + control = [node for node in ("relay0", "relay1", "relay2") if peers[node] == layout["control_relay_peer_id"]] + require(len(control) == 1 and layout["provider_nodes"] == list(NODES), "wrong control/provider layout") + native = work / "state-client/content-late-recovery" + require(all(not (native / name).exists() and not (native / name).is_symlink() + for name in ("cache", "object.bin")), "not cold") + def cli(node, *args): return [str(binary), "--control-socket", str(work / f"runtime-{node}/control/agent.sock"), *args] + def snapshot(baseline, deadline): + return {label: event_rows(command(cli(node, "logs", "--limit", "400"), deadline), baseline) + for label, node in (("client", "client"), ("broker", control[0]))} + baseline = time.time_ns() // 1000000 + time.sleep(0.002) # Only cross the event clock tick, never wait for provider readiness. + tables = json.loads(command(["ip", "netns", "exec", ns["name"], "nft", "-j", "list", "tables"], + time.monotonic_ns()+2_000_000_000)) + require(not any(row.get("table", {}).get("family") == "inet" and row.get("table", {}).get("name") == TABLE + for row in tables["nftables"]), "gate table collision") + gate_start = time.monotonic_ns(); deadline = gate_start + GATE_NS + write(path(work, "gate", "state"), dict(namespace=ns)) + process, success = None, False + def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") + handlers = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} + try: + command(["ip", "netns", "exec", ns["name"], "nft", "-f", "-"], deadline, gate_rules()) + installed = json.loads(command(["ip", "netns", "exec", ns["name"], "nft", "-j", "list", "table", "inet", TABLE], deadline)) + validate_gate(installed); write(path(work, "gate-installed"), installed) + started = time.monotonic_ns() + with path(work, "fetch").open("xb") as stdout, path(work, "fetch", "err").open("xb") as stderr: + os.fchmod(stdout.fileno(), 0o600); os.fchmod(stderr.fileno(), 0o600) + process = subprocess.Popen(cli("client", "content", "fetch", "--manifest", str(native / "manifest.bin"), + "--publisher-key", publication["publisher_hex"], "--cache", str(native / "cache"), + "--output", str(native / "object.bin")), stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr) + write(path(work, "fetch", "state"), process_identity(process.pid)) + while True: + require(process.poll() is None, "fetch ended before late activation") + initial = snapshot(baseline, deadline) + require(count(initial["client"], EVENTS[0]) <= 1, "first snapshot already passed") + if first_snapshot(initial): break + require(time.monotonic_ns() < deadline - 2_000_000_000, "first snapshot not available within gate budget") + time.sleep(0.02) + observed = time.monotonic_ns() + write(path(work, "first-snapshot"), initial) + activated = time.monotonic_ns() + serve = json.loads(command(cli("relay5", "content", "serve", "--manifest", str(native / "manifest.bin"), + "--publisher-key", publication["publisher_hex"], "--cache", str(work / "state-relay5/content-late-recovery/cache"), + "--bind", "50.166.6.1:18080", "--advertised-hostname", "provider-b.volparossa.test"), deadline)) + require(serve["serving"] is True and serve["publications"] == 1 and serve["replication_enabled"] is False, + "late service did not acknowledge registration") + acknowledged = time.monotonic_ns(); write(path(work, "relay5-serve"), serve) + command(["ip", "netns", "exec", ns["name"], "nft", "delete", "table", "inet", TABLE], deadline) + released = time.monotonic_ns(); require(released <= deadline, "gate exceeded six seconds") + require(process.wait(timeout=max(0.001, (started+300_000_000_000-time.monotonic_ns())/1e9)) == 0, + "native late recovery failed") + completed = time.monotonic_ns() + final = snapshot(baseline, time.monotonic_ns()+5_000_000_000) + write(path(work, "final-snapshot"), final) + output = native / "object.bin" + require(output.is_file() and not output.is_symlink() and output.stat().st_size == BYTES, "wrong output") + write(path(work, "object"), dict(bytes=output.stat().st_size, sha256=hashlib.sha256(output.read_bytes()).hexdigest(), + event_baseline_unix_ms=baseline, gate_started_ns=gate_start, fetch_started_ns=started, + first_snapshot_observed_ns=observed, late_activation_started_ns=activated, + late_activation_acknowledged_ns=acknowledged, gate_released_ns=released, fetch_completed_ns=completed, + cold_cache=True, consumer_provider_hints=False, origin_fallback=False)) + success = True + finally: + gate_removed = False + try: + # Already checked the guest before installation. Revalidate only the + # exact root intent/namespace now: bounded release precedes reaping. + remove_gate(work, guest_checked=True); gate_removed = True + finally: + try: + if process is not None and process.poll() is None: + process.terminate() + try: process.wait(timeout=2) + except subprocess.TimeoutExpired: process.kill(); process.wait(timeout=2) + write(path(work, "run-cleanup"), dict(fetch_reaped=process is None or process.poll() is not None, + gate_removed=gate_removed, operation_completed=success)) + finally: + for sig, handler in handlers.items(): signal.signal(sig, handler) + + +def validate(value): + require(value["success"] is True and value["schema_version"] == 1 + and value["concurrent_provider_overlap_claimed"] is False, "wrong late-recovery scope") + publication, output, fetch = (value[k] for k in ("publication", "output", "fetch")) + require(publication["report_kind"] == "volparossa-content-late-provider-seed" + and publication["name"] == "disposable-late-native-publication" + and publication["bytes"] == fetch["bytes"] == output["bytes"] == BYTES + and publication["object_sha256"] == output["sha256"] == SHA + and publication["chunks"] == fetch["chunks"] == 9 + and publication["replica_a_chunks"] == 5 and publication["replica_b_chunks"] == 4 + and publication["replica_a_bytes"] == 1048699 and publication["replica_b_bytes"] == 1048576 + and publication["publisher_removed"] is True and publication["publisher_private_key_persisted"] is False + and publication["recipient_encrypted"] is False, "fresh complementary publication not proven") + peers, layout = value["expected_peers"], value["layout"] + control = [node for node in ("relay0", "relay1", "relay2") if peers[node] == layout["control_relay_peer_id"]] + require(len(control) == 1 and layout["provider_nodes"] == list(NODES) + and all(value[k]["control_relay_peer_id"] == layout["control_relay_peer_id"] for k in ("status_before", "status_after", "fetch")) + and fetch["operation"] == "native_content" and fetch["providers_used"] == len(fetch["provider_peer_ids"]) == 2 + and set(fetch["provider_peer_ids"]) == {peers[node] for node in NODES} + and fetch["peer_bytes"] == BYTES and fetch["origin_body_bytes"] == fetch["origin_range_requests"] == 0 + and fetch["origin_authenticated"] is False, "real independent protected suppliers not proven") + before, live = value["selected_before"], value["selected_route"] + require(before["route_context_id"] == live["route_context_id"] != value["previous_context"] + and before["benchmark_slots"] == live["benchmark_slots"] and before["transport"] == "mptcp", "route changed or not fresh") + BASE["validate_path"](value, peers, list(NODES), False) + for node, size in zip(NODES, (1048699, 1048576)): + require(size <= value["privacy"]["exit"]["provider_application"][node]["response_payload_bytes"] <= size+131072, + "provider shard traffic inconsistent") + validate_gate(value["gate"]) + require(first_snapshot(value["first_snapshot"]), "first generic snapshot was not one verified offer") + final = value["final_snapshot"] + require(2 <= count(final["client"], EVENTS[0]) <= 3 and count(final["client"], EVENTS[1]) >= 2 + and 2 <= count(final["broker"], EVENTS[3]) <= 3 and 3 <= count(final["broker"], EVENTS[4]) <= 48, + "bounded genuine rediscovery not proven") + for label in ("client", "broker"): + require(final[label][:len(value["first_snapshot"][label])] == value["first_snapshot"][label], "snapshot lineage changed") + require(all(row["event"] in EVENTS and type(row["unix_ms"]) is int + and row["unix_ms"] > output["event_baseline_unix_ms"] for row in final[label]), "invalid event interval") + times = [output[k] for k in ("gate_started_ns", "fetch_started_ns", "first_snapshot_observed_ns", + "late_activation_started_ns", "late_activation_acknowledged_ns", "gate_released_ns", "fetch_completed_ns")] + require(all(type(v) is int and v > 0 for v in times) and times == sorted(times) + and times[5]-times[0] <= GATE_NS and 0 < times[-1]-times[1] <= 300_000_000_000 + and output["cold_cache"] is True and output["consumer_provider_hints"] is False and output["origin_fallback"] is False, + "activation ordering or absolute gate/transfer bound violated") + for node in NODES: + records = value["providers"][node] + require(records["before"]["serving"] is False and records["before"]["publications"] == 0 + and records["serve"]["serving"] is True and records["serve"]["publications"] == 1 + and records["serve"]["replication_enabled"] is False + and records["stop"]["serving"] is False and records["stop"]["publications"] == 0, "provider lifecycle incomplete") + require(value["run_cleanup"] == dict(fetch_reaped=True, gate_removed=True, operation_completed=True) + and value["cleanup"] == dict(route_disconnected=True, active_contexts=0, paths_empty=True, + client_output_removed=True, client_manifest_removed=True, client_mount_isolated=True), "cleanup/isolation incomplete") + validate_control(value, control[0]) + + +def validate_control(value, broker): + capture, routes = value["control_underlay"]["capture"], value["control_underlay"]["routes"] + pairs = {f"lc{i}": [BASE["PUBLIC_IPS"][broker], BASE["PUBLIC_IPS"][node]] for i, node in enumerate(NODES)} + require(capture["capture_role"] == "content-control" and capture["content_provider_mode"] is True + and capture["content_control_pairs"] == pairs and set(capture["interfaces"]) == set(pairs) + and set(capture["content_control_packets"]) == set(pairs) + and capture["unexpected_provider_control_packets"] == capture["unexpected_provider_application_packets"] == 0 + and all(value == 0 for counters in capture["provider_application"].values() for value in counters.values()), + "unexpected control-link traffic") + BASE["validate_drained"](capture) + for i, node in enumerate(NODES): + require(all(capture["content_control_packets"][f"lc{i}"][key] > 0 for key in ("inbound", "outbound")), "missing control traffic") + for direction, dev, gateway, source, destination in ( + ("out", f"lc{i}", f"10.241.{86+i}.2", BASE["PUBLIC_IPS"][broker], BASE["PUBLIC_IPS"][node]), + ("back", f"lp{i}", f"10.241.{86+i}.1", BASE["PUBLIC_IPS"][node], BASE["PUBLIC_IPS"][broker])): + rows = routes[node][direction] + require(len(rows) == 1 and all(rows[0].get(k) == v for k, v in + dict(dev=dev, gateway=gateway, prefsrc=source, dst=destination).items()), "wrong dedicated control route") + + +def build_evidence(work): + value = {key: read(path(work, suffix)) for key, suffix in (("publication", "publication"), ("layout", "layout"), + ("status_before", "status-before"), ("status_after", "status-after"), ("fetch", "fetch"), ("output", "object"), + ("selected_before", "selection"), ("selected_route", "live-selection"), ("cleanup", "cleanup"), + ("first_snapshot", "first-snapshot"), ("final_snapshot", "final-snapshot"), ("gate", "gate-installed"), ("run_cleanup", "run-cleanup"))} + value.update(schema_version=1, success=True, concurrent_provider_overlap_claimed=False, + expected_peers=read(work / "a01-expected-peers.json"), + previous_context=read(work / "content-provider-adaptive-selection.json")["route_context_id"], + providers={node: {op: read(path(work, f"{node}-{op}")) for op in ("before", "serve", "stop")} for node in NODES}, + privacy={role: read(path(work, f"privacy-{role}")) for role in ROLES}, + control_underlay=dict(capture=read(path(work, "control-privacy")), routes={node: {direction: + json.loads(path(work, f"control-{node}-{direction}").read_text()) for direction in ("out", "back")} for node in NODES})) + status = path(work, "final-status", "txt").read_text().splitlines() + require("connected: false" in status and "active contexts: 0" in status + and path(work, "final-paths", "txt").read_text() == "", "route still active") + validate(value); return value + + +def main(): + action, *args = sys.argv[1:] + if action == "run" and len(args) == 3: run(Path(args[0]), Path(args[1]), args[2]) + elif action == "cleanup" and len(args) == 1: cleanup(Path(args[0])) + elif action == "evidence" and len(args) == 2: write(Path(args[1]), build_evidence(Path(args[0]))) + else: raise ValueError("invalid late recovery action") + + +if __name__ == "__main__": + try: main() + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): + print("late-provider fixture failed closed", file=sys.stderr); raise SystemExit(1) diff --git a/tests/integration/content-provider-late-recovery.sh b/tests/integration/content-provider-late-recovery.sh new file mode 100644 index 000000000..85405a27b --- /dev/null +++ b/tests/integration/content-provider-late-recovery.sh @@ -0,0 +1,130 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only +# Separate availability proof after adaptive overlap, before cancellation. +# shellcheck disable=SC2154,SC2034 + +content_provider_late_recovery_cleanup() { + [ -f "$WORK/bin/content-provider-late-recovery.py" ] || return 0 + python3 -B "$WORK/bin/content-provider-late-recovery.py" cleanup "$WORK" +} + +content_provider_late_recovery_run() { + PHASE=content-provider-late-recovery-selection + late_prefix=content-provider-late-recovery + benchmark_select_route "$late_prefix" mptcp || fail CONTENT_PROVIDER_LATE_SELECTION_FAILED + benchmark_bind_slots "$WORK/$late_prefix-selection.json" || fail CONTENT_PROVIDER_LATE_SELECTION_INVALID + "$binary_directory/volparossa" --control-socket "$WORK/runtime-client/control/agent.sock" content status \ + >"$WORK/$late_prefix-status-before.json" || fail CONTENT_PROVIDER_LATE_STATUS_FAILED + late_control=$(jq -er '.control_relay_peer_id | select(type == "string" and length > 0)' \ + "$WORK/$late_prefix-status-before.json") || fail CONTENT_PROVIDER_LATE_CONTROL_INVALID + late_broker=$(jq -er --arg peer "$late_control" 'to_entries[] | select(.value == $peer) | .key' \ + "$WORK/a01-expected-peers.json") || fail CONTENT_PROVIDER_LATE_CONTROL_INVALID + case $late_broker in relay0|relay1|relay2) ;; *) fail CONTENT_PROVIDER_LATE_CONTROL_INVALID ;; esac + jq -n --arg peer "$late_control" '{provider_nodes:["relay4","relay5"],control_relay_peer_id:$peer}' \ + >"$WORK/$late_prefix-layout.json" + install -o root -g root -m 0600 "$WORK/a01-expected-peers.json" "$WORK/$late_prefix-peers.json" + content_provider_node "$late_broker" || fail CONTENT_PROVIDER_LATE_CONTROL_INVALID + late_control_ns=$provider_ns; late_control_ip=$provider_ip; late_control_ips=$provider_ip + late_slot=0 + for late_node in relay4 relay5; do + content_provider_node "$late_node" || fail CONTENT_PROVIDER_LATE_CONTROL_INVALID + late_segment=$((86 + late_slot)) + link_nodes "$late_control_ns" "lc$late_slot" "10.241.$late_segment.1/30" \ + "$provider_ns" "lp$late_slot" "10.241.$late_segment.2/30" || fail CONTENT_PROVIDER_LATE_LINK_FAILED + content_provider_filter_link "$late_control_ns" "late_$late_slot" "lc$late_slot" \ + "$late_control_ip" "$provider_ip" || fail CONTENT_PROVIDER_LATE_FILTER_FAILED + content_provider_filter_link "$provider_ns" "late_$late_slot" "lp$late_slot" \ + "$provider_ip" "$late_control_ip" || fail CONTENT_PROVIDER_LATE_FILTER_FAILED + ip -n "$late_control_ns" route replace "$provider_ip/32" via "10.241.$late_segment.2" \ + dev "lc$late_slot" src "$late_control_ip" || fail CONTENT_PROVIDER_LATE_ROUTE_FAILED + ip -n "$provider_ns" route replace "$late_control_ip/32" via "10.241.$late_segment.1" \ + dev "lp$late_slot" src "$provider_ip" || fail CONTENT_PROVIDER_LATE_ROUTE_FAILED + ip -n "$late_control_ns" -j route get "$provider_ip" \ + >"$WORK/$late_prefix-control-$late_node-out.json" || fail CONTENT_PROVIDER_LATE_ROUTE_FAILED + ip -n "$provider_ns" -j route get "$late_control_ip" \ + >"$WORK/$late_prefix-control-$late_node-back.json" || fail CONTENT_PROVIDER_LATE_ROUTE_FAILED + late_control_ips=$late_control_ips,$provider_ip + late_slot=$((late_slot + 1)) + done + PHASE=content-provider-late-recovery-seed + late_root=$WORK/content-provider-seed/late-publication + late_client=$WORK/state-client/content-late-recovery + for late_node in relay4 relay5; do + "$binary_directory/volparossa" --control-socket "$WORK/runtime-$late_node/control/agent.sock" content status \ + >"$WORK/$late_prefix-$late_node-before.json" || fail CONTENT_PROVIDER_LATE_STATUS_FAILED + jq -e '.serving == false and .publications == 0' "$WORK/$late_prefix-$late_node-before.json" >/dev/null \ + || fail CONTENT_PROVIDER_LATE_PROVIDER_NOT_EMPTY + done + if [ -e "$late_root" ] || [ -L "$late_root" ] || [ -e "$late_client" ] || [ -L "$late_client" ]; then + fail CONTENT_PROVIDER_LATE_NOT_FRESH + fi + install -d -o "$AGENT_UID" -g "$AGENT_GID" -m 0700 "$late_client" \ + "$WORK/state-relay4/content-late-recovery" "$WORK/state-relay5/content-late-recovery" + setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" --clear-groups \ + --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- "$binary_directory/examples/content-acceptance-fixture" seed-late-providers "$late_root" \ + "$WORK/state-relay4/content-late-recovery/cache" "$WORK/state-relay5/content-late-recovery/cache" \ + >"$WORK/$late_prefix-seed.log" 2>&1 || fail CONTENT_PROVIDER_LATE_SEED_FAILED + install -o root -g root -m 0600 "$late_root/publication.json" "$WORK/$late_prefix-publication.json" + late_publisher=$(jq -er '.publisher_hex | select(test("^[0-9a-f]{64}$"))' "$WORK/$late_prefix-publication.json") \ + || fail CONTENT_PROVIDER_LATE_KEY_INVALID + install -o "$AGENT_UID" -g "$AGENT_GID" -m 0400 "$late_root/manifest.bin" "$late_client/manifest.bin" + late_pid=$(systemctl show --property=MainPID --value volparossa-alpha-agent@client.service) + case $late_pid in ''|0|*[!0-9]*) fail CONTENT_PROVIDER_LATE_CLIENT_PID_INVALID ;; esac + nsenter --target "$late_pid" --mount setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" \ + --clear-groups --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$late_client/manifest.bin" || fail CONTENT_PROVIDER_LATE_ISOLATION_UNAVAILABLE + for late_private in "$WORK/state-relay3" "$WORK/state-relay4" "$WORK/state-relay5" "$late_root"; do + if nsenter --target "$late_pid" --mount setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" \ + --clear-groups --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$late_private"; then fail CONTENT_PROVIDER_LATE_LOCAL_SHORTCUT; fi + done + PHASE=content-provider-late-recovery-register-a + "$binary_directory/volparossa" --control-socket "$WORK/runtime-relay4/control/agent.sock" \ + content serve --manifest "$late_client/manifest.bin" --publisher-key "$late_publisher" \ + --cache "$WORK/state-relay4/content-late-recovery/cache" --bind 49.165.5.1:18080 \ + --advertised-hostname provider-a.volparossa.test >"$WORK/$late_prefix-relay4-serve.json" \ + || fail CONTENT_PROVIDER_LATE_REGISTER_A_FAILED + start_privacy_observers "$late_prefix-privacy" || fail CONTENT_PROVIDER_LATE_CAPTURE_FAILED + [ -z "$PROVIDER_CONTROL_PID" ] || fail CONTENT_PROVIDER_LATE_CONTROL_CAPTURE_BUSY + ip netns exec "$late_control_ns" python3 "$WORK/bin/privacy-observer.py" content-control \ + "$WORK/$late_prefix-control-privacy.json" "$WORK/$late_prefix-control-privacy.ready" \ + --content-providers "--content-control=$late_control_ips" lc0 lc1 \ + >"$WORK/$late_prefix-control-privacy.log" 2>&1 & + PROVIDER_CONTROL_PID=$! + wait_observer "$PROVIDER_CONTROL_PID" "$WORK/$late_prefix-control-privacy.ready" \ + || fail CONTENT_PROVIDER_LATE_CONTROL_CAPTURE_FAILED + PHASE=content-provider-late-recovery-fetch + python3 -B "$WORK/bin/content-provider-late-recovery.py" run "$WORK" "$binary_directory/volparossa" "$R4" \ + || fail CONTENT_PROVIDER_LATE_FETCH_FAILED + benchmark_capture_paths "$late_prefix-live" mptcp || fail CONTENT_PROVIDER_LATE_PATHS_FAILED + "$binary_directory/volparossa" --control-socket "$WORK/runtime-client/control/agent.sock" content status \ + >"$WORK/$late_prefix-status-after.json" || fail CONTENT_PROVIDER_LATE_STATUS_FAILED + stop_privacy_observers || fail CONTENT_PROVIDER_LATE_CAPTURE_NOT_DRAINED + content_provider_stop_control_observer || fail CONTENT_PROVIDER_LATE_CONTROL_NOT_DRAINED + PHASE=content-provider-late-recovery-cleanup + content_provider_late_recovery_cleanup || fail CONTENT_PROVIDER_LATE_PROCESS_CLEANUP_FAILED + for late_node in relay4 relay5; do + "$binary_directory/volparossa" --control-socket "$WORK/runtime-$late_node/control/agent.sock" content stop \ + >"$WORK/$late_prefix-$late_node-stop.json" || fail CONTENT_PROVIDER_LATE_STOP_FAILED + done + benchmark_disconnect_route "$late_prefix" || fail CONTENT_PROVIDER_LATE_DISCONNECT_FAILED + "$binary_directory/volparossa" --control-socket "$WORK/runtime-client/control/agent.sock" status \ + >"$WORK/$late_prefix-final-status.txt" || fail CONTENT_PROVIDER_LATE_STATUS_FAILED + "$binary_directory/volparossa" --control-socket "$WORK/runtime-client/control/agent.sock" paths \ + >"$WORK/$late_prefix-final-paths.txt" || fail CONTENT_PROVIDER_LATE_STATUS_FAILED + if ! grep -Fx 'connected: false' "$WORK/$late_prefix-final-status.txt" >/dev/null \ + || ! grep -Fx 'active contexts: 0' "$WORK/$late_prefix-final-status.txt" >/dev/null \ + || [ -s "$WORK/$late_prefix-final-paths.txt" ]; then + fail CONTENT_PROVIDER_LATE_DISCONNECT_INCOMPLETE + fi + if [ -L "$late_client/object.bin" ] || [ -L "$late_client/manifest.bin" ]; then + fail CONTENT_PROVIDER_LATE_OUTPUT_UNSAFE + fi + rm -- "$late_client/object.bin" "$late_client/manifest.bin" || fail CONTENT_PROVIDER_LATE_OUTPUT_CLEANUP_FAILED + jq -n '{route_disconnected:true,active_contexts:0,paths_empty:true, + client_output_removed:true,client_manifest_removed:true,client_mount_isolated:true}' \ + >"$WORK/$late_prefix-cleanup.json" + python3 -B "$WORK/bin/content-provider-late-recovery.py" evidence "$WORK" "$WORK/$late_prefix-evidence.json" \ + || fail CONTENT_PROVIDER_LATE_EVIDENCE_INVALID +} diff --git a/tests/integration/content-provider-smoke.py b/tests/integration/content-provider-smoke.py index 7800ca71d..f688ae379 100644 --- a/tests/integration/content-provider-smoke.py +++ b/tests/integration/content-provider-smoke.py @@ -305,6 +305,21 @@ def validate_transfer(evidence): and adaptive["publication"]["publisher_hex"] != publication["publisher_hex"] and adaptive["publication"]["manifest_id"] != publication["manifest_id"], "adaptive proof substituted the topology or reused the earlier route/publication") + # Historical component evidence predates this phase; only new aggregate v2 + # acceptance requires it. If supplied, it must validate and bind to this run. + if "late_provider_recovery" in evidence: + late_check = runpy.run_path(str(Path(__file__).with_name("content-provider-late-recovery.py"))) + late = evidence["late_provider_recovery"] + late_check["validate"](late) + require(late["expected_peers"] == peers + and late["previous_context"] == adaptive["selected_route"]["route_context_id"] + and late["selected_route"]["route_context_id"] not in { + selected["route_context_id"], adaptive["selected_route"]["route_context_id"]} + and late["publication"]["publisher_hex"] not in { + publication["publisher_hex"], adaptive["publication"]["publisher_hex"]} + and late["publication"]["manifest_id"] not in { + publication["manifest_id"], adaptive["publication"]["manifest_id"]}, + "late recovery substituted the topology or reused an earlier route/publication") # Earlier source-exact reports predate this additive phase; never reinterpret them. if "requester_cancellation" in evidence: cancellation = runpy.run_path(str(Path(__file__).with_name("content-cancellation-smoke.py"))) @@ -337,6 +352,14 @@ def build_filter_snapshot(work): return rebuilt +def build_late_provider_recovery(work): + checker = runpy.run_path(str(Path(__file__).with_name("content-provider-late-recovery.py"))) + rebuilt = checker["build_evidence"](work) + require(rebuilt == read(work / "content-provider-late-recovery-evidence.json"), + "late recovery differs from its raw discovery, transfer or cleanup evidence") + return rebuilt + + def build_evidence(work): layout = read(work / "content-provider-layout.json") evidence = dict(success=True, publication=read(work / "content-provider-publication.json"), @@ -356,6 +379,7 @@ def build_evidence(work): site_publication=build_site_publication(work), filter_snapshot=build_filter_snapshot(work), adaptive_workers=build_adaptive_workers(work), + late_provider_recovery=build_late_provider_recovery(work), requester_cancellation=read(work / "content-provider-cancellation.json"), expected_peers=read(work / "a01-expected-peers.json"), selected_route=read(work / "content-provider-live-selection.json"), @@ -405,7 +429,19 @@ def validate_control_underlay(evidence, control_node, provider_nodes): "actual kernel route does not bind the two advertised public endpoints") -def validate_report(report, revision): +def validate_report(report, revision, *, historical=False): + expected_version = 1 if historical else 2 + require(type(report.get("schema_version")) is int + and report["schema_version"] == expected_version, + "wrong aggregate report version for current or historical validation") + if historical: + require("late_provider_recovery" not in report + and "late_provider_recovery" not in report["transfer"], + "historical v1 report must not claim the later recovery phase") + else: + require(report.get("late_provider_recovery") is True + and report["transfer"].get("late_provider_recovery", {}).get("success") is True, + "current aggregate report requires actual late complementary-provider recovery") require(report["report_kind"] == "volparossa-native-content-providers" and report["source_revision"] == revision and report["success"] is True and report["runner_exit_status"] == 0 @@ -428,14 +464,14 @@ def validate_report(report, revision): if __name__ == "__main__": try: if len(sys.argv) != 4: - raise ValueError("usage: evidence WORK OUTPUT | user-publication WORK OUTPUT | report REPORT REVISION") + raise ValueError("usage: evidence WORK OUTPUT | user-publication WORK OUTPUT | report REPORT REVISION | historical-report REPORT REVISION") if sys.argv[1] in ("evidence", "user-publication"): builder = build_evidence if sys.argv[1] == "evidence" else build_user_publication result = builder(Path(sys.argv[2])) with Path(sys.argv[3]).open("x", encoding="ascii") as target: json.dump(result, target, sort_keys=True, separators=(",", ":")) - elif sys.argv[1] == "report": - validate_report(read(Path(sys.argv[2])), sys.argv[3]) + elif sys.argv[1] in ("report", "historical-report"): + validate_report(read(Path(sys.argv[2])), sys.argv[3], historical=sys.argv[1] == "historical-report") else: raise ValueError("unknown evidence mode") except (KeyError, TypeError, ValueError, OSError) as error: diff --git a/tests/integration/content-provider-smoke.sh b/tests/integration/content-provider-smoke.sh index 72b6ce151..e4f3608fa 100644 --- a/tests/integration/content-provider-smoke.sh +++ b/tests/integration/content-provider-smoke.sh @@ -327,6 +327,7 @@ content_provider_run() { fail CONTENT_PROVIDER_ROUTE_CLEANUP_FAILED fi content_provider_adaptive_run + content_provider_late_recovery_run content_provider_cancellation_run python3 -B "$source_directory/tests/integration/content-provider-smoke.py" \ evidence "$WORK" "$WORK/content-provider-evidence.json" || fail CONTENT_PROVIDER_EVIDENCE_INVALID @@ -355,16 +356,17 @@ content_provider_finalize_report() { --argjson complete "$CLEANUP_COMPLETE" \ --argjson remaining "$REMAINING_OWNED_OBJECTS" ' $evidence_input[0] as $evidence | $host_input[0] as $host | - {schema_version:1,report_kind:"volparossa-native-content-providers", + {schema_version:2,report_kind:"volparossa-native-content-providers", source_revision:$revision,run_id:$run_id,phase:$phase, success:($status == 0 and $evidence.success == true and $evidence.ordinary_publication.success == true and $evidence.site_publication.success == true and $evidence.filter_snapshot.success == true and - $evidence.adaptive_workers.success == true and $evidence.requester_cancellation.success == true and $complete and + $evidence.adaptive_workers.success == true and $evidence.late_provider_recovery.success == true and + $evidence.requester_cancellation.success == true and $complete and $remaining == 0 and $host.unchanged == true),transfer:$evidence, runner_exit_status:$status,observed_blocker:(if $blocker == "NONE" then null else $blocker end), cleanup:{complete:$complete,remaining_owned_objects:$remaining},host_state:($host | del(.acceptance_id)), - scope:"explicit native publication and cooperative-origin HTTPS for the same object, normal named static-site HTTP assets, exact public filter export and networkless cache reuse, two policy-authorized providers via generic DHT/control-relay discovery and protected MPTCP/TLS/WireGuard, complete peers and missing origin ranges; named requester cancellation and agent shutdown during pending bootstrap", + scope:"explicit native publication and cooperative-origin HTTPS for the same object, normal named static-site HTTP assets, exact public filter export and networkless cache reuse, two policy-authorized providers via generic DHT/control-relay discovery and protected MPTCP/TLS/WireGuard, complete peers and missing origin ranges; separate late complementary-provider native recovery; named requester cancellation and agent shutdown during pending bootstrap", general_nat_reachability_claimed:false,full_c02_claimed:false, explicit_origin_authenticated_https:($evidence.https.success == true), normal_user_publication:($evidence.ordinary_publication.success == true), @@ -372,6 +374,7 @@ content_provider_finalize_report() { native_static_site:($evidence.site_publication.success == true), public_filter_snapshot:($evidence.filter_snapshot.success == true), adaptive_provider_workers:($evidence.adaptive_workers.success == true), + late_provider_recovery:($evidence.late_provider_recovery.success == true), requester_cancellation:($evidence.requester_cancellation.success == true), browser_integration_claimed:false,arbitrary_https_integration_claimed:false, speed_improvement_claimed:false,full_alpha_acceptance_claimed:false}' \ diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index e4768ea9a..0d7315f22 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -567,6 +567,8 @@ print_plan() { ' open by publisher/name in the Client, verify HTTP assets/ranges and SIGTERM spool cleanup (no browser engine);' \ ' after existing phases, create three separate control-only broker links to R3/R4/R5 and fetch 15 disjoint chunks on a fresh route;' \ ' require real triple provider payload overlap, exact reconstruction and normal service/route teardown;' \ + ' then admit a complementary provider only after one real one-offer discovery snapshot on another fresh route;' \ + ' bound the disposable provider gate below the unchanged Exit connect timeout; require real rediscovery or fail;' \ ' pause only the exact guest Client helper via pidfd, cancel a pending named download and require independent cached delivery;' \ ' stop the original Client agent while bootstrap is pending, resume that helper before teardown and require successful drain;' \ ' retain complete privacy captures/cleanup; no general NAT, arbitrary-browser HTTPS or full-C02 claim.' @@ -1356,7 +1358,8 @@ if [ "$scenario" = content-provider ]; then content-provider-https-smoke.sh content-provider-https-smoke.py content-publication-smoke.sh content-named-smoke.sh \ content-provider-site-smoke.sh content-provider-site-smoke.py \ content-provider-filter-smoke.sh content-provider-filter-smoke.py \ - content-provider-adaptive-smoke.sh content-provider-adaptive-smoke.py content-cancellation-smoke.py; do + content-provider-adaptive-smoke.sh content-provider-adaptive-smoke.py \ + content-provider-late-recovery.sh content-provider-late-recovery.py content-cancellation-smoke.py; do if [ ! -f "$source_directory/tests/integration/$provider_fixture" ] \ || [ -L "$source_directory/tests/integration/$provider_fixture" ]; then printf '%s\n' 'content provider fixture unavailable' >&2 @@ -2152,6 +2155,10 @@ cleanup() { [ "$FINALIZED" = no ] || exit "$original_status" FINALIZED=yes trap - EXIT HUP INT TERM + # Release the fixture gate and reap its exact owned fetch before services or namespaces. + if [ "$scenario" = content-provider ] && command -v content_provider_late_recovery_cleanup >/dev/null 2>&1; then + content_provider_late_recovery_cleanup || original_status=1 + fi # Resume a fixture-paused helper BEFORE any namespace, process or service teardown. if [ "$scenario" = content-provider ] && command -v content_provider_cancellation_resume >/dev/null 2>&1; then content_provider_cancellation_resume || original_status=1 @@ -2689,6 +2696,8 @@ if [ "$scenario" = content-provider ]; then . "$source_directory/tests/integration/content-provider-filter-smoke.sh" # shellcheck source=tests/integration/content-provider-adaptive-smoke.sh . "$source_directory/tests/integration/content-provider-adaptive-smoke.sh" + # shellcheck source=tests/integration/content-provider-late-recovery.sh + . "$source_directory/tests/integration/content-provider-late-recovery.sh" if [ -n "$weather_client_bundle" ]; then # shellcheck source=tests/integration/weather-client-delivery.sh . "$source_directory/tests/integration/weather-client-delivery.sh" @@ -2936,6 +2945,10 @@ fi if [ "$scenario" = content-provider ]; then install -o root -g root -m 0555 "$source_directory/tests/integration/content-provider-readiness.py" \ "$WORK/bin/content-provider-readiness.py" + for late_recovery_helper in content-provider-late-recovery.py content-provider-https-smoke.py content-network-smoke.py; do + install -o root -g root -m 0555 "$source_directory/tests/integration/$late_recovery_helper" \ + "$WORK/bin/$late_recovery_helper" + done fi if [ "$scenario" = content-custody ] || [ "$scenario" = agent-artifact ] || [ "$scenario" = agent-jobs ]; then for custody_script in content-custody-smoke.py content-retain-smoke.py content-provider-smoke.py content-provider-https-smoke.py content-network-smoke.py; do @@ -6193,6 +6206,8 @@ start_privacy_observers() { [ "$scenario" = content-https ] || return 1 ;; weather-products-delivery-weather-privacy|weather-products-delivery-air-quality-privacy) [ "$scenario" = content-provider ] || return 1 ;; + content-provider-late-recovery-privacy) + [ "$scenario" = content-provider ] || return 1 ;; content-provider-privacy|content-provider-adaptive-privacy|content-provider-adaptive-https-privacy|content-provider-https-complete-privacy|content-provider-https-missing-privacy|content-provider-https-baseline-privacy|content-provider-https-origin-only-privacy|content-provider-https-auto-privacy|content-provider-https-digest-origin-only-privacy|content-provider-https-digest-peers-first-privacy|content-provider-https-limited-origin-only-privacy|content-provider-https-limited-peers-first-privacy|content-provider-https-limited-auto-privacy|content-provider-https-checksum-origin-only-privacy|content-provider-https-checksum-peers-first-privacy|content-provider-https-checksum-mismatch-privacy|content-provider-user-privacy|content-provider-named-privacy|content-provider-site-privacy|content-provider-site-cache-only-privacy|content-provider-filter-cold-privacy|content-provider-filter-warm-privacy|weather-client-delivery-privacy) [ "$scenario" = content-provider ] || return 1 ;; content-message-publication-privacy) diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 49f33e4c4..db313d482 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -342,6 +342,7 @@ print_plan() { ' two disposable UDP41000-only broker-provider links with exact route and drained capture evidence;' \ ' actual generic DHT/control-relay discovery and protected MPTCP reconstruction after publisher removal;' \ ' exact policy-only DNS destinations, isolated Client files, both provider PeerIds and complete privacy/cleanup;' \ + ' separate late complementary-provider phase: first one-offer snapshot, bounded gate, real rediscovery and no retry substitution;' \ ' cooperative-origin HTTPS same-object complete/missing-ranges cases; no general NAT/browser HTTPS/full-C02 claim.' elif [ "$scenario" = cloud-private-upload ]; then printf '%s\n' \ @@ -1110,6 +1111,8 @@ WEATHER_PRODUCTS_NAMES = {name.replace("weather-client-delivery-", "weather-prod WEATHER_PRODUCTS_FAILURE_NAMES = {f"weather-products-delivery-{suffix}.json" for suffix in ( "acquisition-failure", "export-status", "tree-before", "input-cleanup")} WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", *WEATHER_CLIENT_HELPERS) +LATE_PROVIDER_NAMES = {f"content-provider-late-recovery-{suffix}.json" for suffix in ( + "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence")} def joined_client_helper(work, names, entrypoint): @@ -1365,6 +1368,12 @@ def collect(home, opt, revision, scenario, guest_status, for root, label in roots: if root.is_dir() and not root.is_symlink(): candidates.extend(weather_client_candidates(root, label, helper, target)) + # Preserve the small, exact late-provider diagnostic set before older + # content globs exhaust the unchanged bounds. These are incomplete-run + # observations, never proof of successful cleanup or live acceptance. + for root, label in roots: + if root.is_dir() and not root.is_symlink(): + candidates.extend((root / name, f"{label}/{name}") for name in sorted(LATE_PROVIDER_NAMES)) for root, label in roots: if root.is_symlink() or not root.is_dir(): continue @@ -1513,7 +1522,9 @@ def collect(home, opt, revision, scenario, guest_status, candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("download-sharing-*"))[:64] if re.fullmatch(r"download-sharing-[a-z0-9-]+\.(json|txt|log)", path.name)) candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("content-*"))[:48] - if re.fullmatch(r"content-[a-z0-9-]+\.(json|txt|log|err|out)", path.name)) + if re.fullmatch(r"content-[a-z0-9-]+\.(json|txt|log|err|out)", path.name) + and (not path.name.startswith("content-provider-late-recovery-") + or scenario == "content-provider" and path.name in LATE_PROVIDER_NAMES)) candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("agent-public-task-*"))[:32] if re.fullmatch(r"agent-public-task-[a-z0-9-]+\.(json|err|log)", path.name)) candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("agent-public-document-*"))[:32] @@ -1531,7 +1542,7 @@ def collect(home, opt, revision, scenario, guest_status, if observed is None: continue data, original_size = observed - if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES | WEATHER_PRODUCTS_FAILURE_NAMES and original_size > len(data): + if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES | WEATHER_PRODUCTS_FAILURE_NAMES | LATE_PROVIDER_NAMES and original_size > len(data): continue # Preserve complete validated original bytes, never a JSON tail. if scenario == "browser-network" and path.name in browser_names and original_size > len(data): # A truncated JSON tail is not reconstructible structured evidence. diff --git a/tests/integration/test-content-provider-late-recovery.py b/tests/integration/test-content-provider-late-recovery.py new file mode 100644 index 000000000..13cfe05b4 --- /dev/null +++ b/tests/integration/test-content-provider-late-recovery.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Synthetic late-recovery gates, not a live DHT/datapath proof.""" +import copy +import json +from pathlib import Path +import runpy +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).parent +CHECK = runpy.run_path(str(HERE / "content-provider-late-recovery.py")) +ADAPTIVE = runpy.run_path(str(HERE / "test-content-provider-adaptive-smoke.py")) + + +def gate_fixture(): + table = CHECK["TABLE"] + return dict(nftables=[dict(metainfo={}), dict(table=dict(family="inet", name=table)), + dict(chain=dict(family="inet", table=table, name="input", type="filter", hook="input", prio=-30, policy="accept")), + dict(rule=dict(family="inet", table=table, chain="input", expr=[ + dict(match=dict(op="==", left=dict(meta=dict(key="iifname")), right="r4x")), + *[dict(match=dict(op="==", left=dict(payload=dict(protocol="ip", field=field)), right=address)) + for field, address in (("saddr", "46.162.3.1"), ("daddr", "49.165.5.1"))], + dict(match=dict(op="==", left=dict(payload=dict(protocol="tcp", field="dport")), right=18080)), + dict(counter=dict(packets=1, bytes=60)), {"drop": None}]))]) + + +def fixture(control_node="relay2"): + value = ADAPTIVE["fixture"](control_node) + for key in ("https", "control_filter", "provider_payload_overlap_ns"): value.pop(key) + value.update(schema_version=1, concurrent_provider_overlap_claimed=False, previous_context="b"*32, gate=gate_fixture()) + value["publication"] = dict(report_kind="volparossa-content-late-provider-seed", bytes=CHECK["BYTES"], chunks=9, + object_sha256=CHECK["SHA"], name="disposable-late-native-publication", publisher_hex="3"*64, manifest_id="f"*64, + publisher_removed=True, publisher_private_key_persisted=False, recipient_encrypted=False, + replica_a_chunks=5, replica_b_chunks=4, replica_a_bytes=1048699, replica_b_bytes=1048576, + created_unix_seconds=1000, expires_unix_seconds=4600) + for route in (value["selected_before"], value["selected_route"]): + route["route_context_id"] = "c"*32 + for item in route["paths"]: item["route_context_id"] = "c"*32 + nodes, peers = CHECK["NODES"], value["expected_peers"] + value["layout"]["provider_nodes"] = list(nodes) + value["fetch"].update(bytes=CHECK["BYTES"], chunks=9, providers_used=2, peer_bytes=CHECK["BYTES"], + provider_peer_ids=[peers[node] for node in nodes]) + value["providers"].pop("relay3") + for role, capture in value["privacy"].items(): + for node, counts in capture["provider_application"].items(): + counts.update(request_packets=100 if role == "exit" and node in nodes else 0, + response_packets=100 if role == "exit" and node in nodes else 0, + response_payload_bytes=1100000 if role == "exit" and node in nodes else 0) + addresses = CHECK["BASE"]["PUBLIC_IPS"] + control = value["control_underlay"]["capture"] + control.update(interfaces=["lc0", "lc1"], observed_frames=40, + content_control_pairs={f"lc{i}": [addresses[control_node], addresses[node]] for i, node in enumerate(nodes)}, + content_control_packets={f"lc{i}": dict(inbound=10, outbound=10) for i in range(2)}, + interface_statistics={f"lc{i}": dict(intake_stopped=True, observed_frames=20, packet_socket_packets=20, + packet_socket_drops=0) for i in range(2)}) + value["control_underlay"]["routes"] = {node: dict( + out=[dict(dev=f"lc{i}", gateway=f"10.241.{86+i}.2", prefsrc=addresses[control_node], dst=addresses[node])], + back=[dict(dev=f"lp{i}", gateway=f"10.241.{86+i}.1", prefsrc=addresses[node], dst=addresses[control_node])]) + for i, node in enumerate(nodes)} + def row(event, when): return dict(event=event, unix_ms=when) + events = CHECK["EVENTS"] + value["first_snapshot"] = dict(client=[row(events[1], 1002), row(events[0], 1002)], + broker=[row(events[3], 1001), row(events[4], 1001)]) + value["final_snapshot"] = copy.deepcopy(value["first_snapshot"]) + value["final_snapshot"]["client"] += [row(events[1], 2000), row(events[0], 2000)] + value["final_snapshot"]["broker"] += [row(events[3], 1900), row(events[4], 1901), row(events[4], 1901)] + value["output"] = dict(bytes=CHECK["BYTES"], sha256=CHECK["SHA"], event_baseline_unix_ms=1000, + gate_started_ns=1_000_000_000, fetch_started_ns=1_100_000_000, first_snapshot_observed_ns=2_000_000_000, + late_activation_started_ns=2_100_000_000, late_activation_acknowledged_ns=2_200_000_000, + gate_released_ns=2_300_000_000, fetch_completed_ns=5_000_000_000, + cold_cache=True, consumer_provider_hints=False, origin_fallback=False) + value["run_cleanup"] = dict(fetch_reaped=True, gate_removed=True, operation_completed=True) + value["cleanup"] = dict(route_disconnected=True, active_contexts=0, paths_empty=True, + client_output_removed=True, client_manifest_removed=True, client_mount_isolated=True) + return value + + +def raw_files(value): + prefix = CHECK["PREFIX"] + files = {f"{prefix}-{suffix}.json": value[key] for key, suffix in (("publication", "publication"), ("layout", "layout"), + ("status_before", "status-before"), ("status_after", "status-after"), ("fetch", "fetch"), ("output", "object"), + ("selected_before", "selection"), ("selected_route", "live-selection"), ("cleanup", "cleanup"), + ("first_snapshot", "first-snapshot"), ("final_snapshot", "final-snapshot"), ("gate", "gate-installed"), + ("run_cleanup", "run-cleanup"))} + files["a01-expected-peers.json"] = value["expected_peers"] + broker = next(node for node in ("relay0", "relay1", "relay2") + if value["expected_peers"][node] == value["layout"]["control_relay_peer_id"]) + files["content-provider-adaptive-selection.json"] = ADAPTIVE["fixture"](broker)["selected_before"] + files[f"{prefix}-final-status.txt"] = "connected: false\nactive contexts: 0\n" + files[f"{prefix}-final-paths.txt"] = "" + for node, records in value["providers"].items(): + for operation, record in records.items(): files[f"{prefix}-{node}-{operation}.json"] = record + for role, capture in value["privacy"].items(): files[f"{prefix}-privacy-{role}.json"] = capture + files[f"{prefix}-control-privacy.json"] = value["control_underlay"]["capture"] + for node, routes in value["control_underlay"]["routes"].items(): + for direction, route in routes.items(): files[f"{prefix}-control-{node}-{direction}.json"] = route + return files + + +class LateRecovery(unittest.TestCase): + def test_exact_late_phase_is_independent_of_payload_overlap(self): + for broker in ("relay0", "relay1", "relay2"): + value = fixture(broker) + value["privacy"]["exit"]["provider_payload_timing"] = {"enabled": False} + CHECK["validate"](value) + self.assertNotEqual(CHECK["SHA"], CHECK["BASE"]["SHA"]) + + def test_rejects_no_rediscovery_early_b_warm_cache_unprotected_paths_and_bad_cleanup(self): + mutations = [(("first_snapshot", "broker"), []), (("final_snapshot", "client"), fixture()["first_snapshot"]["client"]), + (("output", "cold_cache"), False), (("output", "consumer_provider_hints"), True), + (("output", "origin_fallback"), True), (("output", "late_activation_started_ns"), 1), + (("output", "gate_released_ns"), 8_000_000_000), (("output", "fetch_completed_ns"), 400_000_000_000), + (("fetch", "providers_used"), 1), (("fetch", "origin_body_bytes"), 1), + (("privacy", "client", "direct_client_exit_packets"), 1), + (("run_cleanup", "gate_removed"), False), (("cleanup", "client_mount_isolated"), False), + (("providers", "relay5", "before", "serving"), True), (("publication", "object_sha256"), CHECK["BASE"]["SHA"])] + for keys, changed in mutations: + with self.subTest(keys=keys): + value = fixture(); target = value + for key in keys[:-1]: target = target[key] + target[keys[-1]] = changed + with self.assertRaises(ValueError): CHECK["validate"](value) + + def test_gate_can_only_drop_exact_exit_to_a_tcp_content_not_udp_discovery(self): + CHECK["validate_gate"](gate_fixture()) + for keys, changed in [(("nftables", 2, "chain", "hook"), "output"), + (("nftables", 3, "rule", "expr", 0, "match", "right"), "underlay"), + (("nftables", 3, "rule", "expr", 3, "match", "right"), 41000), + (("nftables", 3, "rule", "expr", 3, "match", "left", "payload", "protocol"), "udp")]: + value = gate_fixture(); target = value + for key in keys[:-1]: target = target[key] + target[keys[-1]] = changed + with self.assertRaises(ValueError): CHECK["validate_gate"](value) + + def test_log_snapshots_are_bounded_and_ignore_preexisting_or_noncontent_events(self): + raw = b"1000 INFO event=CONTENT_DISCOVERY_COMPLETED\n1001 INFO event=CONTENT_DISCOVERY_RECEIVED_OFFERS\n" + self.assertEqual(CHECK["event_rows"](raw, 1000), [dict(unix_ms=1001, event="CONTENT_DISCOVERY_RECEIVED_OFFERS")]) + self.assertEqual(CHECK["event_rows"](b"1001 INFO event=OTHER_EVENT\n", 1000), []) + with self.assertRaises(ValueError): CHECK["event_rows"](b"1001 INFO event=OTHER_EVENT\n"*400, 1000) + with self.assertRaises(ValueError): CHECK["event_rows"](b"x"*(CHECK["LIMIT"]+1), 1000) + + def test_cleanup_releases_gate_before_process_ownership_check_and_attempts_both(self): + events = [] + def process(_work): events.append("process"); raise ValueError("replaced") + with patch.dict(CHECK["cleanup"].__globals__, stop_fetch=process, remove_gate=lambda work: events.append("gate")): + with self.assertRaises(ValueError): CHECK["cleanup"](Path("/not-an-actual-guest")) + self.assertEqual(events, ["gate", "process"]) + events.clear() + def gate(_work): events.append("gate"); raise ValueError("gate failed") + with patch.dict(CHECK["cleanup"].__globals__, stop_fetch=lambda work: events.append("process"), remove_gate=gate): + with self.assertRaises(ValueError): CHECK["cleanup"](Path("/not-an-actual-guest")) + self.assertEqual(events, ["gate", "process"]) + with tempfile.TemporaryDirectory() as temporary: CHECK["cleanup"](Path(temporary)) + + def test_coordinator_activates_b_only_after_snapshot_and_always_releases_owned_gate(self): + for failure in (None, "snapshot", "activation"): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); native = work / "state-client/content-late-recovery"; native.mkdir(parents=True) + binary = work / "volparossa"; binary.touch() + value, calls, clock = fixture(), [], [1_000_000_000] + for name, contents in (("layout", value["layout"]), ("peers", value["expected_peers"]), + ("publication", value["publication"])): + CHECK["write"](CHECK["path"](work, name), contents) + class Process: + pid = 77 + returncode = None + def poll(self): return self.returncode + def wait(self, timeout): + calls.append("fetch-wait") + if self.returncode is None: + (native / "object.bin").write_bytes(b"x"*CHECK["BYTES"]) + self.returncode = 0 + return self.returncode + def terminate(self): calls.append("fetch-terminate"); self.returncode = -15 + def kill(self): calls.append("fetch-kill"); self.returncode = -9 + process = Process() + def spawn(args, **kwargs): + calls.append("fetch-start") + self.assertNotIn("--provider", args); self.assertNotIn("--bind", args) + self.assertFalse(any("peer-" in arg or "volparossa.test" in arg for arg in args)) + kwargs["stdout"].write(json.dumps(value["fetch"]).encode()) + return process + def now(): clock[0] += 100_000_000; return clock[0] + def execute(args, deadline, input_data=None): + if args[-2:] == ["list", "tables"]: return b'{"nftables":[]}' + if args[-2:] == ["-f", "-"]: calls.append("gate-install"); return b"" + if "list" in args and "table" in args: return json.dumps(gate_fixture()).encode() + if "delete" in args: calls.append("gate-release"); return b"" + if "logs" in args: + label = "client" if "runtime-client" in args[2] else "broker" + calls.append("snapshot-"+label) + if failure == "snapshot": return b"" + snapshot = value["final_snapshot"] if process.returncode == 0 else value["first_snapshot"] + return "".join(f"{row['unix_ms']} INFO event={row['event']}\n" for row in snapshot[label]).encode() + if "serve" in args: + calls.append("activate-b") + self.assertLess(calls.index("snapshot-client"), calls.index("activate-b")) + self.assertLess(calls.index("snapshot-broker"), calls.index("activate-b")) + if failure == "activation": raise ValueError("registration failed") + return json.dumps(value["providers"]["relay5"]["serve"]).encode() + self.fail(f"unexpected command {args}") + with patch.dict(CHECK["run"].__globals__, guest=lambda work: None, + namespace=lambda work: dict(name="va-00000000-r4", identity="1:2"), command=execute, + process_identity=lambda pid: dict(pid=pid, start_ticks=100), + remove_gate=lambda work, **kwargs: calls.append("cleanup-gate")), \ + patch.object(CHECK["subprocess"], "Popen", side_effect=spawn), \ + patch.object(CHECK["time"], "monotonic_ns", side_effect=now), \ + patch.object(CHECK["time"], "time_ns", return_value=1_000_000_000), \ + patch.object(CHECK["time"], "sleep"): + if failure: + with self.assertRaises(ValueError): CHECK["run"](work, binary, "va-00000000-r4") + else: CHECK["run"](work, binary, "va-00000000-r4") + self.assertIn("cleanup-gate", calls) + self.assertIsNotNone(process.returncode) + cleanup = CHECK["read"](CHECK["path"](work, "run-cleanup")) + self.assertEqual(cleanup, dict(fetch_reaped=True, gate_removed=True, operation_completed=failure is None)) + if failure == "snapshot": self.assertNotIn("activate-b", calls) + elif failure is None: + self.assertLess(calls.index("activate-b"), calls.index("gate-release")) + self.assertLess(calls.index("gate-release"), calls.index("fetch-wait")) + if failure: self.assertLess(calls.index("cleanup-gate"), calls.index("fetch-terminate")) + + def test_builder_requires_exact_original_snapshots_and_disconnected_route(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); expected = fixture() + for name, value in raw_files(expected).items(): + (work / name).write_text(value if isinstance(value, str) else json.dumps(value)) + self.assertEqual(CHECK["build_evidence"](work), expected) + (work / f"{CHECK['PREFIX']}-final-paths.txt").write_text("active path\n") + with self.assertRaises(ValueError): CHECK["build_evidence"](work) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/integration/test-content-provider-smoke.py b/tests/integration/test-content-provider-smoke.py index e4d4e08df..743dd090b 100644 --- a/tests/integration/test-content-provider-smoke.py +++ b/tests/integration/test-content-provider-smoke.py @@ -11,6 +11,7 @@ import socket import struct import subprocess +import tarfile import tempfile from types import SimpleNamespace import unittest @@ -19,7 +20,7 @@ CHECK = runpy.run_path(str(HERE / "content-provider-smoke.py")) -def fixture(control_node="relay2"): +def fixture(control_node="relay2", *, include_late=True): peers = {name: f"peer-{name}" for name in (*CHECK["ROLES"], "relay3", "relay4", "relay5")} provider_nodes = [node for node in CHECK["CANDIDATES"] if node != control_node][:2] paths = [dict(route_context_id="a" * 32, path_id=index + 1, @@ -92,6 +93,9 @@ def fixture(control_node="relay2"): evidence["site_publication"] = runpy.run_path(str(HERE / "test-content-provider-site-smoke.py"))["fixture"](control_node) evidence["filter_snapshot"] = runpy.run_path(str(HERE / "test-content-provider-filter-smoke.py"))["fixture"](control_node) evidence["adaptive_workers"] = runpy.run_path(str(HERE / "test-content-provider-adaptive-smoke.py"))["fixture"]() + if include_late: + evidence["late_provider_recovery"] = runpy.run_path( + str(HERE / "test-content-provider-late-recovery.py"))["fixture"]() cancellation = runpy.run_path(str(HERE / "content-cancellation-smoke.py")) component, _, _ = cancellation["sample"]() named_fetch = evidence["named_publication"]["fetch"] @@ -182,6 +186,111 @@ def user_fixture(base): class ContentProviderContract(unittest.TestCase): + def test_late_recovery_is_separate_and_bound_to_prior_adaptive_context(self): + evidence = fixture() + CHECK["validate_transfer"](evidence) + # Retained component evidence remains readable without acquiring a new claim. + CHECK["validate_transfer"](fixture(include_late=False)) + for mutate in ( + lambda value: value["late_provider_recovery"].update(success=False), + lambda value: value["late_provider_recovery"]["expected_peers"].update(client="wrong-client"), + lambda value: value["late_provider_recovery"].update(previous_context="f" * 32), + lambda value: value["late_provider_recovery"]["selected_route"].update(route_context_id="a" * 32), + lambda value: value["late_provider_recovery"]["publication"].update( + publisher_hex=value["publication"]["publisher_hex"]), + lambda value: value["late_provider_recovery"]["publication"].update( + manifest_id=value["adaptive_workers"]["publication"]["manifest_id"]), + lambda value: value.update(provider_payload_overlap_ns=0), + ): + wrong = copy.deepcopy(evidence) + mutate(wrong) + with self.assertRaises((ValueError, KeyError)): + CHECK["validate_transfer"](wrong) + + def test_late_phase_hooks_preflight_and_cleanup_cannot_be_skipped(self): + source = HERE.joinpath("kvm-alpha-topology.sh").read_text(encoding="utf-8") + runtime = HERE.joinpath("content-provider-smoke.sh").read_text(encoding="utf-8") + self.assertIn("content_provider_adaptive_run\n content_provider_late_recovery_run\n" + " content_provider_cancellation_run", runtime) + self.assertIn("$evidence.late_provider_recovery.success == true", runtime) + self.assertIn("{schema_version:2,report_kind:", runtime) + preflight = source.split('for provider_fixture in ', 1)[1].split('; do', 1)[0] + for name in ("content-provider-late-recovery.sh", "content-provider-late-recovery.py"): + self.assertIn(name, preflight) + self.assertIn('. "$source_directory/tests/integration/content-provider-late-recovery.sh"', source) + cleanup = source.split("\ncleanup() {\n", 1)[1].split("\n}\n", 1)[0] + hook = "content_provider_late_recovery_cleanup || original_status=1" + self.assertLess(cleanup.index(hook), cleanup.index('kill -TERM')) + self.assertLess(cleanup.index(hook), cleanup.index('retire_unit')) + early = cleanup.split(' if [ "$scenario" = dns-cache ]', 1)[0] + # Execute the actual trap prelude with inert hooks: failure is retained, + # cancellation cleanup still runs, and no service/network tool is called. + script = ('set -eu\nscenario=content-provider\nPHASE=late-failure\nOBSERVED_BLOCKER=late-blocker\n' + 'FINALIZED=no\ncontent_provider_late_recovery_cleanup() { printf "late\\n"; return "$1"; }\n') + # The hook has no arguments; bake the synthetic result into its body. + for result in (0, 1): + prefix = script.replace('return "$1"', f'return {result}') + prefix += 'content_provider_cancellation_resume() { printf "cancel\\n"; }\n' + body = prefix + 'cleanup_prelude() {\n' + early + '\n}\ntrue\ncleanup_prelude\n' + body += 'printf "%s:%s:%s\\n" "$original_status" "$failure_phase" "$failure_reason"\n' + checked = subprocess.run(["sh", "-eu", "-c", body], capture_output=True, timeout=2, check=False) + self.assertEqual(checked.returncode, 0, checked.stderr) + self.assertEqual(checked.stdout.decode(), f"late\ncancel\n{result}:late-failure:late-blocker\n") + stage = source.split('for late_recovery_helper in ', 1)[1].split('\n done', 1)[0] + for name in ("content-provider-late-recovery.py", "content-provider-https-smoke.py", "content-network-smoke.py"): + self.assertIn(name, stage) + self.assertIn('install -o root -g root -m 0555', stage) + workflow = HERE.parents[1].joinpath('.github/workflows/alpha-topology.yml').read_text(encoding='utf-8') + self.assertIn('python3 -B tests/integration/test-content-provider-late-recovery.py', workflow) + registration = source.split("start_privacy_observers() {\n", 1)[1].split(" set --\n", 1)[0] + script = "registered() {\n" + registration + '}\nscenario=$1\nregistered "$2"\n' + for scenario in ("content-provider", "content", "content-https"): + for prefix in ("content-provider-late-recovery-privacy", "content-provider-late-recovery-unregistered"): + result = subprocess.run(["sh", "-eu", "-c", script, "sh", scenario, prefix], + capture_output=True, timeout=2, check=False) + self.assertEqual(result.returncode == 0, scenario == "content-provider" + and prefix == "content-provider-late-recovery-privacy") + + def test_late_emergency_metadata_survives_bounds_without_claiming_success(self): + source = HERE.joinpath("run-alpha-topology-vm.sh").read_text(encoding="utf-8") + embedded = source.split("<<'GUEST_DIAGNOSTICS_PYTHON'\n", 1)[1].split("\nGUEST_DIAGNOSTICS_PYTHON\n", 1)[0] + api = {"__name__": "inert_late_collector"} + exec(compile(embedded, "late-collector", "exec"), api) + self.assertEqual(api["LATE_PROVIDER_NAMES"], {f"content-provider-late-recovery-{suffix}.json" + for suffix in ("first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence")}) + for scenario in ("content-provider", "content"): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + home, opt, proc, cgroups = (base / name for name in ("home", "opt", "proc", "cgroups")) + for path in (home, opt, proc, cgroups): path.mkdir() + work = opt / ("va." + "a" * 32 + ".ABC123") + work.mkdir() + (home / "alpha-output").mkdir() + # Earlier public content files would consume the old sorted48 + # scan before the later-named recovery proof was reached. + for index in range(60): (work / f"content-before-{index:02}.json").write_text("{}") + for name in api["LATE_PROVIDER_NAMES"]: (work / name).write_text('{"observed":true}') + oversized = "content-provider-late-recovery-evidence.json" + (work / oversized).write_bytes(b" " * (api["FILE_LIMIT"] + 1)) + alias = "content-provider-late-recovery-cleanup.json" + (work / alias).unlink() + (work / alias).symlink_to(work / "content-before-00.json") + (work / "content-provider-late-recovery-fetch.state").write_text("PRIVATE PROCESS INTENT") + (work / "content-provider-late-recovery-unknown.json").write_text("PRIVATE UNKNOWN") + (work / "content-provider-late-recovery-fetch.err").write_text("PRIVATE RAW LOG") + archive = api["collect"](home, opt, "b" * 40, scenario, 124, cgroups, proc) + with tarfile.open(archive, "r:gz") as bundle: + files = {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()} + selected = {name.rsplit("/", 1)[-1] for name in files if "late-recovery" in name} + expected = api["LATE_PROVIDER_NAMES"] - {oversized, alias} if scenario == "content-provider" else set() + self.assertEqual(selected, expected) + self.assertFalse(any(b"PRIVATE" in value for value in files.values())) + summary = json.loads(files["vm-incomplete.json"]) + self.assertIs(summary["success"], False) + self.assertEqual(summary["cleanup"], {"complete": False, "verified": False}) + self.assertLessEqual(len(summary["diagnostics"]["files"]), 64) + self.assertLessEqual(summary["diagnostics"]["captured_bytes"], 8388608) + def test_adaptive_workers_are_required_and_bound_to_real_component_evidence(self): evidence = fixture() CHECK["validate_transfer"](evidence) @@ -365,6 +474,7 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): "site_publication": "site-evidence", "filter_snapshot": "filter-evidence", "adaptive_workers": "adaptive-evidence", + "late_provider_recovery": "late-recovery-evidence", "requester_cancellation": "cancellation", } files = {f"content-provider-{suffix}.json": evidence[key] for key, suffix in names.items()} @@ -378,6 +488,12 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): if name in files: self.assertEqual(files[name], value, "component fixture cannot replace earlier proof inputs") files[name] = value + late_raw = runpy.run_path(str(HERE / "test-content-provider-late-recovery.py"))["raw_files"]( + evidence["late_provider_recovery"]) + for name, value in late_raw.items(): + if name in files: + self.assertEqual(files[name], value, "late recovery cannot replace earlier proof inputs") + files[name] = value named = evidence["named_publication"] for key, suffix in (("fetch", "fetch"), ("output", "output"), ("cleanup", "cleanup"), ("selected_route", "selection"), ("control_privacy", "control")): @@ -425,6 +541,16 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): with self.assertRaises(ValueError): CHECK["build_evidence"](work) adaptive_report.write_text(json.dumps(evidence["adaptive_workers"]), encoding="ascii") + late_report = work / "content-provider-late-recovery-evidence.json" + late_report.unlink() + with self.assertRaises(OSError): + CHECK["build_evidence"](work) + substituted = copy.deepcopy(evidence["late_provider_recovery"]) + substituted["output"]["fetch_completed_ns"] += 1 + late_report.write_text(json.dumps(substituted), encoding="ascii") + with self.assertRaises(ValueError): + CHECK["build_evidence"](work) + late_report.write_text(json.dumps(evidence["late_provider_recovery"]), encoding="ascii") # Keep every existing exact source/destination/gateway/device check after decoding. wrong = copy.deepcopy(files["content-provider-control-relay4-out.json"]) wrong[0]["dev"] = "underlay" @@ -457,10 +583,10 @@ def test_current_control_relay_is_excluded_without_changing_roles(self): self.assertEqual(fixture("relay4")["layout"]["provider_nodes"], ["relay5", "relay3"]) def test_exact_scoped_report(self): - report = dict(report_kind="volparossa-native-content-providers", source_revision="a" * 40, + report = dict(schema_version=2, report_kind="volparossa-native-content-providers", source_revision="a" * 40, explicit_origin_authenticated_https=True, normal_user_publication=True, native_name_retrieval=True, native_static_site=True, public_filter_snapshot=True, - adaptive_provider_workers=True, requester_cancellation=True, + adaptive_provider_workers=True, late_provider_recovery=True, requester_cancellation=True, success=True, runner_exit_status=0, cleanup=dict(complete=True, remaining_owned_objects=0), host_state=dict(unchanged=True, before_sha256="b" * 64, after_sha256="b" * 64), transfer=fixture(), **{name: False for name in CHECK["SCOPE"]}) @@ -472,6 +598,12 @@ def test_exact_scoped_report(self): lambda item: item.update(native_static_site=False), lambda item: item.update(public_filter_snapshot=False), lambda item: item.update(adaptive_provider_workers=False), + lambda item: item.update(late_provider_recovery=False), + lambda item: item.pop("late_provider_recovery"), + lambda item: item["transfer"].pop("late_provider_recovery"), + lambda item: item.update(schema_version=1), + lambda item: item.update(schema_version=2.0), + lambda item: item.pop("schema_version"), lambda item: item.update(requester_cancellation=False), lambda item: (item.pop("requester_cancellation"), item["transfer"].pop("requester_cancellation")), lambda item: item["cleanup"].update(remaining_owned_objects=1), @@ -481,6 +613,22 @@ def test_exact_scoped_report(self): mutate(wrong) with self.assertRaises(ValueError): CHECK["validate_report"](wrong, "a" * 40) + historical = copy.deepcopy(report) + historical["schema_version"] = 1 + historical.pop("late_provider_recovery") + historical["transfer"].pop("late_provider_recovery") + CHECK["validate_report"](historical, "a" * 40, historical=True) + with self.assertRaises(ValueError): + CHECK["validate_report"](historical, "a" * 40) + with self.assertRaises(ValueError): + CHECK["validate_report"](report, "a" * 40, historical=True) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "historical.json" + path.write_text(json.dumps(historical), encoding="ascii") + for action, expected in (("report", 1), ("historical-report", 0)): + result = subprocess.run(["python3", "-B", str(HERE / "content-provider-smoke.py"), + action, str(path), "a" * 40], capture_output=True, timeout=5, check=False) + self.assertEqual(result.returncode, expected, result.stderr) def test_substituted_discovery_providers_or_incomplete_physical_proof_fail(self): for mutate in ( From 958a9b6bbc3722aab0a794d40e3a2be5d9666aaf Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:22:22 +0200 Subject: [PATCH 24/49] docs: record actual Overmorrow build and partial browser proof --- docs/IMPLEMENTATION_STATUS.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index ac3b632bd..fa2858c66 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -31,13 +31,13 @@ The immediate milestones build on the retained candidates and evidence below: | --- | --- | --- | | Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The separate late-provider fixture is implemented and passes its focused tests; its real discovery timing is unproven. The latest local run on `cc5f3558` completes only the base two-provider fetch with measured overlap and one discovery query. The separate CI primary-Exit discovery failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather` | Build and exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | Reuse the provider contracts and core retrieval path; client adaptation, a reproducible client build and its own runtime proof remain required. Do not create a second network scheduler or source-acquisition implementation. | +| Extend the same provider to Overmorrow — `volparossa-weather` | Exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | The pinned Flutter web release builds and fifteen synthetic component tests pass. Actual browser interaction, core delivery and native Android/iOS packaging remain unverified. Reuse the existing provider and core retrieval path; no second network scheduler or source-acquisition implementation. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the separate late-provider recovery proof, which needs no weather downloads; the successful base fetch does not prove late discovery convergence. The Drizz.li/Weather candidate and frozen input -bundle remain intact; Overmorrow's client work is still required. Keep shared interface +bundle remain intact; Overmorrow's runtime and delivery proof is still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. @@ -53,6 +53,22 @@ reports remain explicitly readable. These checks validate the fixture, not live late-provider discovery or improved transfer speed. The existing concurrent transfer and privacy requirements remain mandatory. +The Overmorrow candidate +[`54abb095`](https://github.com/VOLPAROSSA/volparossa-weather/commit/54abb0951ee0936da1f9faaa4c51cce02f29e33e) +connects the actual pinned Flutter app's provider dispatcher, phone/tablet layouts, +forecast surface and air-quality detail route to the same local HTTP bridge. +Its release build and fifteen synthetic Flutter tests pass, as do thirty-one +HTTP checks and twenty-five build/deployment checks. Native weather and particulate +times, missing versus zero values and independent expiry are preserved. The +self-hosted CanvasKit deployment contains 61 files and 22,953,717 bytes within +the unchanged asset limits. A separate isolated Firefox trial reaches actual +CanvasKit rendering through the local bridge with synthetic products and verifies +native timelines, visible values, missing observations/AQI, area and attribution. +It then times out during refresh; the cause is not yet resolved. Browser and +fixture processes exit, the private profile is removed and the host network +namespace is unchanged. This partial trial does not prove the remaining browser +interactions, peer delivery, native mobile execution or forecast accuracy. + After each milestone, or when a substantive blocker changes, reassess the next shared dependency against the existing evidence and application worktrees. Protected compute for Code/Browser, private storage for Chat/Image/Cloud, From 9b602bb5c1558cb510f0746ca31b094083e7347c Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:31:43 +0200 Subject: [PATCH 25/49] fix: bound standalone provider VM scratch and memory --- tests/integration/run-alpha-topology-vm.sh | 15 ++--- .../integration/test-browser-filter-runner.py | 57 +++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index db313d482..ae2bafa85 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -734,18 +734,19 @@ elif [ "$scenario" = agent-cooperative-code ] || [ "$scenario" = agent-cooperati [ "$(readlink -f -- "$output_directory")" = "$output_directory" ] || exit 64 RUN_DIRECTORY_PARENT=$(dirname -- "$output_directory") RUN_DIRECTORY_PREFIX=cooperative-code-kvm -elif [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle${weather_client_bundle:-}" ]; then - # The real Browser join builds Rust in this 4 GiB guest. A growing qcow2 +elif [ "$scenario" = content-provider ]; then + # Every provider proof builds Rust in this 4 GiB guest. A growing qcow2 # overlay on tmpfs otherwise competes directly with the guest's RAM. # Keep scratch beside validated workspace evidence, never in its export. [ "$(readlink -f -- "$output_directory")" = "$output_directory" ] || exit 64 RUN_DIRECTORY_PARENT=$(dirname -- "$output_directory") - RUN_DIRECTORY_PREFIX=browser-filter-kvm + RUN_DIRECTORY_PREFIX=content-provider-kvm + [ -z "$browser_filter_bundle" ] || RUN_DIRECTORY_PREFIX=browser-filter-kvm [ -z "${weather_client_bundle:-}" ] || RUN_DIRECTORY_PREFIX=weather-client-kvm runner_scratch_filesystem=$(stat -f -c '%T' -- "$RUN_DIRECTORY_PARENT") || exit 69 case $runner_scratch_filesystem in ''|tmpfs|ramfs) - printf '%s\n' 'Joined client VM scratch requires non-memory-backed workspace storage' >&2 + printf '%s\n' 'Content-provider VM scratch requires non-memory-backed workspace storage' >&2 exit 69 ;; esac fi @@ -2049,14 +2050,14 @@ set -- -no-reboot case $scenario in wifi-mesh|wifi-link) set -- ;; esac guest_memory_mib=$(guest_memory_for_scenario) runner_reserve_mib=1024 -if [ "$scenario" = content-provider ] && [ -n "$browser_filter_bundle${weather_client_bundle:-}" ]; then +if [ "$scenario" = content-provider ]; then runner_reserve_mib=2048 fi if [ "$guest_memory_mib" -gt 4096 ] || [ "$runner_reserve_mib" -eq 2048 ]; then # A read-only preflight snapshot, not a host memory reservation. Refuse # before QEMU unless the guest plus its selected host reserve fits. The - # explicit Browser join requires 2 GiB reserve; larger model fixtures keep - # their existing 1 GiB reserve. No guest or model resource limit increases. + # standalone and joined provider proofs require 2 GiB reserve; larger model + # fixtures keep their existing 1 GiB reserve. No guest/model limit increases. runner_available_kib=$(awk '$1 == "MemAvailable:" && $3 == "kB" {print $2; found++} END {if (found != 1) exit 1}' /proc/meminfo) case $runner_available_kib in ''|*[!0-9]*) exit 69 ;; esac if [ "$runner_available_kib" -lt "$(((guest_memory_mib + runner_reserve_mib) * 1024))" ]; then diff --git a/tests/integration/test-browser-filter-runner.py b/tests/integration/test-browser-filter-runner.py index 435b872af..bf5316592 100644 --- a/tests/integration/test-browser-filter-runner.py +++ b/tests/integration/test-browser-filter-runner.py @@ -55,44 +55,50 @@ def directory_fragment(self): source = (HERE / "run-alpha-topology-vm.sh").read_text() return source.split("RUN_DIRECTORY_PARENT=/tmp\n", 1)[1].split("QEMU_PID=\n", 1)[0] - def test_joined_browser_scratch_is_created_next_to_workspace_output_not_inside_it(self): + def test_standalone_and_joined_provider_scratch_is_private_disk_sibling(self): # Run only the actual path-selection/mktemp/chmod fragment, not the # VM driver. This temporary directory itself is on the workspace disk. with tempfile.TemporaryDirectory(prefix="filter-runner-test-", dir=HERE) as temporary: parent = Path(temporary).resolve(); output = parent / "evidence"; output.mkdir() - script = ('set -eu\nscenario=content-provider\nbrowser_filter_bundle=/selected/bundle\n' + script = ('set -eu\nscenario=content-provider\nbrowser_filter_bundle=$2\n' 'output_directory=$1\nRUN_DIRECTORY_PARENT=/tmp\n' + self.directory_fragment() + 'printf "%s\\n" "$RUN_DIRECTORY"\n') - result = subprocess.run(["/bin/sh", "-c", script, "fixture", str(output)], capture_output=True, timeout=5) - self.assertEqual(result.returncode, 0, result.stderr) - scratch = Path(result.stdout.decode().strip()) - self.assertEqual(scratch.parent, parent) - self.assertTrue(scratch.name.startswith("browser-filter-kvm.")) - self.assertNotEqual(scratch, output) - self.assertEqual(stat.S_IMODE(scratch.stat().st_mode), 0o700) - self.assertEqual(list(output.iterdir()), []) - - def test_joined_browser_rejects_symlink_output_and_memory_backed_parent(self): + for bundle, prefix in (("", "content-provider-kvm."), ("/selected/bundle", "browser-filter-kvm.")): + with self.subTest(bundle=bundle): + result = subprocess.run(["/bin/sh", "-c", script, "fixture", str(output), bundle], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + scratch = Path(result.stdout.decode().strip()) + self.assertEqual(scratch.parent, parent) + self.assertTrue(scratch.name.startswith(prefix)) + self.assertNotEqual(scratch, output) + self.assertEqual(stat.S_IMODE(scratch.stat().st_mode), 0o700) + self.assertEqual(list(output.iterdir()), []) + + def test_standalone_and_joined_provider_reject_symlink_output_and_memory_backed_parent(self): with tempfile.TemporaryDirectory(prefix="filter-runner-test-", dir=HERE) as temporary: parent = Path(temporary).resolve(); output = parent / "evidence"; output.mkdir() alias = parent / "alias"; alias.symlink_to(output) - prefix = 'set -eu\nscenario=content-provider\nbrowser_filter_bundle=/selected/bundle\noutput_directory=$1\nRUN_DIRECTORY_PARENT=/tmp\n' - for path, filesystem, expected in ((alias, None, 64), (output, "tmpfs", 69), (output, "ramfs", 69), (output, "failed", 69)): - stub = '' if filesystem is None else ('stat() { return 1; }\n' if filesystem == "failed" - else 'stat() { printf "%s\\n" "' + filesystem + '"; }\n') - result = subprocess.run(["/bin/sh", "-c", prefix + stub + self.directory_fragment(), "fixture", str(path)], - capture_output=True, timeout=5) - self.assertEqual(result.returncode, expected, result.stderr) - self.assertEqual({entry.name for entry in parent.iterdir()}, {"evidence", "alias"}) - - def test_ordinary_provider_and_existing_scratch_selection_unchanged(self): + prefix = 'set -eu\nscenario=content-provider\nbrowser_filter_bundle=$2\noutput_directory=$1\nRUN_DIRECTORY_PARENT=/tmp\n' + for bundle in ("", "/selected/bundle"): + for path, filesystem, expected in ((alias, None, 64), (output, "tmpfs", 69), + (output, "ramfs", 69), (output, "failed", 69)): + with self.subTest(bundle=bundle, path=path, filesystem=filesystem): + stub = '' if filesystem is None else ('stat() { return 1; }\n' if filesystem == "failed" + else 'stat() { printf "%s\\n" "' + filesystem + '"; }\n') + result = subprocess.run(["/bin/sh", "-c", prefix + stub + self.directory_fragment(), + "fixture", str(path), bundle], capture_output=True, timeout=5) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertEqual({entry.name for entry in parent.iterdir()}, {"evidence", "alias"}) + + def test_standalone_provider_selects_disk_and_other_scenario_paths_are_unchanged(self): # Stub mktemp/chmod so an ordinary /tmp-path check creates no files. script = ('set -eu\nscenario=$1\nbrowser_filter_bundle=$2\noutput_directory=$3\nRUN_DIRECTORY_PARENT=/tmp\n' 'mktemp() { printf "%s\\n" "${2%XXXXXX}abcdef"; }\nchmod() { :; }\n' + self.directory_fragment() + 'printf "%s\\n" "$RUN_DIRECTORY"\n') with tempfile.TemporaryDirectory(prefix="filter-runner-test-", dir=HERE) as temporary: output = Path(temporary).resolve() / "evidence"; output.mkdir() - for scenario, bundle, prefix in (("content-provider", "", "/tmp/volparossa-alpha-kvm."), + for scenario, bundle, prefix in (("content-provider", "", str(output.parent / "content-provider-kvm.")), ("alpha", "", "/tmp/volparossa-alpha-kvm."), ("agent-native-editor", "", str(output.parent / "native-editor-kvm.")), ("agent-cooperative-code", "", str(output.parent / "cooperative-code-kvm."))): @@ -111,7 +117,10 @@ def test_memory_preflight_requires_original_four_gib_guest_plus_two_gib_reserve( cases = (("content-provider", "bundle", 6144 * 1024 - 1, 69, None), ("content-provider", "bundle", 6144 * 1024, 0, "4096:2048"), ("content-provider", "bundle", "unknown", 69, None), - ("content-provider", "", 0, 0, "4096:1024"), + ("content-provider", "", 0, 69, None), + ("content-provider", "", 6144 * 1024 - 1, 69, None), + ("content-provider", "", 6144 * 1024, 0, "4096:2048"), + ("content-provider", "", "unknown", 69, None), ("alpha", "", 0, 0, "4096:1024"), ("agent-native-editor", "", 9216 * 1024 - 1, 69, None), ("agent-native-editor", "", 9216 * 1024, 0, "8192:1024")) From 64ced81ef8e4931773e327af77b859c9da40e1b5 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:06:42 +0200 Subject: [PATCH 26/49] docs: record Weather milestone and sequential provider trial --- docs/IMPLEMENTATION_STATUS.md | 83 +++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index fa2858c66..2b9ef822e 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -12,7 +12,7 @@ detailed evidence ledger, including historical failures—not an installation gu [Original v1 scorecard](#fixed-alpha-v1-scorecard) · [Completion requirements](#definition-of-done) -Last updated: 2026-10-11 +Last updated: 2026-10-12 ## Current integration and active work @@ -29,15 +29,16 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Verify late complementary-provider recovery and the earlier adaptive-provider correction in the real topology. | The separate late-provider fixture is implemented and passes its focused tests; its real discovery timing is unproven. The latest local run on `cc5f3558` completes only the base two-provider fetch with measured overlap and one discovery query. The separate CI primary-Exit discovery failure remains unresolved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Observe real worker-budget admission, restore proven concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `9b602bb5` reconstructs native and HTTPS adaptive objects sequentially and stops before late recovery/cancellation. Resource pressure intentionally caps the runtime at one worker, but this run retains no pressure snapshot. Capture that decision before changing policy or fixture resources; do not weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather` | Exercise the actual client against the shared provider, including unavailable/stale fields and no implicit Open-Meteo fallback. | The pinned Flutter web release builds and fifteen synthetic component tests pass. Actual browser interaction, core delivery and native Android/iOS packaging remain unverified. Reuse the existing provider and core retrieval path; no second network scheduler or source-acquisition implementation. | +| Extend the same provider to Overmorrow — `volparossa-weather` | Deliver core-cache products into the actual client, then verify native packaging. | The pinned web build, sixteen Flutter tests and actual isolated browser interaction with synthetic products pass. Core delivery and native Android/iOS execution remain unverified. Cover the client's shorter timeout and the bridge's cancellation cleanup when testing cold retrieval; reuse the shared provider without another scheduler. | Respect the observed source rate limit instead of repeatedly relaunching the -joined trial or routing requests through other peers. Continue the separate -late-provider recovery proof, which needs no weather downloads; the successful -base fetch does not prove late discovery convergence. The Drizz.li/Weather candidate and frozen input -bundle remain intact; Overmorrow's runtime and delivery proof is still required. Keep shared interface +joined trial or routing requests through other peers. Continue the standalone +provider proof, which needs no weather downloads, after investigating the failed +concurrency requirement; successful reconstruction does not prove late discovery +convergence. The Drizz.li/Weather candidate and frozen input +bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. @@ -53,21 +54,65 @@ reports remain explicitly readable. These checks validate the fixture, not live late-provider discovery or improved transfer speed. The existing concurrent transfer and privacy requirements remain mandatory. -The Overmorrow candidate -[`54abb095`](https://github.com/VOLPAROSSA/volparossa-weather/commit/54abb0951ee0936da1f9faaa4c51cce02f29e33e) +The standalone run on `9b602bb5c1558cb510f0746ca31b094083e7347c` +builds successfully and completes both adaptive native and cooperative-HTTPS +fetches: each reconstructs 15,728,640 bytes in sixty chunks from three providers, +with no origin body bytes or range requests. The evidence validator then rejects +the native result because its three 5%-to-75% bulk-payload windows do not overlap +(`-6,849,213,952 ns`). The separate HTTPS windows also fail that requirement +(`-6,581,197,091 ns`). Both captures report zero timing errors and packet-socket +drops. This is successful reconstruction, not proven concurrent bulk transfer. +The earlier two-provider native phase reconstructs 2,097,275 bytes in nine chunks; +independent existing route/control checks pass and its bulk windows overlap by +343,764,366 ns. That base result does not satisfy the three-provider requirement. +Helper logs show that the three adaptive flow openings occur sequentially even +though one discovery already returned all three providers. Later socket acquires +take only milliseconds. The runtime uses concurrent drivers but deliberately +limits admission to one worker under resource pressure or low memory/FD headroom. +The retained evidence does not identify which budget condition applied here; +record that decision in the next targeted proof before choosing a correction. +The run stops at `content-provider-adaptive-cleanup` with +`CONTENT_PROVIDER_ADAPTIVE_EVIDENCE_INVALID`; late recovery and requester +cancellation are not reached. The failed report is retained at +`.git/ci-evidence/content-provider-late-01.EmI8OY/output/content-provider-smoke.json`, +SHA-256 `9fd02652bd74b9733bc9193c753edae9ee360b49d452a311655a43bfd7dc7bb9`. +Cleanup leaves zero owned objects and unchanged guest-root network state. +The VM and owned temporary overlay are removed; development-host DNS and +IPv4/IPv6 route hashes match their pre-run values. A development-host firewall +comparison remains unavailable. No weather origin was queried by this run. + +Weather [PR7](https://github.com/VOLPAROSSA/volparossa-weather/pull/7), merged into +`main` at `44b721fd42c86880f5a8838ae18efd2c8b2dd0a1`, includes the Overmorrow +browser milestone connects the actual pinned Flutter app's provider dispatcher, phone/tablet layouts, forecast surface and air-quality detail route to the same local HTTP bridge. -Its release build and fifteen synthetic Flutter tests pass, as do thirty-one -HTTP checks and twenty-five build/deployment checks. Native weather and particulate +Its release build and sixteen synthetic Flutter tests pass, as do thirty-one +HTTP checks and thirty-seven build/deployment/browser-harness checks. Native weather and particulate times, missing versus zero values and independent expiry are preserved. The -self-hosted CanvasKit deployment contains 61 files and 22,953,717 bytes within -the unchanged asset limits. A separate isolated Firefox trial reaches actual -CanvasKit rendering through the local bridge with synthetic products and verifies -native timelines, visible values, missing observations/AQI, area and attribution. -It then times out during refresh; the cause is not yet resolved. Browser and -fixture processes exit, the private profile is removed and the host network -namespace is unchanged. This partial trial does not prove the remaining browser -interactions, peer delivery, native mobile execution or forecast accuracy. +self-hosted CanvasKit deployment contains 61 files and 22,953,908 bytes within +the unchanged asset limits. The isolated Firefox trial now passes against the +actual CanvasKit UI and local HTTP bridge with synthetic products: nine pointer +refreshes each issue exactly one weather and one air-quality request. Independent +failures/stale responses preserve the other valid product, idle expiry removes +old values without another request, and the existing air-quality detail route +opens without fetching again. The refresh control remains disabled while loading. +Native timelines, missing observations/AQI, area and attribution are verified. +Browser and fixture processes exit, the private profile is removed and the host +network namespace is unchanged. The report at +`volparossa-weather/.git/codex-weather-overmorrow/build/overmorrow-browser-06/report.json` +has SHA-256 `d718fc9d8f6364fae6134a944f01cca3af8932ebe73e33224fdffcb3a40e5996`. +This proves browser interaction, not peer delivery, native mobile execution or +forecast accuracy. A shorter client timeout can still race the bridge's cleanup +and make the next product temporarily busy; real cold delivery must cover that +case. The existing frozen Drizz.li bundle remains valid, but new joined inputs +must bind fresh proof to their actual bridge and preparer sources. +Before merge, three CodeQL file-race findings were corrected: NOAA now inspects +and reads one no-follow descriptor and executes its pinned bytes from private +scratch; the preparation test also inspects and reads the same opened file. +All 139 focused JavaScript tests and the PR's component/security checks pass. +The relocated real ecCodes binary starts under the existing limits; that check +does not replace a new origin/field-decode trial. Historical source receipts +retain their original revisions and the frozen bundle is not rewritten. After each milestone, or when a substantive blocker changes, reassess the next shared dependency against the existing evidence and application worktrees. From 0f01713b914592ba0ecc41090781b01f31dd2f16 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:07:26 +0200 Subject: [PATCH 27/49] docs: clarify merged Weather milestone wording --- docs/IMPLEMENTATION_STATUS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 2b9ef822e..850039a70 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -83,8 +83,7 @@ comparison remains unavailable. No weather origin was queried by this run. Weather [PR7](https://github.com/VOLPAROSSA/volparossa-weather/pull/7), merged into `main` at `44b721fd42c86880f5a8838ae18efd2c8b2dd0a1`, includes the Overmorrow -browser milestone -connects the actual pinned Flutter app's provider dispatcher, phone/tablet layouts, +browser milestone. It connects the actual pinned Flutter app's provider dispatcher, phone/tablet layouts, forecast surface and air-quality detail route to the same local HTTP bridge. Its release build and sixteen synthetic Flutter tests pass, as do thirty-one HTTP checks and thirty-seven build/deployment/browser-harness checks. Native weather and particulate @@ -106,6 +105,7 @@ forecast accuracy. A shorter client timeout can still race the bridge's cleanup and make the next product temporarily busy; real cold delivery must cover that case. The existing frozen Drizz.li bundle remains valid, but new joined inputs must bind fresh proof to their actual bridge and preparer sources. + Before merge, three CodeQL file-race findings were corrected: NOAA now inspects and reads one no-follow descriptor and executes its pinned bytes from private scratch; the preparation test also inspects and reads the same opened file. From 7ad108c0211183abba0e27a7922849a0455c1d9c Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:30:55 +0200 Subject: [PATCH 28/49] feat: retain bounded provider worker budget diagnostics --- .../volparossa-agent/src/content/parallel.rs | 111 ++--- .../src/content/parallel/adaptive_tests.rs | 97 ++++- .../src/content/worker_budget.rs | 407 +++++++++++++++++- .../src/content/worker_budget/diagnostics.rs | 242 +++++++++++ .../volparossa-agent/src/resource_headroom.rs | 181 +++++++- docs/IMPLEMENTATION_STATUS.md | 17 +- .../content-provider-adaptive-smoke.sh | 32 +- .../content-provider-worker-budget.py | 228 ++++++++++ tests/integration/kvm-alpha-topology.sh | 9 +- tests/integration/run-alpha-topology-vm.sh | 1 + .../test-content-provider-worker-budget.py | 223 ++++++++++ 11 files changed, 1455 insertions(+), 93 deletions(-) create mode 100644 crates/volparossa-agent/src/content/worker_budget/diagnostics.rs create mode 100644 tests/integration/content-provider-worker-budget.py create mode 100644 tests/integration/test-content-provider-worker-budget.py diff --git a/crates/volparossa-agent/src/content/parallel.rs b/crates/volparossa-agent/src/content/parallel.rs index 33d86ff2d..44ecf0ae9 100644 --- a/crates/volparossa-agent/src/content/parallel.rs +++ b/crates/volparossa-agent/src/content/parallel.rs @@ -186,58 +186,67 @@ async fn pull_inner( } limits.session_timeout = limits.session_timeout.min(remaining); limits.exchange_timeout = limits.exchange_timeout.min(remaining); - let budget = &context.content.worker_budget; - let allowance = budget.available_workers(); - if allowance == 0 { - return Ok(IndexedReceipt::default()); - } - let (download, workers) = prepare_download(manifest, store, &providers, limits, allowance)?; - let scope = RecentProviderScope::for_route(context, policy).await; - let observed_at = Instant::now(); - let observed_wall = now(); - let active = AtomicUsize::new(0); - let drivers = workers - .into_iter() - .zip(&providers) - .map(|(worker, source)| { - drivers::assigned( - worker, - deadline, - &active, - || budget.try_acquire(), - move |worker| peer(context, policy, &source.provider, worker), - ) - }) - .collect(); - let (progress, measurements, timed_out) = Box::pin(receive_workers( - download, - store, - drivers, - || budget.allowance(active.load(Ordering::Acquire)), - deadline, - )) - .await?; - let elapsed = observed_at.elapsed(); - if timed_out { - content_event(context, "CONTENT_PROVIDER_TRANSFER_FAILED").await; - content_event(context, "CONTENT_PROVIDER_TRANSFER_DEADLINE").await; + // One bounded, opt-in summary of the actual admission decisions, including a zero + // allowance or interrupted batch. It contains no publication or provider identity. + // The inner future owns all workers, so leases drain before the summary is dropped. + let budget = context.content.worker_budget.for_batch(); + let result = async { + let allowance = budget.available_workers(); + if allowance == 0 { + return Ok(IndexedReceipt::default()); + } + let (download, workers) = prepare_download(manifest, store, &providers, limits, allowance)?; + let scope = RecentProviderScope::for_route(context, policy).await; + let observed_at = Instant::now(); + let observed_wall = now(); + let active = AtomicUsize::new(0); + let drivers = workers + .into_iter() + .zip(&providers) + .map(|(worker, source)| { + drivers::assigned( + worker, + deadline, + &active, + || budget.try_acquire(), + move |worker| peer(context, policy, &source.provider, worker), + ) + }) + .collect(); + let (progress, measurements, timed_out) = Box::pin(receive_workers( + download, + store, + drivers, + || budget.allowance(active.load(Ordering::Acquire)), + deadline, + )) + .await?; + let elapsed = observed_at.elapsed(); + if timed_out { + content_event(context, "CONTENT_PROVIDER_TRANSFER_FAILED").await; + content_event(context, "CONTENT_PROVIDER_TRANSFER_DEADLINE").await; + } + let same_route = RecentProviderScope::for_route(context, policy).await == scope; + record_batch( + context, + manifest, + providers, + progress, + measurements, + BatchMeasurement { + scope, + observed_at, + observed_wall, + elapsed, + valid: !timed_out && same_route, + }, + ) + .await } - let same_route = RecentProviderScope::for_route(context, policy).await == scope; - record_batch( - context, - manifest, - providers, - progress, - measurements, - BatchMeasurement { - scope, - observed_at, - observed_wall, - elapsed, - valid: !timed_out && same_route, - }, - ) - .await + .await; + // Ok can contain partial progress: diagnostics call this "returned", not "complete". + budget.finish(result.is_ok()); + result } fn prepare_download( diff --git a/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs b/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs index d1c45b140..621e27e89 100644 --- a/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs +++ b/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs @@ -86,12 +86,19 @@ fn fixture() -> Fixture { #[tokio::test] async fn adaptive_workers_open_three_real_provider_streams_with_resource_leases_and_exact_hash() { - tokio::time::timeout(Duration::from_secs(15), actual_streams()) + tokio::time::timeout(Duration::from_secs(15), actual_streams(false)) .await .expect("all driver and writer owners finish within one deadline"); } -async fn actual_streams() { +#[tokio::test] +async fn adaptive_workers_keep_one_stream_under_pressure_with_measured_admission() { + tokio::time::timeout(Duration::from_secs(15), actual_streams(true)) + .await + .expect("pressure preserves bounded foreground progress"); +} + +async fn actual_streams(pressured: bool) { let Fixture { _directory, manifest, @@ -100,7 +107,9 @@ async fn actual_streams() { bytes, } = fixture(); // Three stream allowances come from explicit memory/FD headroom, not a product count cap. - let budget = WorkerBudget::for_test(3 * 32 * 8 * 1024 * 1024, 3 * 4 * 8, false); + let shared_budget = WorkerBudget::for_test(3 * 32 * 8 * 1024 * 1024, 3 * 4 * 8, pressured); + let budget = shared_budget.for_test_batch(); + let expected_peak = if pressured { 1 } else { 3 }; let active = AtomicUsize::new(0); let peak = AtomicUsize::new(0); let opened = AtomicUsize::new(0); @@ -157,9 +166,9 @@ async fn actual_streams() { .await .unwrap(); assert!(!timed_out); - assert_eq!(peak.load(Ordering::SeqCst), 3); - assert!(opened.load(Ordering::SeqCst) >= 3); - assert!(progress.iter().filter(|item| item.bytes > 0).count() >= 3); + assert_eq!(peak.load(Ordering::SeqCst), expected_peak); + assert!(opened.load(Ordering::SeqCst) >= expected_peak); + assert!(progress.iter().filter(|item| item.bytes > 0).count() >= expected_peak); assert_eq!( progress.iter().map(|item| item.bytes).sum::(), bytes.len() as u64 @@ -170,10 +179,20 @@ async fn actual_streams() { .iter() .filter(|item| item.elapsed.is_some()) .count() - >= 3 + >= expected_peak ); assert_eq!(active.load(Ordering::SeqCst), 0); - assert_eq!(budget.available_workers(), 3); + budget.finish(true); + let diagnosis = budget.snapshot_for_test().unwrap(); + assert_eq!(diagnosis["initial_allowance"], expected_peak); + assert_eq!(diagnosis["min_allowance"], expected_peak); + assert_eq!(diagnosis["max_allowance"], expected_peak); + assert_eq!(diagnosis["peak_leases"], expected_peak); + assert_eq!(diagnosis["active_leases"], 0); + assert_eq!(diagnosis["admitted_leases"], opened.load(Ordering::SeqCst)); + assert_eq!(diagnosis["denied_leases"], 0); + assert_eq!(diagnosis["outcome"], "returned"); + assert_eq!(shared_budget.available_workers(), expected_peak); let mut output = Vec::new(); reassemble(&manifest, &mut [&mut cache], now(), &mut output).unwrap(); assert_eq!(output, bytes); @@ -222,3 +241,65 @@ async fn adaptive_worker_lease_race_and_dormant_completion_never_open_a_stream() ); assert_eq!(active.load(Ordering::SeqCst), 0); } + +#[tokio::test] +async fn adaptive_worker_interruption_releases_observed_leases_before_batch_summary() { + let Fixture { + _directory, + manifest, + mut cache, + .. + } = fixture(); + let shared_budget = WorkerBudget::for_test(3 * 32 * 8 * 1024 * 1024, 3 * 4 * 8, false); + let budget = shared_budget.for_test_batch(); + let active = AtomicUsize::new(0); + let started = tokio::sync::Notify::new(); + let deadline = Instant::now() + Duration::from_secs(5); + let (download, workers) = ParallelDownload::new_adaptive( + &manifest, + 3, + &mut cache, + TransferLimits::default(), + budget.available_workers(), + ) + .unwrap(); + let tasks = workers + .into_iter() + .map(|worker| { + drivers::assigned( + worker, + deadline, + &active, + || budget.try_acquire(), + |worker| async { + let _worker = worker; + started.notify_one(); + std::future::pending::>().await + }, + ) + }) + .collect(); + let mut transfer = Box::pin(receive_workers( + download, + &mut cache, + tasks, + || budget.allowance(active.load(Ordering::Acquire)), + deadline, + )); + tokio::select! { + biased; + _ = &mut transfer => panic!("workers should still be pending"), + () = started.notified() => {} + } + assert!(active.load(Ordering::Acquire) > 0); + assert!(shared_budget.available_workers() < 3); + // Dropping the caller's future cancels all owned drivers without a detached task. + drop(transfer); + assert_eq!(active.load(Ordering::Acquire), 0); + assert_eq!(shared_budget.available_workers(), 3); + let diagnosis = budget.snapshot_for_test().unwrap(); + assert_eq!(diagnosis["outcome"], "interrupted"); + assert_eq!(diagnosis["active_leases"], 0); + assert!(diagnosis["peak_leases"].as_u64().unwrap() > 0); + assert_eq!(diagnosis["denied_leases"], 0); +} diff --git a/crates/volparossa-agent/src/content/worker_budget.rs b/crates/volparossa-agent/src/content/worker_budget.rs index 79a25d06d..8c05e051b 100644 --- a/crates/volparossa-agent/src/content/worker_budget.rs +++ b/crates/volparossa-agent/src/content/worker_budget.rs @@ -9,6 +9,11 @@ use std::{ time::{Duration, Instant}, }; +use crate::resource_headroom::{PressureObservation, PressureState}; + +mod diagnostics; +use diagnostics::{Diagnostics, Outcome}; + const BYTES_PER_WORKER: u64 = 8 * 1024 * 1024; const FDS_PER_WORKER: u64 = 8; const MEMORY_HEADROOM_DIVISOR: u64 = 32; @@ -30,14 +35,20 @@ struct State { struct Snapshot { memory: u64, fds: u64, - pressured: bool, + pressure: PressureObservation, } impl Snapshot { + fn memory_workers(self) -> u64 { + self.memory / MEMORY_HEADROOM_DIVISOR / BYTES_PER_WORKER + } + + fn fd_workers(self) -> u64 { + self.fds / FD_HEADROOM_DIVISOR / FDS_PER_WORKER + } + fn capacity(self) -> usize { - let memory = self.memory / MEMORY_HEADROOM_DIVISOR / BYTES_PER_WORKER; - let fds = self.fds / FD_HEADROOM_DIVISOR / FDS_PER_WORKER; - usize::try_from(memory.min(fds)).unwrap_or(usize::MAX) + usize::try_from(self.memory_workers().min(self.fd_workers())).unwrap_or(usize::MAX) } } @@ -53,10 +64,10 @@ impl State { return value; } } - let value = crate::resource_headroom::capture().map(|snapshot| Snapshot { - memory: snapshot.memory_bytes, - fds: snapshot.available_fds, - pressured: snapshot.pressured, + let value = crate::resource_headroom::capture_with_pressure().map(|observed| Snapshot { + memory: observed.headroom.memory_bytes, + fds: observed.headroom.available_fds, + pressure: observed.pressure, }); self.sampled = Some((at, value)); value @@ -64,20 +75,43 @@ impl State { } impl WorkerBudget { + /// Opt-in aggregate diagnostics share the exact decisions and samples below. + /// With this debug target disabled, no aggregate allocation or log is made. + pub(super) fn for_batch(&self) -> BatchBudget<'_> { + BatchBudget::new( + self, + tracing::enabled!(target: "volparossa_agent::content::worker_budget", tracing::Level::DEBUG), + ) + } + /// Currently unreserved worker units; this does not authorize a future socket open. pub(super) fn available_workers(&self) -> usize { + self.available_workers_observed(None) + } + + fn available_workers_observed(&self, diagnostic: Option<&Diagnostics>) -> usize { let Ok(mut state) = self.0.lock() else { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, None, Some(0)); + } return 0; }; let Some(snapshot) = state.snapshot() else { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, Some(state.held), Some(0)); + } return 0; }; - let capacity = if snapshot.pressured { + let capacity = if snapshot.pressure.pressured() { snapshot.capacity().min(1) } else { snapshot.capacity() }; - capacity.saturating_sub(state.held) + let available = capacity.saturating_sub(state.held); + if let Some(diagnostic) = diagnostic { + diagnostic.observe(Some(snapshot), Some(state.held), Some(available)); + } + available } /// Absolute allowance for one download, including its existing reservations. @@ -85,35 +119,78 @@ impl WorkerBudget { /// closed. This preserves existing foreground progress; it is not a one-stream global /// eviction policy. Actual RAM/descriptor capacity remains shared across all downloads. pub(super) fn allowance(&self, local_active: usize) -> usize { + self.allowance_observed(local_active, None) + } + + fn allowance_observed(&self, local_active: usize, diagnostic: Option<&Diagnostics>) -> usize { let Ok(mut state) = self.0.lock() else { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, None, Some(0)); + } return 0; }; let Some(snapshot) = state.snapshot() else { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, Some(state.held), Some(0)); + } return 0; }; let others = state.held.saturating_sub(local_active); let available = snapshot.capacity().saturating_sub(others); - if snapshot.pressured { + let allowance = if snapshot.pressure.pressured() { available.min(1) } else { available + }; + if let Some(diagnostic) = diagnostic { + diagnostic.observe(Some(snapshot), Some(state.held), Some(allowance)); } + allowance } /// Atomically reserve shared accounting before allocating a new protected worker flow. pub(super) fn try_acquire(&self) -> Option { - let mut state = self.0.lock().ok()?; - let snapshot = state.snapshot()?; - let capacity = if snapshot.pressured { + self.try_acquire_observed(None) + } + + fn try_acquire_observed(&self, diagnostic: Option<&Diagnostics>) -> Option { + let Ok(mut state) = self.0.lock() else { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, None, None); + diagnostic.admission(false); + } + return None; + }; + let Some(snapshot) = state.snapshot() else { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, Some(state.held), None); + diagnostic.admission(false); + } + return None; + }; + let capacity = if snapshot.pressure.pressured() { snapshot.capacity().min(1) } else { snapshot.capacity() }; if state.held >= capacity { + if let Some(diagnostic) = diagnostic { + diagnostic.observe(Some(snapshot), Some(state.held), None); + diagnostic.admission(false); + } return None; } state.held = state.held.checked_add(1)?; - Some(WorkerLease(Arc::clone(&self.0))) + if let Some(diagnostic) = diagnostic { + diagnostic.observe(Some(snapshot), Some(state.held), None); + diagnostic.admission(true); + } + Some(WorkerLease(Some(Arc::clone(&self.0)))) + } + + #[cfg(test)] + pub(super) fn for_test_batch(&self) -> BatchBudget<'_> { + BatchBudget::new(self, true) } #[cfg(test)] @@ -122,24 +199,128 @@ impl WorkerBudget { fixed: Some(Snapshot { memory, fds, - pressured, + pressure: PressureObservation { + cpu: if pressured { + PressureState::AtOrAboveThreshold + } else { + PressureState::BelowThreshold + }, + memory: PressureState::BelowThreshold, + io: PressureState::BelowThreshold, + }, }), ..State::default() }))) } } -/// Not cloneable: one live worker owns one shared reservation until its flow has closed. -pub(super) struct WorkerLease(Arc>); +/// A diagnostic owner, not an additional reservation or provider-count policy. +pub(super) struct BatchBudget<'a> { + budget: &'a WorkerBudget, + diagnostic: Option, +} -impl Drop for WorkerLease { +impl<'a> BatchBudget<'a> { + fn new(budget: &'a WorkerBudget, enabled: bool) -> Self { + Self { + budget, + diagnostic: enabled.then(Diagnostics::new), + } + } + + pub(super) fn available_workers(&self) -> usize { + match &self.diagnostic { + Some(diagnostic) => self.budget.available_workers_observed(Some(diagnostic)), + None => self.budget.available_workers(), + } + } + + pub(super) fn allowance(&self, local_active: usize) -> usize { + match &self.diagnostic { + Some(diagnostic) => self + .budget + .allowance_observed(local_active, Some(diagnostic)), + None => self.budget.allowance(local_active), + } + } + + pub(super) fn try_acquire(&self) -> Option { + let lease = match &self.diagnostic { + Some(diagnostic) => self.budget.try_acquire_observed(Some(diagnostic)), + None => self.budget.try_acquire(), + }; + lease.map(|lease| DiagnosticLease { + lease: Some(lease), + diagnostic: self.diagnostic.clone(), + }) + } + + /// `returned` only means the caller returned Ok, never that transfer was complete. + pub(super) fn finish(&self, success: bool) { + if let Some(diagnostic) = &self.diagnostic { + diagnostic.set_outcome(if success { + Outcome::Returned + } else { + Outcome::Failed + }); + } + } + + #[cfg(test)] + pub(super) fn snapshot_for_test(&self) -> Option { + self.diagnostic.as_ref().map(Diagnostics::snapshot_for_test) + } +} + +impl Drop for BatchBudget<'_> { fn drop(&mut self) { - if let Ok(mut state) = self.0.lock() { + if let Some(diagnostic) = &self.diagnostic { + diagnostic.emit(); + } + } +} + +pub(super) struct DiagnosticLease { + lease: Option, + diagnostic: Option, +} + +impl Drop for DiagnosticLease { + fn drop(&mut self) { + // Release both counts under the same budget -> diagnostics lock order. + // A new acquire cannot inflate the local peak before this release is counted. + if let Some(mut lease) = self.lease.take() { + lease.release(self.diagnostic.as_ref()); + } + } +} + +/// Not cloneable: one live worker owns one shared reservation until its flow has closed. +pub(super) struct WorkerLease(Option>>); + +impl WorkerLease { + fn release(&mut self, diagnostic: Option<&Diagnostics>) { + let Some(owner) = self.0.take() else { + return; + }; + if let Ok(mut state) = owner.lock() { state.held = state.held.saturating_sub(1); + if let Some(diagnostic) = diagnostic { + diagnostic.release(); + } + } else if let Some(diagnostic) = diagnostic { + diagnostic.observe(None, None, None); + diagnostic.release(); } } } +impl Drop for WorkerLease { + fn drop(&mut self) { + self.release(None); + } +} + #[cfg(test)] mod tests { use super::*; @@ -162,7 +343,15 @@ mod tests { ); let third = budget.try_acquire().unwrap(); assert!(other.try_acquire().is_none(), "no concurrent overcommit"); - budget.0.lock().unwrap().fixed.as_mut().unwrap().pressured = true; + budget + .0 + .lock() + .unwrap() + .fixed + .as_mut() + .unwrap() + .pressure + .cpu = PressureState::AtOrAboveThreshold; assert_eq!(budget.allowance(3), 1); assert!(other.try_acquire().is_none()); drop((second, third)); @@ -193,7 +382,15 @@ mod tests { let first = budget.try_acquire().unwrap(); let surplus = budget.try_acquire().unwrap(); let other_download = budget.try_acquire().unwrap(); - budget.0.lock().unwrap().fixed.as_mut().unwrap().pressured = true; + budget + .0 + .lock() + .unwrap() + .fixed + .as_mut() + .unwrap() + .pressure + .cpu = PressureState::AtOrAboveThreshold; assert_eq!(budget.allowance(2), 1); assert_eq!(budget.allowance(1), 1); drop(surplus); @@ -216,4 +413,170 @@ mod tests { drop(first); assert_eq!(budget.available_workers(), 1); } + + #[test] + fn batch_diagnostics_use_actual_shared_decisions_without_reducing_planner_allowance_on_acquire() + { + let budget = WorkerBudget::for_test( + 3 * BYTES_PER_WORKER * MEMORY_HEADROOM_DIVISOR, + 3 * FDS_PER_WORKER * FD_HEADROOM_DIVISOR, + false, + ); + let batch = budget.for_test_batch(); + assert_eq!(budget.available_workers(), 3); + assert_eq!(batch.available_workers(), 3); + let first = batch.try_acquire().unwrap(); + let other_batch = budget.try_acquire().unwrap(); + assert_eq!(batch.allowance(1), 2); + let second = batch.try_acquire().unwrap(); + assert!(batch.try_acquire().is_none()); + assert_eq!(batch.snapshot_for_test().unwrap()["min_allowance"], 2); + drop(other_batch); + assert_eq!(batch.allowance(2), 3); + budget.0.lock().unwrap().fixed.as_mut().unwrap().pressure = PressureObservation { + cpu: PressureState::AtOrAboveThreshold, + memory: PressureState::Unavailable, + io: PressureState::AtOrAboveThreshold, + }; + assert_eq!(batch.allowance(2), 1); + drop((first, second)); + batch.finish(true); + let value = batch.snapshot_for_test().unwrap(); + for (key, expected) in [ + ("initial_allowance", 3), + ("min_allowance", 1), + ("max_allowance", 3), + ("min_memory_workers", 3), + ("min_fd_workers", 3), + ("peak_shared_held", 3), + ("admitted_leases", 2), + ("denied_leases", 1), + ("peak_leases", 2), + ("active_leases", 0), + ] { + assert_eq!(value[key], expected, "{key}"); + } + let causes = value["causes"].as_object().unwrap(); + assert!(causes.iter().all(|(key, value)| value + == &matches!( + key.as_str(), + "cpu_pressure" | "memory_unavailable" | "io_pressure" + ))); + assert_eq!(value["outcome"], "returned"); + assert_eq!(budget.available_workers(), 1); + } + + #[test] + fn batch_diagnostics_are_absent_when_disabled_and_default_to_interrupted() { + fn send_sync() {} + send_sync::>(); + send_sync::(); + let budget = WorkerBudget::for_test( + BYTES_PER_WORKER * MEMORY_HEADROOM_DIVISOR, + FDS_PER_WORKER * FD_HEADROOM_DIVISOR, + false, + ); + { + let batch = BatchBudget::new(&budget, false); + assert_eq!(batch.available_workers(), 1); + let lease = batch.try_acquire().unwrap(); + assert!(batch.try_acquire().is_none()); + drop(lease); + batch.finish(true); + assert!(batch.snapshot_for_test().is_none()); + } + let batch = budget.for_test_batch(); + let diagnostic = batch.diagnostic.clone().unwrap(); + let lease = batch.try_acquire().unwrap(); + drop(lease); + drop(batch); + let value = diagnostic.snapshot_for_test(); + assert_eq!(value["outcome"], "interrupted"); + assert_eq!(value["admitted_leases"], 1); + assert_eq!(value["active_leases"], 0); + assert!(value["initial_allowance"].is_null()); + assert!(value["min_allowance"].is_null()); + assert!(value["max_allowance"].is_null()); + assert_eq!(budget.available_workers(), 1); + let failed = budget.for_test_batch(); + failed.finish(false); + assert_eq!(failed.snapshot_for_test().unwrap()["outcome"], "failed"); + } + + #[test] + fn batch_drop_emits_exactly_one_closed_event_only_when_debug_is_enabled() { + #[derive(Clone)] + struct Capture(Arc>>); + + impl std::io::Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + for level in [tracing::Level::INFO, tracing::Level::DEBUG] { + let bytes = Arc::new(Mutex::new(Vec::new())); + let writer = Capture(Arc::clone(&bytes)); + let subscriber = tracing_subscriber::fmt() + .json() + .without_time() + .with_max_level(level) + .with_writer(move || writer.clone()) + .finish(); + tracing::subscriber::with_default(subscriber, || { + let budget = WorkerBudget::for_test( + BYTES_PER_WORKER * MEMORY_HEADROOM_DIVISOR, + FDS_PER_WORKER * FD_HEADROOM_DIVISOR, + false, + ); + for outcome in [Some(true), Some(false), None] { + let before = bytes.lock().unwrap().len(); + let batch = budget.for_batch(); + assert_eq!( + batch.snapshot_for_test().is_some(), + level == tracing::Level::DEBUG + ); + assert_eq!(batch.available_workers(), 1); + let lease = batch.try_acquire().unwrap(); + drop(lease); + if let Some(success) = outcome { + batch.finish(success); + } + assert_eq!(bytes.lock().unwrap().len(), before, "only Drop emits"); + drop(batch); + assert_eq!(budget.available_workers(), 1); + } + }); + let output = String::from_utf8(bytes.lock().unwrap().clone()).unwrap(); + if level == tracing::Level::INFO { + assert!(output.is_empty()); + continue; + } + let events: Vec = output + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(events.len(), 3); + for (event, outcome) in events.iter().zip(["returned", "failed", "interrupted"]) { + assert_eq!(event["target"], "volparossa_agent::content::worker_budget"); + assert_eq!(event["level"], "DEBUG"); + let fields = event["fields"].as_object().unwrap(); + assert_eq!(fields.len(), 2); + assert_eq!(fields["code"], "CONTENT_WORKER_BUDGET_SUMMARY"); + let summary: serde_json::Value = + serde_json::from_str(fields["budget"].as_str().unwrap()).unwrap(); + assert_eq!(summary.as_object().unwrap().len(), 13); + assert_eq!(summary["outcome"], outcome); + assert_eq!(summary["admitted_leases"], 1); + assert_eq!(summary["peak_leases"], 1); + assert_eq!(summary["active_leases"], 0); + assert_eq!(summary["causes"].as_object().unwrap().len(), 8); + } + } + } } diff --git a/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs b/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs new file mode 100644 index 000000000..0100f7cf9 --- /dev/null +++ b/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs @@ -0,0 +1,242 @@ +//! One opt-in, bounded batch summary; never raw resource data or peer/user identifiers. + +use std::sync::{Arc, Mutex}; + +use serde::Serialize; + +use super::{PressureState, Snapshot}; + +#[derive(Clone, Copy, Default, Serialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum Outcome { + Returned, + Failed, + #[default] + Interrupted, +} + +#[derive(Clone, Copy, Default, Serialize)] +struct Causes { + cpu_pressure: bool, + memory_pressure: bool, + io_pressure: bool, + cpu_unavailable: bool, + memory_unavailable: bool, + io_unavailable: bool, + resource_unavailable: bool, + lock_unavailable: bool, +} + +/// Counts saturate at u32::MAX; this never clamps the actual resource decision. +#[derive(Clone, Serialize)] +struct Summary { + schema_version: u8, + outcome: Outcome, + initial_allowance: Option, + min_allowance: Option, + max_allowance: Option, + min_memory_workers: Option, + min_fd_workers: Option, + peak_shared_held: u32, + admitted_leases: u32, + denied_leases: u32, + peak_leases: u32, + active_leases: u32, + causes: Causes, +} + +impl Default for Summary { + fn default() -> Self { + Self { + schema_version: 1, + outcome: Outcome::Interrupted, + initial_allowance: None, + min_allowance: None, + max_allowance: None, + min_memory_workers: None, + min_fd_workers: None, + peak_shared_held: 0, + admitted_leases: 0, + denied_leases: 0, + peak_leases: 0, + active_leases: 0, + causes: Causes::default(), + } + } +} + +#[derive(Clone)] +pub(super) struct Diagnostics(Arc>); + +fn bounded(value: impl TryInto) -> u32 { + value.try_into().unwrap_or(u32::MAX) +} + +fn minimum(current: &mut Option, value: u32) { + *current = Some(current.map_or(value, |prior| prior.min(value))); +} + +impl Diagnostics { + pub(super) fn new() -> Self { + Self(Arc::new(Mutex::new(Summary::default()))) + } + + pub(super) fn observe( + &self, + snapshot: Option, + held: Option, + allowance: Option, + ) { + let mut summary = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(allowance) = allowance { + let allowance = bounded(allowance); + summary.initial_allowance.get_or_insert(allowance); + minimum(&mut summary.min_allowance, allowance); + summary.max_allowance = Some( + summary + .max_allowance + .map_or(allowance, |prior| prior.max(allowance)), + ); + } + let Some(held) = held else { + summary.causes.lock_unavailable = true; + return; + }; + summary.peak_shared_held = summary.peak_shared_held.max(bounded(held)); + let Some(snapshot) = snapshot else { + summary.causes.resource_unavailable = true; + return; + }; + minimum( + &mut summary.min_memory_workers, + bounded(snapshot.memory_workers()), + ); + minimum(&mut summary.min_fd_workers, bounded(snapshot.fd_workers())); + let causes = &mut summary.causes; + causes.cpu_pressure |= snapshot.pressure.cpu == PressureState::AtOrAboveThreshold; + causes.memory_pressure |= snapshot.pressure.memory == PressureState::AtOrAboveThreshold; + causes.io_pressure |= snapshot.pressure.io == PressureState::AtOrAboveThreshold; + causes.cpu_unavailable |= snapshot.pressure.cpu == PressureState::Unavailable; + causes.memory_unavailable |= snapshot.pressure.memory == PressureState::Unavailable; + causes.io_unavailable |= snapshot.pressure.io == PressureState::Unavailable; + } + + pub(super) fn admission(&self, admitted: bool) { + let mut summary = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if admitted { + summary.admitted_leases = summary.admitted_leases.saturating_add(1); + summary.active_leases = summary.active_leases.saturating_add(1); + summary.peak_leases = summary.peak_leases.max(summary.active_leases); + } else { + summary.denied_leases = summary.denied_leases.saturating_add(1); + } + } + + pub(super) fn release(&self) { + let mut summary = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + summary.active_leases = summary.active_leases.saturating_sub(1); + } + + pub(super) fn set_outcome(&self, outcome: Outcome) { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .outcome = outcome; + } + + pub(super) fn emit(&self) { + let summary = self + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + // Serialization is closed and fixed-size; no errors or raw data are interpolated. + if let Ok(budget) = serde_json::to_string(&summary) { + tracing::debug!(target: "volparossa_agent::content::worker_budget", + code = "CONTENT_WORKER_BUDGET_SUMMARY", budget = %budget); + } + } + + #[cfg(test)] + pub(super) fn snapshot_for_test(&self) -> serde_json::Value { + serde_json::to_value( + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(), + ) + .unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::resource_headroom::PressureObservation; + + #[test] + fn missing_resource_and_lock_do_not_invent_psi_observations() { + for (held, reason) in [ + (None, "lock_unavailable"), + (Some(0), "resource_unavailable"), + ] { + let diagnostic = Diagnostics::new(); + diagnostic.observe(None, held, Some(0)); + let value = diagnostic.snapshot_for_test(); + assert!(value["min_memory_workers"].is_null()); + assert!(value["min_fd_workers"].is_null()); + assert_eq!(value["initial_allowance"], 0); + let causes = value["causes"].as_object().unwrap(); + assert_eq!(causes.len(), 8); + assert!(causes.iter().all(|(key, value)| value == &(key == reason))); + } + } + + #[test] + fn diagnostic_counts_are_finite_saturating_and_schema_is_closed() { + let diagnostic = Diagnostics::new(); + diagnostic.observe( + Some(Snapshot { + memory: u64::MAX, + fds: u64::MAX, + pressure: PressureObservation::default(), + }), + Some(usize::MAX), + Some(usize::MAX), + ); + { + let mut summary = diagnostic.0.lock().unwrap(); + summary.admitted_leases = u32::MAX; + summary.denied_leases = u32::MAX; + summary.active_leases = u32::MAX; + } + diagnostic.admission(true); + diagnostic.admission(false); + let value = diagnostic.snapshot_for_test(); + assert_eq!(value.as_object().unwrap().len(), 13); + for key in [ + "initial_allowance", + "min_allowance", + "max_allowance", + "min_memory_workers", + "min_fd_workers", + "peak_shared_held", + "admitted_leases", + "denied_leases", + "peak_leases", + "active_leases", + ] { + assert_eq!(value[key], u32::MAX); + } + assert_eq!(value["outcome"], "interrupted"); + } +} diff --git a/crates/volparossa-agent/src/resource_headroom.rs b/crates/volparossa-agent/src/resource_headroom.rs index b93aa19f5..c5d6916bf 100644 --- a/crates/volparossa-agent/src/resource_headroom.rs +++ b/crates/volparossa-agent/src/resource_headroom.rs @@ -18,7 +18,62 @@ pub(crate) struct ResourceHeadroom { pub(crate) pressured: bool, } +/// Closed observations, not raw PSI text, device/process identities or policy overrides. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum PressureState { + BelowThreshold, + AtOrAboveThreshold, + /// Missing, unreadable and malformed PSI all retain the existing pressured policy. + #[default] + Unavailable, +} + +impl PressureState { + fn from_average(value: Option, threshold: f64) -> Self { + match value { + Some(value) if value >= threshold => Self::AtOrAboveThreshold, + Some(_) => Self::BelowThreshold, + None => Self::Unavailable, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct PressureObservation { + pub(crate) cpu: PressureState, + pub(crate) memory: PressureState, + pub(crate) io: PressureState, +} + +impl PressureObservation { + fn sample(mut observe: impl FnMut(&str) -> Option) -> Self { + // Observe each bounded source once, even when an earlier one is pressured, + // so diagnostics and the admission decision describe the same sample. + Self { + cpu: PressureState::from_average(observe("cpu"), 20.0), + memory: PressureState::from_average(observe("memory"), 1.0), + io: PressureState::from_average(observe("io"), 10.0), + } + } + + pub(crate) fn pressured(self) -> bool { + [self.cpu, self.memory, self.io] + .into_iter() + .any(|state| state != PressureState::BelowThreshold) + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ObservedHeadroom { + pub(crate) headroom: ResourceHeadroom, + pub(crate) pressure: PressureObservation, +} + pub(crate) fn capture() -> Option { + capture_with_pressure().map(|observation| observation.headroom) +} + +pub(crate) fn capture_with_pressure() -> Option { let memory = available_memory(&system_text(Path::new("/proc/meminfo"))?)?; let memory = cgroup_headroom(memory)?; let limit = descriptor_limit(&system_text(Path::new("/proc/self/limits"))?)?; @@ -28,14 +83,14 @@ pub(crate) fn capture() -> Option { entry.ok()?; count.checked_add(1) })?; - // Missing telemetry selects the caller's pressured policy, never a claim of an idle host. - let pressured = pressure("cpu").is_none_or(|value| value >= 20.0) - || pressure("memory").is_none_or(|value| value >= 1.0) - || pressure("io").is_none_or(|value| value >= 10.0); - Some(ResourceHeadroom { - memory_bytes: memory, - available_fds: limit.saturating_sub(open), - pressured, + let pressure = PressureObservation::sample(pressure); + Some(ObservedHeadroom { + headroom: ResourceHeadroom { + memory_bytes: memory, + available_fds: limit.saturating_sub(open), + pressured: pressure.pressured(), + }, + pressure, }) } @@ -151,4 +206,114 @@ mod tests { assert_eq!(pressure_average("some avg10=NaN\n"), None); assert_eq!(pressure_average("full avg10=0.00\n"), None); } + + #[test] + fn pressure_observations_preserve_thresholds_and_all_causes_from_one_sample() { + use PressureState::{AtOrAboveThreshold as High, BelowThreshold as Low, Unavailable}; + + let mut sampled = Vec::new(); + let pressure = PressureObservation::sample(|kind| { + sampled.push(kind.to_owned()); + Some(match kind { + "cpu" => 20.0, + "memory" => 1.0, + "io" => 10.0, + _ => panic!("only the three fixed PSI sources may be sampled"), + }) + }); + assert_eq!(sampled, ["cpu", "memory", "io"]); + assert_eq!( + pressure, + PressureObservation { + cpu: High, + memory: High, + io: High + } + ); + assert!(pressure.pressured()); + + for cpu in [Some(0.0), Some(19.999), Some(20.0), Some(100.0), None] { + for memory in [Some(0.0), Some(0.999), Some(1.0), Some(100.0), None] { + for io in [Some(0.0), Some(9.999), Some(10.0), Some(100.0), None] { + let pressure = PressureObservation::sample(|kind| match kind { + "cpu" => cpu, + "memory" => memory, + "io" => io, + _ => unreachable!(), + }); + let expected = cpu.is_none_or(|value| value >= 20.0) + || memory.is_none_or(|value| value >= 1.0) + || io.is_none_or(|value| value >= 10.0); + assert_eq!(pressure.pressured(), expected); + for (value, threshold, state) in [ + (cpu, 20.0, pressure.cpu), + (memory, 1.0, pressure.memory), + (io, 10.0, pressure.io), + ] { + assert_eq!( + state, + match value { + None => Unavailable, + Some(value) if value >= threshold => High, + Some(_) => Low, + } + ); + } + } + } + } + } + + #[test] + fn missing_or_invalid_psi_is_unavailable_not_an_idle_sample() { + use PressureState::{AtOrAboveThreshold as High, BelowThreshold as Low, Unavailable}; + + assert_eq!( + PressureObservation::default(), + PressureObservation { + cpu: Unavailable, + memory: Unavailable, + io: Unavailable, + } + ); + assert!(PressureObservation::default().pressured()); + for invalid in [ + "", + "full avg10=0.00\n", + "some avg60=0.00\n", + "some avg10=NaN\n", + "some avg10=inf\n", + "some avg10=-0.01\n", + "some avg10=100.01\n", + "some avg10=invalid\n", + ] { + let pressure = PressureObservation::sample(|kind| { + pressure_average(match kind { + "cpu" => invalid, + "memory" => "some avg10=0.00\n", + "io" => "some avg10=10.00\n", + _ => unreachable!(), + }) + }); + assert_eq!( + pressure, + PressureObservation { + cpu: Unavailable, + memory: Low, + io: High + } + ); + assert!(pressure.pressured()); + } + let idle = PressureObservation::sample(|_| pressure_average("some avg10=0.00\n")); + assert_eq!( + idle, + PressureObservation { + cpu: Low, + memory: Low, + io: Low + } + ); + assert!(!idle.pressured()); + } } diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 850039a70..388922028 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,7 +29,7 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Observe real worker-budget admission, restore proven concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `9b602bb5` reconstructs native and HTTPS adaptive objects sequentially and stops before late recovery/cancellation. Resource pressure intentionally caps the runtime at one worker, but this run retains no pressure snapshot. Capture that decision before changing policy or fixture resources; do not weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Use the new worker-budget observations in the real provider trial, restore proven concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `9b602bb5` reconstructs native and HTTPS adaptive objects sequentially and stops before late recovery/cancellation. Pressure can cap admission at one worker, but this run retains no pressure snapshot. The new diagnostics distinguish pressure, missing observations, resource capacity and shared reservations without changing admission. Obtain live evidence before changing policy or fixture resources; do not weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather` | Deliver core-cache products into the actual client, then verify native packaging. | The pinned web build, sixteen Flutter tests and actual isolated browser interaction with synthetic products pass. Core delivery and native Android/iOS execution remain unverified. Cover the client's shorter timeout and the bridge's cancellation cleanup when testing cold retrieval; reuse the shared provider without another scheduler. | @@ -43,6 +43,21 @@ changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. +Worker-budget diagnostics are now opt-in through one dedicated debug target. +Each provider batch retains only bounded allowance and reservation counts, +resource-capacity minima and closed pressure/unavailability flags from the same +samples used for admission. No object, peer, route, path or user identifiers are +added. Default logging remains unchanged. The disposable fixture exports separate +native and HTTPS observations before transfer validation and finalizes interrupted +observations after agent retirement; diagnostic failure cannot turn a failed +transfer into a pass. Seventeen focused Rust tests pass, including four driver +tests for three concurrent duplex streams without pressure, one under pressure, +refused admission and release after interruption. Twelve extraction checks, +eight existing adaptive-evidence checks and eighteen runner checks also pass. +These are component results, not a new protected-network overlap result. +The pressure thresholds, shared resource budget and packet-overlap gates +are unchanged; the next real trial must establish the actual limiting condition. + The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first completed, single-offer discovery. A temporary provider-side gate is bounded to diff --git a/tests/integration/content-provider-adaptive-smoke.sh b/tests/integration/content-provider-adaptive-smoke.sh index 9ef0972d3..016b19191 100644 --- a/tests/integration/content-provider-adaptive-smoke.sh +++ b/tests/integration/content-provider-adaptive-smoke.sh @@ -3,6 +3,18 @@ # Sourced only by the disposable content-provider KVM topology. # shellcheck disable=SC2154,SC2034 +content_provider_worker_budget_capture() { + # Diagnostic-only: never replace the real transfer/evidence result on failure. + python3 -B "$source_directory/tests/integration/content-provider-worker-budget.py" \ + "$1" "$WORK" "$2" || true +} + +content_provider_worker_budget_finalize() { + # Agent retirement drains RAII summaries after a cancelled/failed CLI command. + python3 -B "$source_directory/tests/integration/content-provider-worker-budget.py" \ + finalize "$WORK" || true +} + content_provider_adaptive_event_count() { awk -v baseline="$adaptive_baseline_ms" -v event="event=$2" ' $1 ~ /^[0-9]+$/ && $1 > baseline && $3 == event { count++ } @@ -102,6 +114,8 @@ content_provider_adaptive_https_run() { content_provider_adaptive_start_control_observer "$ah_prefix-control-privacy" \ || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_CONTROL_FAILED PHASE=content-provider-adaptive-https-fetch + content_provider_worker_budget_capture mark https + ah_fetch_status=0 # Reuse the existing non-browser consumer: it checks real inherited UID/caps/netns and # executes ordinary origin-digest CLI without a publisher key or manifest argument. timeout --signal=TERM --kill-after=5s 120s ip netns exec "$CLIENT" setpriv \ @@ -111,7 +125,13 @@ content_provider_adaptive_https_run() { "$binary_directory/volparossa" "$WORK/runtime-client/control/agent.sock" "$ah_cache" "$ah_user" \ "$ah_parent_ns" "$ah_client_ns" "$WORKER_UID" "$WORKER_GID" "$ah_control_gid" \ >"$WORK/$ah_prefix-application.json" 2>"$WORK/$ah_prefix-application.err" \ - || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_FETCH_FAILED + || ah_fetch_status=$? + if [ "$ah_fetch_status" -eq 0 ]; then + content_provider_worker_budget_capture finish https + else + content_provider_worker_budget_capture pending https + fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_FETCH_FAILED + fi jq -e '.final' "$WORK/$ah_prefix-application.json" >"$WORK/$ah_prefix-fetch.json" \ || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_RECEIPT_INVALID benchmark_capture_paths "$ah_prefix-live" mptcp || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_PATHS_FAILED @@ -258,12 +278,20 @@ content_provider_adaptive_run() { content_provider_adaptive_start_control_observer content-provider-adaptive-control-privacy \ || fail CONTENT_PROVIDER_ADAPTIVE_CONTROL_CAPTURE_UNAVAILABLE PHASE=content-provider-adaptive-fetch + content_provider_worker_budget_capture mark native + adaptive_fetch_status=0 timeout --signal=TERM --kill-after=5s 120s "$binary_directory/volparossa" \ --control-socket "$WORK/runtime-client/control/agent.sock" \ content fetch --manifest "$adaptive_manifest" --publisher-key "$adaptive_publisher" \ --cache "$adaptive_client/cache" --output "$adaptive_client/object.bin" \ >"$WORK/content-provider-adaptive-fetch.json" 2>"$WORK/content-provider-adaptive-fetch.err" \ - || fail CONTENT_PROVIDER_ADAPTIVE_RUNTIME_FETCH_FAILED + || adaptive_fetch_status=$? + if [ "$adaptive_fetch_status" -eq 0 ]; then + content_provider_worker_budget_capture finish native + else + content_provider_worker_budget_capture pending native + fail CONTENT_PROVIDER_ADAPTIVE_RUNTIME_FETCH_FAILED + fi benchmark_capture_paths content-provider-adaptive-live mptcp \ || fail CONTENT_PROVIDER_ADAPTIVE_LIVE_PATHS_UNAVAILABLE jq -e --arg context "$adaptive_context" '.route_context_id == $context' \ diff --git a/tests/integration/content-provider-worker-budget.py b/tests/integration/content-provider-worker-budget.py new file mode 100644 index 000000000..42e087663 --- /dev/null +++ b/tests/integration/content-provider-worker-budget.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Closed resource diagnostics for the explicit disposable provider fixture only. + +The private markers bind byte ranges of one owned agent log. Exported reports +contain only bounded budget summaries, never log envelopes, identifiers or paths. +These observations do not establish transfer completion, concurrency or cleanup. +""" +import io +import json +import os +from pathlib import Path +import stat +import sys +import tempfile + +TARGET = "volparossa_agent::content::worker_budget" +CODE = "CONTENT_WORKER_BUDGET_SUMMARY" +PHASES = ("native", "https") +MAX_INPUT = 8 * 1024 * 1024 +MAX_LINE = 16384 +MAX_LINES = 8192 +MAX_BUDGET = 4096 +MAX_RECORDS = 16 +MAX_OUTPUT = 65536 +U32 = 2**32 - 1 +OBSERVATIONS = {"initial_allowance", "min_allowance", "max_allowance", + "min_memory_workers", "min_fd_workers"} +COUNTERS = {"peak_shared_held", "admitted_leases", "denied_leases", "peak_leases", "active_leases"} +CAUSES = {"cpu_pressure", "memory_pressure", "io_pressure", "cpu_unavailable", + "memory_unavailable", "io_unavailable", "resource_unavailable", "lock_unavailable"} + + +def require(condition): + if not condition: + raise ValueError("invalid bounded worker-budget evidence") + + +def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result) + result[key] = value + return result + + +def decode(raw): + def invalid_constant(_value): + raise ValueError("non-finite JSON") + return json.loads(raw, object_pairs_hook=unique_object, parse_constant=invalid_constant) + + +def summary(value): + require(type(value) is dict and set(value) == + {"schema_version", "outcome", "causes"} | OBSERVATIONS | COUNTERS) + require(type(value["schema_version"]) is int and value["schema_version"] == 1) + require(type(value["outcome"]) is str and value["outcome"] in ("returned", "failed", "interrupted")) + for key in OBSERVATIONS | COUNTERS: + number = value[key] + require(key in OBSERVATIONS and number is None or type(number) is int and 0 <= number <= U32) + require(type(value["causes"]) is dict and set(value["causes"]) == CAUSES + and all(type(flag) is bool for flag in value["causes"].values())) + low, high = value["min_allowance"], value["max_allowance"] + require((low is None) == (high is None) and (low is None or low <= high)) + require(value["active_leases"] <= value["peak_leases"] <= value["admitted_leases"]) + return value + + +def extract(raw): + """Accept complete JSON log lines only; never return an unvalidated raw line.""" + require(type(raw) is bytes and len(raw) <= MAX_INPUT) + records = [] + incomplete = False + stream = io.BytesIO(raw) + count = 0 + while line := stream.readline(MAX_LINE + 2): + count += 1 + require(count <= MAX_LINES and len(line.removesuffix(b"\n")) <= MAX_LINE) + if not line.endswith(b"\n"): + incomplete = True + break + if not line.strip(): + continue + event = decode(line) + require(type(event) is dict) + if event.get("target") != TARGET: + continue + fields = event.get("fields") + require(type(fields) is dict) + if fields.get("code") != CODE: + continue + require(event.get("level") == "DEBUG" and set(fields) == {"code", "budget"}) + budget = fields["budget"] + require(type(budget) is str and len(budget.encode("utf-8")) <= MAX_BUDGET) + records.append(summary(decode(budget))) + require(len(records) <= MAX_RECORDS) + return records, incomplete + + +def checked_open(path): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = os.fstat(fd) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_mode & 0o777 == 0o600) + return fd, info + except BaseException: + os.close(fd) + raise + + +def read_private(path, maximum): + fd, info = checked_open(path) + try: + require(info.st_size <= maximum) + raw = os.read(fd, maximum + 1) + after = os.fstat(fd) + require(len(raw) == info.st_size and after.st_size == info.st_size + and after.st_mtime_ns == info.st_mtime_ns and after.st_ctime_ns == info.st_ctime_ns) + return raw + finally: + os.close(fd) + + +def write_private(path, value, maximum, *, exclusive=False): + raw = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode("ascii") + require(len(raw) <= maximum) + if exclusive: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as destination: + destination.write(raw) + return + # Output paths are fixed within this fixture's owner-controlled directory. + # Refuse unexpected targets rather than following or overwriting a link. + if os.path.lexists(path): + fd, _ = checked_open(path) + os.close(fd) + fd, temporary = tempfile.mkstemp(prefix=".worker-budget-", suffix=".part", dir=path.parent) + try: + with os.fdopen(fd, "wb") as destination: + destination.write(raw) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def paths(work, phase): + require(phase in PHASES) + suffix = "" if phase == "native" else "-https" + return (work / ".content-worker-budget-{}.part".format(phase), + work / f"content-provider-adaptive{suffix}-worker-budget.json") + + +def mark(work, phase): + marker, _ = paths(work, phase) + fd, info = checked_open(work / "agent-client.log") + try: + if info.st_size: + require(os.pread(fd, 1, info.st_size - 1) == b"\n") + value = dict(schema_version=1, phase=phase, device=info.st_dev, inode=info.st_ino, + start=info.st_size, end=None) + write_private(marker, value, MAX_BUDGET, exclusive=True) + finally: + os.close(fd) + + +def finish(work, phase, *, pending=False): + marker, output = paths(work, phase) + report = dict(schema_version=1, kind="content-worker-budget-diagnostic", phase=phase, + phase_closed=not pending, complete=False, status="invalid", records=[]) + try: + value = decode(read_private(marker, MAX_BUDGET)) + require(type(value) is dict and set(value) == + {"schema_version", "phase", "device", "inode", "start", "end"}) + require(type(value["schema_version"]) is int and value["schema_version"] == 1 and value["phase"] == phase) + require(all(type(value[key]) is int and value[key] >= 0 for key in ("device", "inode", "start"))) + require(value["end"] is None or type(value["end"]) is int and value["end"] >= value["start"]) + fd, info = checked_open(work / "agent-client.log") + try: + require((info.st_dev, info.st_ino) == (value["device"], value["inode"])) + end = info.st_size if value["end"] is None else value["end"] + require(value["start"] <= end <= info.st_size and end - value["start"] <= MAX_INPUT) + raw = os.pread(fd, end - value["start"], value["start"]) + require(len(raw) == end - value["start"] and os.fstat(fd).st_size >= end) + finally: + os.close(fd) + records, incomplete = extract(raw) + if not pending and value["end"] is None: + value["end"] = end + write_private(marker, value, MAX_BUDGET) + report.update(records=records, status="incomplete" if incomplete or pending else + "observed" if records else "missing", complete=bool(records) and not incomplete and not pending) + except (OSError, ValueError, KeyError, TypeError, UnicodeError, RecursionError): + pass # The closed invalid report replaces neither the failure nor absent evidence with success. + write_private(output, report, MAX_OUTPUT) + return report + + +def finalize(work): + for phase in PHASES: + marker, _ = paths(work, phase) + if os.path.lexists(marker): + finish(work, phase) + + +def main(argv): + require(len(argv) >= 3 and argv[1] in ("mark", "finish", "pending", "finalize")) + work = Path(argv[2]) + require(work.is_absolute() and work.resolve() == work) + info = work.stat() + require(stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and not info.st_mode & 0o022) + if argv[1] == "finalize": + require(len(argv) == 3) + finalize(work) + else: + require(len(argv) == 4) + if argv[1] == "mark": + mark(work, argv[3]) + else: + finish(work, argv[3], pending=argv[1] == "pending") + + +if __name__ == "__main__": + try: + main(sys.argv) + except (OSError, ValueError, KeyError, TypeError, UnicodeError, RecursionError): + print("worker-budget diagnostic unavailable", file=sys.stderr) + sys.exit(1) diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 0d7315f22..5f97b7a73 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1358,7 +1358,7 @@ if [ "$scenario" = content-provider ]; then content-provider-https-smoke.sh content-provider-https-smoke.py content-publication-smoke.sh content-named-smoke.sh \ content-provider-site-smoke.sh content-provider-site-smoke.py \ content-provider-filter-smoke.sh content-provider-filter-smoke.py \ - content-provider-adaptive-smoke.sh content-provider-adaptive-smoke.py \ + content-provider-adaptive-smoke.sh content-provider-adaptive-smoke.py content-provider-worker-budget.py \ content-provider-late-recovery.sh content-provider-late-recovery.py content-cancellation-smoke.py; do if [ ! -f "$source_directory/tests/integration/$provider_fixture" ] \ || [ -L "$source_directory/tests/integration/$provider_fixture" ]; then @@ -2325,6 +2325,9 @@ cleanup() { done fi for cleanup_unit in $AGENT_UNITS; do retire_unit "$cleanup_unit" || true; done + if [ "$scenario" = content-provider ] && command -v content_provider_worker_budget_finalize >/dev/null 2>&1; then + content_provider_worker_budget_finalize + fi if { [ "$scenario" = content-provider ] || [ "$scenario" = content-custody ]; } \ && ip netns exec "$CLIENT" nft list table inet vpa_content_adaptive_client_control >/dev/null 2>&1; then ip netns exec "$CLIENT" nft delete table inet vpa_content_adaptive_client_control || original_status=1 @@ -3966,6 +3969,10 @@ launch_agent() { [ "$(unit_load_state "$agent_unit")" = not-found ] || fail AGENT_UNIT_COLLISION AGENT_UNITS="$AGENT_UNITS $agent_unit" agent_rust_log=volparossa_agent=info + if [ "$scenario" = content-provider ] && [ "$node" = client ]; then + # Disposable fixture opt-in: closed per-batch resources only, no object/route/peer identities. + agent_rust_log=$agent_rust_log,volparossa_agent::content::worker_budget=debug + fi if [ "$scenario" = browser-network ] && [ "$node" = client ]; then agent_rust_log=$agent_rust_log,volparossa_agent::browser_gateway::connect=debug fi diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index ae2bafa85..b1215b4b0 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -914,6 +914,7 @@ SAFE_NAMES = {"runner.stdout", "runner.stderr", "guest-exit-status", "current-ph "private-unbound-proof.json", "private-unbound-build.log", "private-unbound-package.json", "private-unbound-package-lifecycle.json", "worker-network-diagnostics.txt", "host-state-before.json", "host-state-after.json", + "content-provider-adaptive-worker-budget.json", "content-provider-adaptive-https-worker-budget.json", "report.json", "local-link-smoke.json", "wifi-link-smoke.json", "reciprocity-smoke.json", "mixed-link-smoke.json", "mpquic-growth-smoke.json", "mptcp-growth-smoke.json", "mptcp-refill-smoke.json", "sharing-smoke.json", "download-sharing-smoke.json", "uplink-link-smoke.json", "crash-recovery.json", "content-network-smoke.json", "content-message-smoke.json", "content-https-smoke.json", "content-provider-smoke.json", "content-replication-smoke.json", "content-mailbox-smoke.json", "dns-cache-smoke.json", diff --git a/tests/integration/test-content-provider-worker-budget.py b/tests/integration/test-content-provider-worker-budget.py new file mode 100644 index 000000000..55204aded --- /dev/null +++ b/tests/integration/test-content-provider-worker-budget.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Synthetic closed-log and failure-retention checks; no daemon or VM starts.""" +import json +import os +from pathlib import Path +import runpy +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +HELPER = HERE / "content-provider-worker-budget.py" +CHECK = runpy.run_path(str(HELPER)) + + +def budget(**changes): + value = dict(schema_version=1, outcome="returned", initial_allowance=3, + min_allowance=1, max_allowance=3, min_memory_workers=3, + min_fd_workers=100, peak_shared_held=3, admitted_leases=3, + denied_leases=0, peak_leases=3, active_leases=0, + causes={key: False for key in CHECK["CAUSES"]}) + value.update(changes) + return value + + +def event(value=None, **changes): + envelope = dict(timestamp="not-for-export", level="DEBUG", target=CHECK["TARGET"], + fields=dict(code=CHECK["CODE"], budget=json.dumps(value or budget()))) + envelope.update(changes) + return json.dumps(envelope).encode() + b"\n" + + +class WorkerBudgetDiagnostics(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="provider-budget-") + self.addCleanup(self.directory.cleanup) + self.work = Path(self.directory.name) + self.log = self.work / "agent-client.log" + self.log.touch(mode=0o600) + + def append(self, raw): + with self.log.open("ab") as output: + output.write(raw) + + def marker(self, phase): + CHECK["mark"](self.work, phase) + + def finish(self, phase, pending=False): + return CHECK["finish"](self.work, phase, pending=pending) + + def report(self, phase): + return CHECK["paths"](self.work, phase)[1] + + def test_phase_boundaries_prevent_native_https_and_later_event_bleed(self): + self.append(event(budget(initial_allowance=99))) + self.marker("native") + self.append(event()) + native = self.finish("native") + self.assertEqual(native["status"], "observed") + self.assertTrue(native["complete"]) + self.marker("https") + self.append(event(budget(outcome="failed"))) + https = self.finish("https") + self.append(event(budget(outcome="interrupted"))) + CHECK["finalize"](self.work) + self.assertEqual(json.loads(self.report("native").read_bytes()), native) + self.assertEqual(json.loads(self.report("https").read_bytes()), https) + self.assertEqual([item["outcome"] for item in https["records"]], ["failed"]) + self.assertNotIn("success", native) + + def test_unfinished_failed_command_is_finalized_after_summary_arrives(self): + self.marker("native") + pending = self.finish("native", pending=True) + self.assertFalse(pending["phase_closed"]) + self.assertFalse(pending["complete"]) + self.append(event(budget(outcome="interrupted"))) + CHECK["finalize"](self.work) + final = json.loads(self.report("native").read_bytes()) + self.assertEqual(final["records"][0]["outcome"], "interrupted") + self.assertTrue(final["phase_closed"]) + self.assertTrue(final["complete"]) + self.assertFalse(self.report("https").exists(), "unstarted phase must not acquire evidence") + + def test_missing_observation_is_not_fabricated_zero_or_success(self): + self.marker("native") + missing = self.finish("native") + self.assertEqual(missing["status"], "missing") + self.assertFalse(missing["complete"]) + nulls = budget(**{key: None for key in CHECK["OBSERVATIONS"]}) + nulls["causes"]["resource_unavailable"] = True + parsed, _ = CHECK["extract"](event(nulls)) + self.assertIsNone(parsed[0]["initial_allowance"]) + self.assertEqual(parsed[0]["denied_leases"], 0) + self.assertTrue(parsed[0]["causes"]["resource_unavailable"]) + + def test_only_closed_budget_survives_unrelated_log_envelopes(self): + self.marker("native") + self.append(event(target="unrelated", fields={"private_url": "https://private.invalid/secret"})) + self.append(event()) + report = self.finish("native") + self.assertEqual(report["records"], [budget()]) + raw = self.report("native").read_text() + for private in ("timestamp", "not-for-export", "private_url", "private.invalid", "target", "device", "inode"): + self.assertNotIn(private, raw) + self.assertEqual(self.report("native").stat().st_mode & 0o777, 0o600) + + def test_unknown_keys_free_form_values_bool_counters_and_invalid_ranges_rejected(self): + changes = ({"outcome": "secret URL"}, {"initial_allowance": -1}, {"peak_shared_held": True}, + {"min_fd_workers": 2**32}, {"active_leases": 4}, {"min_allowance": 4}, + {"max_allowance": None}, {"peer_id": "not-allowed"}, {"schema_version": True}) + for mutation in changes: + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + CHECK["extract"](event(budget(**mutation))) + for mutate in (lambda v: v["causes"].update(extra=False), + lambda v: v["causes"].update(cpu_unavailable="false")): + value = budget(); mutate(value) + with self.assertRaises(ValueError): + CHECK["extract"](event(value)) + + def test_envelope_and_nested_json_duplicates_nonfinite_and_extra_fields_rejected(self): + raw = event() + with self.assertRaises(ValueError): + CHECK["extract"](raw.replace(b'"level": "DEBUG"', b'"level":"DEBUG","level":"DEBUG"')) + fields = {"code": CHECK["CODE"], "budget": json.dumps(budget()).replace('"schema_version": 1', + '"schema_version":1,"schema_version":1')} + with self.assertRaises(ValueError): + CHECK["extract"](event(fields=fields)) + for fields in ({"code": CHECK["CODE"], "budget": json.dumps(budget()), "secret": "no"}, + {"code": CHECK["CODE"], "budget": {"not": "a string"}}, + {"code": CHECK["CODE"], "budget": "NaN"}): + with self.assertRaises(ValueError): + CHECK["extract"](event(fields=fields)) + + def test_resource_bounds_reject_excess_input_line_budget_or_summaries(self): + samples = (b"x" * (CHECK["MAX_INPUT"] + 1), + b"x" * (CHECK["MAX_LINE"] + 1) + b"\n", + b"\n" * (CHECK["MAX_LINES"] + 1), + event(fields={"code": CHECK["CODE"], "budget": " " * (CHECK["MAX_BUDGET"] + 1)}), + event() * (CHECK["MAX_RECORDS"] + 1)) + for raw in samples: + with self.assertRaises(ValueError): + CHECK["extract"](raw) + + def test_partial_trailing_record_is_incomplete_not_success(self): + self.marker("native") + self.append(event() + b'{"target":') + report = self.finish("native", pending=True) + self.assertEqual(report["status"], "incomplete") + self.assertFalse(report["complete"]) + self.assertEqual(report["records"], [budget()]) + + def test_rotation_truncation_symlink_and_hardlink_refused(self): + for action in ("rotation", "truncate", "symlink", "hardlink"): + with self.subTest(action=action), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); log = work / "agent-client.log" + log.write_bytes(event()); log.chmod(0o600) + CHECK["mark"](work, "native") + if action == "truncate": + log.write_bytes(b"") + else: + log.rename(work / "old") + if action == "symlink": log.symlink_to(work / "old") + elif action == "hardlink": os.link(work / "old", log) + else: log.write_bytes(event()); log.chmod(0o600) + report = CHECK["finish"](work, "native") + self.assertEqual(report["status"], "invalid") + self.assertEqual(report["records"], []) + + def test_malformed_logs_and_markers_export_only_closed_invalid_status(self): + self.marker("native") + self.append(b"secret malformed diagnostic\n") + self.assertEqual(self.finish("native")["status"], "invalid") + marker, _ = CHECK["paths"](self.work, "native") + marker.write_text('{"private":"do not export"}') + report = self.finish("native") + self.assertEqual(report["records"], []) + self.assertNotIn("private", self.report("native").read_text()) + marker.write_text("[" * 1500 + "0" + "]" * 1500) + self.assertEqual(self.finish("native")["status"], "invalid") + + def test_actual_cli_marker_finish_and_failed_acceptance_retains_reports(self): + self.work.chmod(0o755) # Actual fixture root is searchable by service UIDs, never writable by them. + for args in (("mark", "native"), ("pending", "native")): + result = subprocess.run([sys.executable, "-B", str(HELPER), args[0], str(self.work), args[1]], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, b"") + self.append(event()) + result = subprocess.run([sys.executable, "-B", str(HELPER), "finalize", str(self.work)], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + # An unrelated acceptance failure neither changes nor deletes this diagnostic. + report = self.report("native").read_bytes() + (self.work / "content-provider-smoke.json").write_text('{"success":false}') + self.assertEqual(self.report("native").read_bytes(), report) + + def test_glue_opt_in_cleanup_order_and_closed_emergency_allowlist(self): + topology = (HERE / "kvm-alpha-topology.sh").read_text() + optin = ('if [ "$scenario" = content-provider ] && [ "$node" = client ]; then\n' + ' # Disposable fixture opt-in: closed per-batch resources only, no object/route/peer identities.\n' + ' agent_rust_log=$agent_rust_log,volparossa_agent::content::worker_budget=debug') + self.assertIn(optin, topology) + retire = topology.index('for cleanup_unit in $AGENT_UNITS; do retire_unit "$cleanup_unit"') + finalize = topology.index(" content_provider_worker_budget_finalize", retire) + self.assertLess(retire, finalize) + self.assertLess(finalize, topology.index(" copy_artifacts || original_status=1", finalize)) + shell = (HERE / "content-provider-adaptive-smoke.sh").read_text() + for phase in CHECK["PHASES"]: + self.assertIn(f"content_provider_worker_budget_capture mark {phase}", shell) + self.assertIn(f"content_provider_worker_budget_capture pending {phase}", shell) + self.assertIn(f"content_provider_worker_budget_capture finish {phase}", shell) + self.assertIn('|| fail CONTENT_PROVIDER_ADAPTIVE_EVIDENCE_INVALID', shell) + emergency = (HERE / "run-alpha-topology-vm.sh").read_text() + for phase in CHECK["PHASES"]: + self.assertIn('"' + self.report(phase).name + '"', emergency) + for name in ("kvm-alpha-topology.sh", "run-alpha-topology-vm.sh", "content-provider-adaptive-smoke.sh"): + subprocess.run(["/bin/sh", "-n", str(HERE / name)], check=True, timeout=5) + + +if __name__ == "__main__": + unittest.main() From fa89523b0c49768d7c56c58f2cec3e0c902dccbd Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:02:49 +0200 Subject: [PATCH 29/49] test: require genuine resource readiness before adaptive transfers --- docs/IMPLEMENTATION_STATUS.md | 43 ++- .../content-provider-adaptive-smoke.sh | 7 + .../content-provider-resource-ready.py | 197 ++++++++++++++ tests/integration/kvm-alpha-topology.sh | 1 + tests/integration/run-alpha-topology-vm.sh | 1 + .../test-content-provider-resource-ready.py | 256 ++++++++++++++++++ 6 files changed, 503 insertions(+), 2 deletions(-) create mode 100644 tests/integration/content-provider-resource-ready.py create mode 100644 tests/integration/test-content-provider-resource-ready.py diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 388922028..a4f2b010c 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,7 +29,7 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Use the new worker-budget observations in the real provider trial, restore proven concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `9b602bb5` reconstructs native and HTTPS adaptive objects sequentially and stops before late recovery/cancellation. Pressure can cap admission at one worker, but this run retains no pressure snapshot. The new diagnostics distinguish pressure, missing observations, resource capacity and shared reservations without changing admission. Obtain live evidence before changing policy or fixture resources; do not weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Resolve the observed fixture I/O pressure, prove concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `7ad108c0` reconstructs native and HTTPS adaptive objects sequentially. Both budget reports identify I/O pressure as the sole pressure/unavailability flag, holding admission at one worker despite memory and FD headroom. Identify and reduce the actual storage bottleneck or establish bounded readiness; do not override pressure protection or weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather` | Deliver core-cache products into the actual client, then verify native packaging. | The pinned web build, sixteen Flutter tests and actual isolated browser interaction with synthetic products pass. Core delivery and native Android/iOS execution remain unverified. Cover the client's shorter timeout and the bridge's cancellation cleanup when testing cold retrieval; reuse the shared provider without another scheduler. | @@ -56,7 +56,46 @@ refused admission and release after interruption. Twelve extraction checks, eight existing adaptive-evidence checks and eighteen runner checks also pass. These are component results, not a new protected-network overlap result. The pressure thresholds, shared resource budget and packet-overlap gates -are unchanged; the next real trial must establish the actual limiting condition. +are unchanged. + +The real diagnostic run on `7ad108c0211183abba0e27a7922849a0455c1d9c` +identifies I/O pressure as the admission limit in both adaptive transfers. Each +complete budget report records allowance one throughout, three admitted leases, +peak concurrency one, no denied leases and zero active leases on return. Memory +and FD capacity minima are twelve and twenty-nine workers respectively; CPU, +memory and unavailable-observation flags are false. This establishes why these +transfers were serialized, not which activity caused the underlying I/O pressure. +Earlier batches in the same run had no pressure and allowed twelve workers, +including an observed two-worker peak. The pressure therefore was not continuous +since the build; it reappeared around adaptive preparation or execution. +Inspect fixture storage and readiness before changing resources; do not bypass +the owner's pressure protection to satisfy a concurrency test. + +Both transfers reconstruct 15,728,640 bytes in sixty chunks from three providers, +with no origin body bytes or range requests. Their three-provider bulk windows +have negative overlap: `-6,629,715,618 ns` native and `-5,983,140,384 ns` HTTPS. +The earlier two-provider transfer passes its existing path/control checks and +reconstructs 2,097,275 bytes in nine chunks, with `373,929,069 ns` bulk overlap; +that does not satisfy the separate three-provider requirement. The run stops +at `content-provider-adaptive-cleanup` with +`CONTENT_PROVIDER_ADAPTIVE_EVIDENCE_INVALID`; late recovery and requester +cancellation are not reached. The report is retained at +`.git/ci-evidence/content-provider-budget-01.VUAE79/output/content-provider-smoke.json`, +SHA-256 `c5dd6bebef5d803ccbba8dd4e5f11b8ddad86b824febe294f5882432b337dc19`. +The separate native and HTTPS worker-budget reports are complete and observed. +Cleanup leaves zero owned objects, with identical guest-root network-state +hashes. The VM and its owned scratch overlay are removed. Development-host DNS +and IPv4/IPv6 route hashes remain unchanged; host-firewall comparison remains +unavailable. No weather origin was queried. + +The next fixture candidate waits for three consecutive guest-global PSI samples, +one second apart, below the unchanged production pressure thresholds before each +adaptive fetch. Its single thirty-second deadline fails visibly if readiness is +unavailable. Fifteen focused checks pass, alongside the twelve budget-extraction, +eight adaptive-evidence and eighteen runner checks. Bounded native/HTTPS receipts explicitly do +not prove memory, FD or cgroup capacity. This adds no product delay or admission +override: cache writes may reintroduce I/O pressure, and the live worker budget +and three-provider packet-overlap requirements still decide the result. The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first diff --git a/tests/integration/content-provider-adaptive-smoke.sh b/tests/integration/content-provider-adaptive-smoke.sh index 016b19191..5d202b330 100644 --- a/tests/integration/content-provider-adaptive-smoke.sh +++ b/tests/integration/content-provider-adaptive-smoke.sh @@ -110,6 +110,10 @@ content_provider_adaptive_https_run() { TLS_POLICY_SERVER_PID=$! wait_observer "$TLS_POLICY_SERVER_PID" "$ah_origin_report.ready" || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_ORIGIN_FAILED install -o "$WORKER_UID" -g "$WORKER_GID" -m 0400 "$WORK/destination/$ah_prefix-origin.pem" "$ah_user/origin.pem" + # Fixture readiness only: current production admission and triple overlap still decide. + PHASE=content-provider-adaptive-https-resource-ready + python3 -B "$source_directory/tests/integration/content-provider-resource-ready.py" "$WORK" https \ + || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_RESOURCE_UNAVAILABLE start_privacy_observers "$ah_prefix-privacy" || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_CAPTURE_FAILED content_provider_adaptive_start_control_observer "$ah_prefix-control-privacy" \ || fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_CONTROL_FAILED @@ -271,6 +275,9 @@ content_provider_adaptive_run() { || fail CONTENT_PROVIDER_ADAPTIVE_REGISTRATION_INCOMPLETE done capture_product_logs + PHASE=content-provider-adaptive-resource-ready + python3 -B "$source_directory/tests/integration/content-provider-resource-ready.py" "$WORK" native \ + || fail CONTENT_PROVIDER_ADAPTIVE_RESOURCE_UNAVAILABLE adaptive_baseline_ms=$(client_log_baseline_ms) || fail CONTENT_PROVIDER_ADAPTIVE_BASELINE_UNAVAILABLE PHASE=content-provider-adaptive-capture start_privacy_observers content-provider-adaptive-privacy \ diff --git a/tests/integration/content-provider-resource-ready.py b/tests/integration/content-provider-resource-ready.py new file mode 100644 index 000000000..42e064c1a --- /dev/null +++ b/tests/integration/content-provider-resource-ready.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Bounded guest-fixture PSI precondition, never worker admission or transfer proof. + +The production budget still checks current PSI, cgroup-constrained memory and +descriptor capacity itself. Cold-cache I/O may reintroduce pressure after this +precondition; the unchanged overlap checks must then fail normally. +""" +import json +import math +import os +from pathlib import Path +import re +import stat +import sys +import time + +THRESHOLDS = {"cpu": 20.0, "memory": 1.0, "io": 10.0} +MAX_SOURCE_BYTES = 16 * 1024 +DEADLINE_NS = 30_000_000_000 +INTERVAL_NS = 1_000_000_000 +REQUIRED = 3 +MAX_SAMPLES = DEADLINE_NS // INTERVAL_NS +STATES = {"below_threshold", "pressured", "unavailable"} +SCOPE = {"guest_global_psi": True, "memory_capacity_checked": False, + "fd_capacity_checked": False, "cgroup_capacity_checked": False, + "worker_capacity_proven": False, "production_policy_overridden": False} +OUTPUTS = {"native": "content-provider-adaptive-resource-ready.json", + "https": "content-provider-adaptive-https-resource-ready.json"} + + +def require(condition): + if not condition: + raise ValueError("invalid resource readiness evidence") + + +def read_bounded(path): + descriptor = os.open(path, os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + info = os.fstat(descriptor) + require(stat.S_ISREG(info.st_mode) and info.st_size <= MAX_SOURCE_BYTES) + # Proc files normally report size zero; bound actual reads independently. + chunks = [] + size = 0 + while size <= MAX_SOURCE_BYTES: + raw = os.read(descriptor, MAX_SOURCE_BYTES + 1 - size) + if not raw: + break + chunks.append(raw) + size += len(raw) + require(size <= MAX_SOURCE_BYTES) + return b"".join(chunks).decode("utf-8", errors="strict") + finally: + os.close(descriptor) + + +def average(text): + # Match the production source and first `some avg10` field, not `full`/avg60. + line = next((line for line in text.splitlines() if line.startswith("some ")), None) + require(line is not None) + field = next((field[6:] for field in line.split() if field.startswith("avg10=")), None) + require(field is not None) + require(re.fullmatch(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?", field) is not None) + value = float(field) + require(math.isfinite(value) and 0.0 <= value <= 100.0) + return value + + +def observe(reader=read_bounded): + result = {} + for kind, threshold in THRESHOLDS.items(): + try: + value = average(reader(Path("/proc/pressure") / kind)) + result[kind] = "pressured" if value >= threshold else "below_threshold" + except (OSError, ValueError, UnicodeError): + result[kind] = "unavailable" + return result + + +def validate_observation(observation): + require(type(observation) is dict and observation.keys() == THRESHOLDS.keys()) + require(all(type(value) is str and value in STATES for value in observation.values())) + + +def validate(receipt): + require(type(receipt) is dict and receipt.keys() == { + "schema_version", "kind", "phase", "scope", "status", "deadline_ms", + "sample_interval_ms", "required_consecutive", "samples", "consecutive", + "elapsed_ms", "last_observation"}) + require(type(receipt["schema_version"]) is int and receipt["schema_version"] == 1) + require(receipt["kind"] == "content-provider-psi-readiness") + require(type(receipt["phase"]) is str and receipt["phase"] in OUTPUTS) + require(type(receipt["status"]) is str and receipt["status"] in {"ready", "timeout", "unavailable"}) + scope = receipt["scope"] + require(type(scope) is dict and scope == SCOPE and all(type(value) is bool for value in scope.values())) + for key, expected in (("deadline_ms", 30000), ("sample_interval_ms", 1000), ("required_consecutive", REQUIRED)): + require(type(receipt[key]) is int and receipt[key] == expected) + for key, maximum in (("samples", MAX_SAMPLES), ("consecutive", REQUIRED), ("elapsed_ms", 2**64 - 1)): + require(type(receipt[key]) is int and 0 <= receipt[key] <= maximum) + samples, consecutive = receipt["samples"], receipt["consecutive"] + require(consecutive <= samples) + require(receipt["elapsed_ms"] >= max(0, samples - 1) * 1000) + observation = receipt["last_observation"] + if observation is None: + require(samples == 0) + else: + validate_observation(observation) + require(samples > 0) + if receipt["status"] == "ready": + require(consecutive == REQUIRED and (REQUIRED - 1) * 1000 <= receipt["elapsed_ms"] < 30000) + require(all(value == "below_threshold" for value in observation.values())) + elif receipt["status"] == "timeout": + require(receipt["elapsed_ms"] >= 30000 and consecutive < REQUIRED) + else: + require(observation is not None and "unavailable" in observation.values() and consecutive == 0) + require(receipt["elapsed_ms"] < 30000) + return receipt + + +def wait_ready(phase, *, sampler=observe, clock=time.monotonic_ns, sleeper=time.sleep): + require(phase in OUTPUTS) + start = clock() + deadline = start + DEADLINE_NS + samples = consecutive = 0 + now = start + observation = None + status = "timeout" + while samples < MAX_SAMPLES: + sample_start = clock() + now = sample_start + if sample_start >= deadline: + break + observation = sampler() + validate_observation(observation) + samples += 1 + now = clock() + if now >= deadline: + break + if "unavailable" in observation.values(): + consecutive = 0 + status = "unavailable" + break + consecutive = consecutive + 1 if all(value == "below_threshold" for value in observation.values()) else 0 + if consecutive == REQUIRED: + status = "ready" + break + # Re-check the same absolute deadline after early/interrupted sleeps. + next_sample = min(sample_start + INTERVAL_NS, deadline) + while now < next_sample: + sleeper((next_sample - now) / 1_000_000_000) + now = clock() + elapsed_ms = (now - start) // 1_000_000 + return validate(dict(schema_version=1, kind="content-provider-psi-readiness", phase=phase, + scope=dict(SCOPE), status=status, deadline_ms=30000, sample_interval_ms=1000, + required_consecutive=REQUIRED, samples=samples, consecutive=consecutive, + elapsed_ms=elapsed_ms, last_observation=observation)) + + +def run(work, phase, *, waiter=wait_ready): + require(phase in OUTPUTS and work.is_absolute() and work.resolve() == work) + directory = os.open(work, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW) + try: + info = os.fstat(directory) + require(info.st_uid == os.geteuid() and not info.st_mode & 0o022) + # Never overwrite a prior trial's receipt, including dangling symlinks. + name = OUTPUTS[phase] + try: + os.stat(name, dir_fd=directory, follow_symlinks=False) + except FileNotFoundError: + pass + else: + raise ValueError("readiness receipt already exists") + receipt = validate(waiter(phase)) + require(receipt["phase"] == phase) + raw = (json.dumps(receipt, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode() + require(len(raw) <= 2048) + output = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC | os.O_NOFOLLOW, + 0o600, dir_fd=directory) + with os.fdopen(output, "wb") as stream: + stream.write(raw) + return receipt["status"] == "ready" + finally: + os.close(directory) + + +def main(): + try: + require(len(sys.argv) == 3) + return 0 if run(Path(sys.argv[1]), sys.argv[2]) else 1 + except (OSError, ValueError, UnicodeError): + # Never export raw proc text, paths or exception details. + print("CONTENT_PROVIDER_RESOURCE_READINESS_INVALID", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 5f97b7a73..a7903917e 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1359,6 +1359,7 @@ if [ "$scenario" = content-provider ]; then content-provider-site-smoke.sh content-provider-site-smoke.py \ content-provider-filter-smoke.sh content-provider-filter-smoke.py \ content-provider-adaptive-smoke.sh content-provider-adaptive-smoke.py content-provider-worker-budget.py \ + content-provider-resource-ready.py \ content-provider-late-recovery.sh content-provider-late-recovery.py content-cancellation-smoke.py; do if [ ! -f "$source_directory/tests/integration/$provider_fixture" ] \ || [ -L "$source_directory/tests/integration/$provider_fixture" ]; then diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index b1215b4b0..707e25dad 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -915,6 +915,7 @@ SAFE_NAMES = {"runner.stdout", "runner.stderr", "guest-exit-status", "current-ph "private-unbound-package.json", "private-unbound-package-lifecycle.json", "worker-network-diagnostics.txt", "host-state-before.json", "host-state-after.json", "content-provider-adaptive-worker-budget.json", "content-provider-adaptive-https-worker-budget.json", + "content-provider-adaptive-resource-ready.json", "content-provider-adaptive-https-resource-ready.json", "report.json", "local-link-smoke.json", "wifi-link-smoke.json", "reciprocity-smoke.json", "mixed-link-smoke.json", "mpquic-growth-smoke.json", "mptcp-growth-smoke.json", "mptcp-refill-smoke.json", "sharing-smoke.json", "download-sharing-smoke.json", "uplink-link-smoke.json", "crash-recovery.json", "content-network-smoke.json", "content-message-smoke.json", "content-https-smoke.json", "content-provider-smoke.json", "content-replication-smoke.json", "content-mailbox-smoke.json", "dns-cache-smoke.json", diff --git a/tests/integration/test-content-provider-resource-ready.py b/tests/integration/test-content-provider-resource-ready.py new file mode 100644 index 000000000..ffed38200 --- /dev/null +++ b/tests/integration/test-content-provider-resource-ready.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Synthetic PSI/clock/descriptor checks; never start a VM or alter host resources.""" +import copy +import json +import os +from pathlib import Path +import re +import runpy +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +HELPER = HERE / "content-provider-resource-ready.py" +CHECK = runpy.run_path(str(HELPER)) + + +def observation(**changes): + result = dict.fromkeys(CHECK["THRESHOLDS"], "below_threshold") + result.update(changes) + return result + + +class Clock: + def __init__(self): + self.now = 0 + self.sleeps = [] + + def clock(self): + return self.now + + def sleep(self, seconds): + self.sleeps.append(seconds) + self.now += round(seconds * 1_000_000_000) + + +def wait(samples=None, *, clock=None, sampler=None, phase="native"): + clock = clock or Clock() + values = iter(samples or [observation()] * 3) + return CHECK["wait_ready"](phase, sampler=sampler or (lambda: next(values)), + clock=clock.clock, sleeper=clock.sleep) + + +class ResourceReadiness(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="provider-psi-test-") + self.addCleanup(self.directory.cleanup) + self.work = Path(self.directory.name) + + def test_exact_production_threshold_and_byte_limit_parity(self): + rust = (HERE.parents[1] / "crates/volparossa-agent/src/resource_headroom.rs").read_text() + actual = {kind: float(value) for kind, value in re.findall( + r'PressureState::from_average\(observe\("(cpu|memory|io)"\), ([0-9.]+)\)', rust)} + self.assertEqual(actual, CHECK["THRESHOLDS"]) + self.assertIn("const MAX_SYSTEM_FILE_BYTES: u64 = 16 * 1024;", rust) + self.assertEqual(CHECK["MAX_SOURCE_BYTES"], 16 * 1024) + for kind, threshold in actual.items(): + for value, expected in ((0, "below_threshold"), (threshold - 0.001, "below_threshold"), + (threshold, "pressured"), (100, "pressured")): + result = CHECK["observe"](lambda path: f"some avg10={value if path.name == kind else 0}\n") + self.assertEqual(result[kind], expected) + + def test_requires_three_consecutive_samples_a_second_apart(self): + clock = Clock() + result = wait(clock=clock) + self.assertEqual(result["status"], "ready") + self.assertEqual((result["samples"], result["consecutive"], result["elapsed_ms"]), (3, 3, 2000)) + self.assertEqual(clock.sleeps, [1, 1]) + self.assertEqual(result["scope"], CHECK["SCOPE"]) + self.assertFalse(result["scope"]["worker_capacity_proven"]) + + def test_pressure_resets_sequence_without_resetting_deadline(self): + result = wait([observation(), observation(), observation(io="pressured"), + observation(), observation(), observation()]) + self.assertEqual((result["status"], result["samples"], result["elapsed_ms"]), ("ready", 6, 5000)) + result = wait(sampler=lambda: observation(io="pressured")) + self.assertEqual((result["status"], result["samples"], result["consecutive"], result["elapsed_ms"]), + ("timeout", 30, 0, 30000)) + + def test_boundary_final_three_need_to_finish_before_absolute_deadline(self): + result = wait([observation(io="pressured")] * 27 + [observation()] * 3) + self.assertEqual((result["status"], result["elapsed_ms"]), ("ready", 29000)) + result = wait([observation(io="pressured")] * 28 + [observation()] * 2) + self.assertEqual((result["status"], result["consecutive"], result["elapsed_ms"]), ("timeout", 2, 30000)) + clock = Clock() + calls = 0 + + def sampler(): + nonlocal calls + calls += 1 + if calls == 3: + clock.now = 30_000_000_000 + return observation() + + result = wait(clock=clock, sampler=sampler) + self.assertEqual((result["status"], result["samples"], result["consecutive"]), ("timeout", 3, 2)) + + def test_slow_sample_and_sleep_overrun_do_not_extend_deadline_or_fabricate_readiness(self): + clock = Clock() + + def slow(): + clock.now += 31_000_000_000 + return observation() + + result = wait(clock=clock, sampler=slow) + self.assertEqual((result["status"], result["samples"], result["elapsed_ms"]), ("timeout", 1, 31000)) + self.assertEqual(clock.sleeps, []) + clock = Clock() + clock.sleep = lambda seconds: setattr(clock, "now", 31_000_000_000) + result = wait(clock=clock) + self.assertEqual((result["status"], result["samples"], result["elapsed_ms"]), ("timeout", 1, 31000)) + + def test_early_sleep_return_does_not_count_early_samples(self): + clock = Clock() + clock.sleep = lambda seconds: setattr(clock, "now", clock.now + min(250_000_000, round(seconds * 1e9))) + result = wait(clock=clock) + self.assertEqual((result["samples"], result["elapsed_ms"]), (3, 2000)) + + def test_unavailable_fails_immediately_and_is_not_idle(self): + clock = Clock() + result = wait([observation(), observation(cpu="unavailable")], clock=clock) + self.assertEqual((result["status"], result["samples"], result["consecutive"]), ("unavailable", 2, 0)) + self.assertEqual(clock.sleeps, [1]) + seen = [] + + def missing(path): + seen.append(path.name) + raise FileNotFoundError("private path must never enter receipt") + + self.assertEqual(CHECK["observe"](missing), dict.fromkeys(CHECK["THRESHOLDS"], "unavailable")) + self.assertEqual(seen, ["cpu", "memory", "io"]) + + def test_malformed_nonfinite_and_missing_some_fields_fail_closed(self): + texts = ("", "full avg10=0", "some avg60=0", "some avg10=NaN", "some avg10=inf", + "some avg10=-1", "some avg10=100.1", "some avg10=1_0", "some avg10=", "some avg10=０") + for text in texts: + with self.subTest(text=text): + self.assertEqual(CHECK["observe"](lambda _: text), + dict.fromkeys(CHECK["THRESHOLDS"], "unavailable")) + self.assertEqual(CHECK["average"]("full avg10=99\nsome avg10=0.25 avg60=90\n"), 0.25) + + def test_bounded_descriptor_read_and_no_symlink_fifo_directory_or_bad_utf8(self): + source = self.work / "source" + source.write_bytes(b"some avg10=0\n" + b" " * (CHECK["MAX_SOURCE_BYTES"] - 13)) + self.assertEqual(len(CHECK["read_bounded"](source)), CHECK["MAX_SOURCE_BYTES"]) + source.write_bytes(source.read_bytes() + b"x") + with self.assertRaises(ValueError): + CHECK["read_bounded"](source) + link = self.work / "link" + link.symlink_to(source) + fifo = self.work / "fifo" + os.mkfifo(fifo) + for path in (link, fifo, self.work): + with self.subTest(path=path.name), self.assertRaises((OSError, ValueError)): + CHECK["read_bounded"](path) + source.write_bytes(b"\xff") + with self.assertRaises(UnicodeError): + CHECK["read_bounded"](source) + + def test_zero_size_proc_like_file_still_has_actual_byte_bound_and_always_closes(self): + source = self.work / "source" + source.write_bytes(b"") + actual_open, actual_close = os.open, os.close + descriptors = [] + closed = [] + + def opening(*args, **kwargs): + self.assertTrue(args[1] & os.O_NOFOLLOW) + self.assertTrue(args[1] & os.O_NONBLOCK) + descriptor = actual_open(*args, **kwargs) + descriptors.append(descriptor) + return descriptor + + def closing(descriptor): + closed.append(descriptor) + actual_close(descriptor) + + with patch.object(os, "open", side_effect=opening), patch.object(os, "close", side_effect=closing): + with patch.object(os, "read", return_value=b"x" * (CHECK["MAX_SOURCE_BYTES"] + 1)) as read: + with self.assertRaises(ValueError): + CHECK["read_bounded"](source) + read.assert_called_once_with(descriptors[-1], CHECK["MAX_SOURCE_BYTES"] + 1) + with patch.object(os, "read", side_effect=OSError("private error")): + with self.assertRaises(OSError): + CHECK["read_bounded"](source) + self.assertEqual(descriptors, closed) + + def test_each_outcome_is_closed_bounded_private_retained_and_never_overwritten(self): + for phase, expected in (("native", wait()), ("https", wait(phase="https", sampler=lambda: observation(io="pressured")))): + self.assertEqual(CHECK["run"](self.work, phase, waiter=lambda _: expected), expected["status"] == "ready") + output = self.work / CHECK["OUTPUTS"][phase] + self.assertEqual(json.loads(output.read_bytes()), expected) + self.assertEqual(output.stat().st_mode & 0o777, 0o600) + self.assertLess(output.stat().st_size, 2048) + with self.assertRaises(ValueError): + CHECK["run"](self.work, phase, waiter=lambda _: self.fail("must reject before sampling")) + unavailable = wait(sampler=lambda: observation(io="unavailable")) + self.assertEqual(CHECK["validate"](unavailable)["status"], "unavailable") + + def test_unsafe_output_targets_and_directory_fail_without_sampling(self): + target = self.work / CHECK["OUTPUTS"]["native"] + target.symlink_to(self.work / "absent") + with self.assertRaises(ValueError): + CHECK["run"](self.work, "native", waiter=lambda _: self.fail("unsafe target")) + self.work.chmod(0o777) + try: + with self.assertRaises(ValueError): + CHECK["run"](self.work, "https", waiter=lambda _: self.fail("unsafe directory")) + finally: + self.work.chmod(0o700) + + def test_schema_rejects_forged_ready_free_form_fields_bool_counts_and_expanded_authority(self): + original = wait() + for changes in ({"extra": "private"}, {"samples": True}, {"samples": 31}, {"schema_version": True}, + {"elapsed_ms": 30000}, {"elapsed_ms": 1999}, {"consecutive": 2}, + {"phase": "unknown"}, {"last_observation": observation(io="pressured")}, + {"scope": dict(CHECK["SCOPE"], worker_capacity_proven=True)}): + with self.subTest(changes=changes), self.assertRaises(ValueError): + CHECK["validate"](dict(original, **changes)) + value = copy.deepcopy(original) + value["scope"]["memory_capacity_checked"] = 0 + with self.assertRaises(ValueError): + CHECK["validate"](value) + + def test_cli_invalid_input_does_not_export_raw_paths_or_exceptions(self): + result = subprocess.run([sys.executable, "-B", str(HELPER), str(self.work / "private"), "invalid"], + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, b"") + self.assertEqual(result.stderr, b"CONTENT_PROVIDER_RESOURCE_READINESS_INVALID\n") + + def test_shell_places_both_gates_after_setup_before_captures_and_retains_exports(self): + shell = (HERE / "content-provider-adaptive-smoke.sh").read_text() + native = shell[shell.index("content_provider_adaptive_run() {"):] + https = shell[shell.index("content_provider_adaptive_https_run() {"):shell.index("content_provider_adaptive_run() {")] + self.assertLess(native.index("capture_product_logs"), native.index('"$WORK" native')) + self.assertLess(native.index('"$WORK" native'), native.index("start_privacy_observers")) + self.assertLess(https.index('"$ah_user/origin.pem"'), https.index('"$WORK" https')) + self.assertLess(https.index('"$WORK" https'), https.index("start_privacy_observers")) + self.assertIn("|| fail CONTENT_PROVIDER_ADAPTIVE_RESOURCE_UNAVAILABLE", native) + self.assertIn("|| fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_RESOURCE_UNAVAILABLE", https) + runner = (HERE / "run-alpha-topology-vm.sh").read_text() + safe_names = runner[runner.index("SAFE_NAMES ="):runner.index("def ", runner.index("SAFE_NAMES ="))] + for name in CHECK["OUTPUTS"].values(): + self.assertIn(f'"{name}"', safe_names) + topology = (HERE / "kvm-alpha-topology.sh").read_text() + provider_preflight = topology[topology.index("for provider_fixture in "):] + provider_preflight = provider_preflight[:provider_preflight.index("; do")] + self.assertIn("content-provider-resource-ready.py", provider_preflight) + + +if __name__ == "__main__": + unittest.main() From a5e698ccb20c8d4a5288ca9d49e2286b1ff37019 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:07:40 +0200 Subject: [PATCH 30/49] docs: record Weather lifecycle milestone and resource-gated trial --- docs/IMPLEMENTATION_STATUS.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index a4f2b010c..fa4956ec2 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -31,7 +31,7 @@ The immediate milestones build on the retained candidates and evidence below: | --- | --- | --- | | Continue shared cache delivery — `volparossa`, `volparossa-weather` | Resolve the observed fixture I/O pressure, prove concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `7ad108c0` reconstructs native and HTTPS adaptive objects sequentially. Both budget reports identify I/O pressure as the sole pressure/unavailability flag, holding admission at one worker despite memory and FD headroom. Identify and reduce the actual storage bottleneck or establish bounded readiness; do not override pressure protection or weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather` | Deliver core-cache products into the actual client, then verify native packaging. | The pinned web build, sixteen Flutter tests and actual isolated browser interaction with synthetic products pass. Core delivery and native Android/iOS execution remain unverified. Cover the client's shorter timeout and the bridge's cancellation cleanup when testing cold retrieval; reuse the shared provider without another scheduler. | +| Extend the same provider to Overmorrow — `volparossa-weather` | Deliver core-cache products into the actual client, then verify native packaging. | PR8 aligns client/bridge deadlines, bounds busy retries and preserves cancellation cleanup. The pinned web build, twenty-three Flutter tests and actual isolated browser interaction with synthetic products pass. Cold core delivery and native Android/iOS execution remain unverified; reuse the shared provider without another scheduler. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the standalone @@ -96,6 +96,12 @@ eight adaptive-evidence and eighteen runner checks. Bounded native/HTTPS receipt not prove memory, FD or cgroup capacity. This adds no product delay or admission override: cache writes may reintroduce I/O pressure, and the live worker budget and three-provider packet-overlap requirements still decide the result. +Its first launch attempt stops before VM boot at the unchanged memory preflight: +the four-GiB guest plus two-GiB development-host reserve do not fit. The candidate +is committed as `fa89523b0c49768d7c56c58f2cec3e0c902dccbd`; retry only when that +capacity is genuinely available, without competing heavy jobs or reduced limits. +The retained runner log is +`.git/ci-evidence/content-provider-readiness-01.X2JNlh/runner.log`. The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first @@ -135,7 +141,19 @@ The VM and owned temporary overlay are removed; development-host DNS and IPv4/IPv6 route hashes match their pre-run values. A development-host firewall comparison remains unavailable. No weather origin was queried by this run. -Weather [PR7](https://github.com/VOLPAROSSA/volparossa-weather/pull/7), merged into +Weather [PR8](https://github.com/VOLPAROSSA/volparossa-weather/pull/8) is merged into +`main` at `ce1e28696d47808bd2908242325d1da478cbf0a1`. Each Overmorrow product now +has one 195-second deadline including local abort/drain, with up to five bounded +busy-only retries. Request and response-body cancellation are tested; unconfirmed +core-process cleanup keeps admission closed, even if scratch cleanup also fails. +Twenty-three Flutter tests, fifty-four focused Node checks and thirty-seven Python +harness checks pass. The new release build and actual isolated CanvasKit/browser +trial pass with synthetic products and complete cleanup. The report hash is +`dd0f4ae08d15eccb655d4c761593a1028a88277c5d53dc75fb09a77a0136aa71`; +core/peer delivery and native mobile execution remain open. CI and security checks +pass; the merged temporary branch is removed without deleting its worktree. + +The preceding Weather [PR7](https://github.com/VOLPAROSSA/volparossa-weather/pull/7), merged into `main` at `44b721fd42c86880f5a8838ae18efd2c8b2dd0a1`, includes the Overmorrow browser milestone. It connects the actual pinned Flutter app's provider dispatcher, phone/tablet layouts, forecast surface and air-quality detail route to the same local HTTP bridge. From dbbb1973b528fb6964043abcea2f0e7850057666 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:34:56 +0200 Subject: [PATCH 31/49] test: add exact Overmorrow profile to joined weather delivery --- docs/IMPLEMENTATION_STATUS.md | 25 +- tests/integration/run-alpha-topology-vm.sh | 2 +- .../integration/test-weather-client-bundle.py | 214 +++++++++++++++++- .../test-weather-products-delivery-shell.py | 33 ++- .../test-weather-products-delivery.py | 84 ++++++- tests/integration/weather-client-bundle.py | 201 ++++++++++++++-- tests/integration/weather-client-delivery.sh | 4 +- .../integration/weather-products-delivery.py | 59 +++-- .../integration/weather-products-delivery.sh | 14 +- 9 files changed, 583 insertions(+), 53 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index fa4956ec2..4c2e62514 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -31,7 +31,7 @@ The immediate milestones build on the retained candidates and evidence below: | --- | --- | --- | | Continue shared cache delivery — `volparossa`, `volparossa-weather` | Resolve the observed fixture I/O pressure, prove concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `7ad108c0` reconstructs native and HTTPS adaptive objects sequentially. Both budget reports identify I/O pressure as the sole pressure/unavailability flag, holding admission at one worker despite memory and FD headroom. Identify and reduce the actual storage bottleneck or establish bounded readiness; do not override pressure protection or weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather` | Deliver core-cache products into the actual client, then verify native packaging. | PR8 aligns client/bridge deadlines, bounds busy retries and preserves cancellation cleanup. The pinned web build, twenty-three Flutter tests and actual isolated browser interaction with synthetic products pass. Cold core delivery and native Android/iOS execution remain unverified; reuse the shared provider without another scheduler. | +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR8 aligns client/bridge deadlines and cancellation. The new closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows; component tests pass. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the standalone @@ -103,6 +103,14 @@ capacity is genuinely available, without competing heavy jobs or reduced limits. The retained runner log is `.git/ci-evidence/content-provider-readiness-01.X2JNlh/runner.log`. +The following readiness attempt uses clean source +`a5e698ccb20c8d4a5288ca9d49e2286b1ff37019` in a separate worktree, retaining +the same guest and host-memory limits. It starts after capacity becomes +available; its private evidence directory is +`.git/ci-evidence/content-provider-readiness-02.WgVnio`. +No result is claimed while the trial runs. Overmorrow changes are excluded +from that snapshot, and competing heavy builds remain deferred. + The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first completed, single-offer discovery. A temporary provider-side gate is bounded to @@ -141,6 +149,21 @@ The VM and owned temporary overlay are removed; development-host DNS and IPv4/IPv6 route hashes match their pre-run values. A development-host firewall comparison remains unavailable. No weather origin was queried by this run. +The next joined Weather candidate adds immutable bundle profile 3 for Overmorrow, +without changing Drizz.li profiles 1 and 2. It reuses actual source acquisition, +separate public weather/particulate publications and independent protected cold +transfers. The production bridge serves the existing CanvasKit build; the new +driver checks native timestamps and values in their actual ordered table cells, +precipitation windows, per-product attribution and a real pointer refresh. +Thirty bundle, twenty-six product-evidence, twelve shell and eighteen runner +checks pass independently, alongside three Weather preparation and ten browser +driver checks. The current synthetic UI/build proof binds the bundled assets; +older ECMWF/NOAA receipts remain historical and cannot attest changed acquisition +code. Real origin acquisition and joined UI delivery still require the live +trial. A prior cold core fetch followed by UI use is not browser-triggered +first-cold delivery. No SDK rebuild, renderer CDN, new scheduler, resource-limit +increase or extra source request is introduced by this candidate. + Weather [PR8](https://github.com/VOLPAROSSA/volparossa-weather/pull/8) is merged into `main` at `ce1e28696d47808bd2908242325d1da478cbf0a1`. Each Overmorrow product now has one 195-second deadline including local abort/drain, with up to five bounded diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 707e25dad..f7defc87a 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -828,7 +828,7 @@ if [ -n "$weather_client_bundle" ]; then python3 -B "$HERE/weather-client-bundle.py" verify-runtime "$WEATHER_CLIENT_ARCHIVE" "$weather_client_manifest_sha256" WEATHER_CLIENT_ARCHIVE_SHA256=$(sha256sum "$WEATHER_CLIENT_ARCHIVE" | awk '{ print $1 }') WEATHER_CLIENT_MANIFEST_SHA256=$weather_client_manifest_sha256 - printf '%s\n' 'Additional proof: public model HTTPS acquisition (ECMWF, plus NOAA for the two-product profile), protected peer transfer and real Drizz.li UI; origin acquisition is not an overlay claim.' + printf '%s\n' 'Additional proof: public model HTTPS acquisition (ECMWF, plus NOAA for the two-product profiles), protected peer transfer and the exact bundled Drizz.li or Overmorrow UI; origin acquisition is not an overlay claim.' fi CODE_ARCHIVE_SHA256=none CODE_MANIFEST_SHA256=none diff --git a/tests/integration/test-weather-client-bundle.py b/tests/integration/test-weather-client-bundle.py index bcef56dae..57c5eadaa 100644 --- a/tests/integration/test-weather-client-bundle.py +++ b/tests/integration/test-weather-client-bundle.py @@ -31,8 +31,11 @@ def setUp(self): self.files = self.profile_files() def profile_files(self, version=1): + fixed = W[{1: "FIXED", 2: "FIXED_V2", 3: "FIXED_V3"}[version]] + static = ({W["STATIC_V3"] + name for name in W["STATIC_REQUIRED_V3"]} if version == 3 + else {W["STATIC"] + "index.html"}) return {name: dict(bytes=len(self.raw), mode=W["mode"](name), sha256=self.digest) - for name in W["FIXED" if version == 1 else "FIXED_V2"] | {W["STATIC"] + "index.html"}} + for name in fixed | static} def manifest(self, value=None, version=1): raw = W["encode"](value or dict(version=version, kind=W["KIND"], files=self.files)) @@ -76,7 +79,7 @@ def test_v1_runtime_gate_preserves_existing_archive_and_schema(self): self.assertIn(W["WEATHER"] + "integrations/drizzli/browser-smoke.json", self.files) def test_runtime_version_action_checks_whole_archive_without_executing_or_changing_profiles(self): - for version in (1, 2): + for version in (1, 2, 3): self.files = self.profile_files(version) archive, expected = self.archive(version=version) before = archive.read_bytes() @@ -141,6 +144,213 @@ def test_v2_roundtrip_is_readonly_and_never_executes_captured_inputs(self): self.assertTrue(all(path.stat().st_mode & 0o777 == 0o500 for path in (destination, *destination.rglob("*")) if path.is_dir())) + def test_v3_roundtrip_and_runtime_gate_preserve_original_archive_and_closed_files(self): + self.files = self.profile_files(3) + archive, expected = self.archive(version=3) + before = archive.read_bytes() + destination = self.root / "v3-tree" + checked = W["checked_archive"](archive, expected, destination, runtime=True, report_version=True) + self.assertEqual(checked["bundle_version"], 3) + self.assertEqual(W["verify_tree"](destination, expected)["files"], len(self.files)) + self.assertEqual(archive.read_bytes(), before) + self.assertEqual(json.loads((destination / "INPUTS.json").read_bytes())["version"], 3) + for name, item in self.files.items(): + self.assertEqual((destination / name).read_bytes(), self.raw) + self.assertEqual((destination / name).stat().st_mode & 0o777, item["mode"]) + self.assertIn(W["WEATHER"] + "integrations/overmorrow/prepare_core_browser.mjs", self.files) + self.assertIn(W["WEATHER"] + "integrations/drizzli/prepare-core-browser.mjs", self.files) + self.assertFalse(any(name.startswith(W["UI_STAGE"]) for name in self.files)) + + def test_v3_static_closure_and_limits_do_not_spill_into_legacy_profiles(self): + for version in (1, 2, 3): + base = self.profile_files(version) + wrong_prefix = W["STATIC"] if version == 3 else W["STATIC_V3"] + files = {**base, wrong_prefix + "index.html": dict(bytes=5, mode=0o400, sha256=self.digest)} + raw, expected = self.manifest(dict(version=version, kind=W["KIND"], files=files)) + with self.subTest(version=version), self.assertRaises(ValueError): W["manifest"](raw, expected) + for bad in ("flutter_service_worker.js", "canvaskit/skwasm.wasm", "canvaskit/canvaskit.js.symbols", + "assets/worker.js", "extra.html", "../escape", ".hidden"): + files = self.profile_files(3) + files[W["STATIC_V3"] + bad] = dict(bytes=10, mode=0o400, sha256=self.digest) + raw, expected = self.manifest(dict(version=3, kind=W["KIND"], files=files)) + with self.subTest(bad=bad), self.assertRaises(ValueError): W["manifest"](raw, expected) + for missing in W["FIXED_V3"] | {W["STATIC_V3"] + name for name in W["STATIC_REQUIRED_V3"]}: + files = self.profile_files(3); del files[missing] + raw, expected = self.manifest(dict(version=3, kind=W["KIND"], files=files)) + with self.subTest(missing=missing), self.assertRaises(ValueError): W["manifest"](raw, expected) + files = self.profile_files(3) + for name in ("assets/font.otf", "assets/shaders/ink_sparkle.frag", "assets/AssetManifest.bin.json"): + files[W["STATIC_V3"] + name] = dict(bytes=32, mode=0o400, sha256=self.digest) + raw, expected = self.manifest(dict(version=3, kind=W["KIND"], files=files)) + W["manifest"](raw, expected) + files[W["STATIC_V3"] + "canvaskit/canvaskit.wasm"]["bytes"] = 8 * 1024**2 + 1 + raw, expected = self.manifest(dict(version=3, kind=W["KIND"], files=files)) + with self.assertRaises(ValueError): W["manifest"](raw, expected) + files = self.profile_files(3) + for index in range(5): + files[W["STATIC_V3"] + f"assets/large{index}.bin"] = dict(bytes=8*1024**2, mode=0o400, sha256=self.digest) + raw, expected = self.manifest(dict(version=3, kind=W["KIND"], files=files)) + with self.assertRaises(ValueError): W["manifest"](raw, expected) + + def v3_proof_fixture(self): + _, records = self.proof_fixture(2) + selected = self.profile_files(3) + hashes = {name[len(W["STATIC_V3"]):]: item["sha256"] for name, item in selected.items() + if name.startswith(W["STATIC_V3"])} + tree = hashlib.sha256(json.dumps(hashes, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + source = dict(upstream="b" * 40, tree="c" * 40, patch_sha256=self.digest, + overlay_sha256=dict.fromkeys(W["OVERMORROW_OVERLAY"], self.digest), + deployed_tree_sha256=tree, deployed_files=len(hashes), deployed_bytes=len(hashes) * len(self.raw), + bridge_modules=dict.fromkeys(W["OVERMORROW_BRIDGE"], self.digest), + fixture_modules=dict.fromkeys(W["OVERMORROW_FIXTURE"], self.digest), + browser_helpers=dict.fromkeys(W["PROVEN_BROWSER_HELPERS"], self.digest), + isolation_helper_sha256=self.digest, driver_sha256=self.digest, fixture_sha256=self.digest) + proof = dict(schema=1, passed=True, phase="complete", wrapper_outcome="completed", source=source, + runtime=dict(sha256=dict.fromkeys(W["PROVEN_RUNTIME_FILES"], self.digest), node_sha256=self.digest), + scope=dict(actual_overmorrow_ui=True, actual_http_bridge=True, synthetic_weather=True, + synthetic_air_quality=True, core_executed=False, overlay_proven=False, + upstream_origin_proven=False, native_mobile_proven=False), + cleanup=dict.fromkeys(("browser_exited", "fixture_exited", "private_profile_removed", + "normal_browser_stop", "normal_fixture_stop"), True), + outer_cleanup=dict.fromkeys(("namespace_wrapper_exited", "private_profile_removed", "host_netns_unchanged"), True), + checks=dict.fromkeys(("actual_canvas_and_semantics", "independent_native_timelines", "native_values_visible", + "no_fabricated_current_or_aqi", "fixed_public_area", "source_attribution", "actual_refresh", + "weather_failure_and_stale_preserve_air_quality", "weather_idle_expiry_without_request", + "air_quality_failure_and_stale_preserve_weather", "air_quality_idle_expiry_without_request", + "existing_aq_detail_without_new_request", "same_origin_loaded_resources"), True)) + proof["checks"].update(provider_requests=10, air_quality_provider_requests=10, + initial_semantics_sha256=self.digest, final_semantics_sha256=self.digest) + records["browser-proof.json"] = proof + records["upstream.json"] = dict(commit="b" * 40, tree="c" * 40, license_sha256=self.digest, + pubspec_lock_sha256=self.digest, upstream_patch_sha256=self.digest, flutter="3.38.5", flutter_commit="d" * 40) + records["build-verification.json"] = dict(schema=1, upstream_commit="b" * 40, + upstream_patch_sha256=self.digest, pubspec_lock_sha256=self.digest, + profile="VOLPAROSSA_WEATHER=true", flutter="3.38.5", flutter_commit="d" * 40, + actual_release_web_build=True, synthetic_products=True, browser_interaction_verified=True, + core_delivery_verified=False, native_android_ios_verified=False, forecast_accuracy_verified=False, + deployment=dict(tree_sha256=tree, files=len(hashes), bytes=len(hashes) * len(self.raw), + static_profile="flutter-canvaskit", main_js_sha256=self.digest, bootstrap_sha256=self.digest, + notices_sha256=self.digest), + browser_trial=dict(passed=True, report_sha256=self.digest), + request_lifecycle=dict(provider_source_sha256=self.digest, cold_core_delivery_verified=False)) + # Historical origin proof intentionally predates current wrapper changes. + records["noaa-gefs-smoke.json"]["source_modules"] = dict.fromkeys(W["NOAA_MODULES"], "e" * 64) + return selected, records + + def check_v3_proof(self, selected, records): + with patch.dict(W["overmorrow_source_proof"].__globals__, small_json=lambda path: records[path.name]): + W["overmorrow_source_proof"](Path("/inert/weather"), Path("/inert/firefox"), + selected, Path("/inert/browser-proof.json")) + + def test_v3_current_ui_provenance_binds_exact_tree_sources_runtime_and_licenses(self): + self.check_v3_proof(*self.v3_proof_fixture()) + paths = {*(W["STATIC_V3"] + name for name in W["STATIC_REQUIRED_V3"]), + *(W["WEATHER"] + name for name in W["OVERMORROW_BRIDGE"] | W["OVERMORROW_FIXTURE"]), + *(W["BROWSER"] + name for name in W["PROVEN_BROWSER_HELPERS"]), + *(W["STAGE"] + name for name in W["PROVEN_RUNTIME_FILES"] | {"volparossa.cfg"}), + *(W["UI_STAGE_V3"] + name for name in W["CLIENT_FILES_V3"]), + *(W["WEATHER"] + "integrations/overmorrow/overlay/" + name for name in W["OVERMORROW_OVERLAY"]), + W["WEATHER"] + W["OVERMORROW_PROOF"], W["WEATHER"] + "LICENSE", + *(W["WEATHER"] + name for name in ("integrations/overmorrow/upstream.patch", + "integrations/overmorrow/smoke_browser.py", "integrations/overmorrow/browser-fixture.mjs", + "integrations/drizzli/smoke_browser.py"))} + for name in paths: + selected, records = self.v3_proof_fixture(); selected[name]["sha256"] = "f" * 64 + with self.subTest(name=name), self.assertRaises(ValueError): self.check_v3_proof(selected, records) + + def test_v3_module_maps_cannot_be_truncated_or_expanded(self): + for group in ("overlay_sha256", "bridge_modules", "fixture_modules", "browser_helpers"): + for fault in ("empty", "missing", "extra", "invalid"): + selected, records = self.v3_proof_fixture() + target = records["browser-proof.json"]["source"][group] + if fault == "empty": target.clear() + elif fault == "missing": target.pop(next(iter(target))) + elif fault == "extra": target["unreviewed"] = self.digest + else: target[next(iter(target))] = "invalid" + with self.subTest(group=group, fault=fault), self.assertRaises(ValueError): + self.check_v3_proof(selected, records) + + def test_v3_historical_origin_evidence_is_not_current_execution_authority(self): + selected, records = self.v3_proof_fixture() + self.check_v3_proof(selected, records) + self.assertNotEqual(records["noaa-gefs-smoke.json"]["source_modules"]["src/noaa-gefs.mjs"], + selected[W["WEATHER"] + "src/noaa-gefs.mjs"]["sha256"]) + for filename, path in (("noaa-gefs-smoke.json", ("scope", "core_executed")), + ("noaa-gefs-smoke.json", ("scope", "overlay_proven")), + ("ecmwf-precipitation-smoke.json", ("core_overlay_roundtrip_proven",)), + ("browser-proof.json", ("scope", "upstream_origin_proven")), + ("build-verification.json", ("core_delivery_verified",))): + selected, records = self.v3_proof_fixture(); parent = records[filename] + for key in path[:-1]: parent = parent[key] + parent[path[-1]] = True + with self.subTest(filename=filename, path=path), self.assertRaises(ValueError): + self.check_v3_proof(selected, records) + # This historical exception belongs only to v3: v2 still requires current hashes. + selected, records = self.proof_fixture(2) + records["noaa-gefs-smoke.json"]["source_modules"]["src/noaa-gefs.mjs"] = "e" * 64 + with self.assertRaises(ValueError): self.check_proof(selected, records, 2) + + def test_v3_exact_synthetic_scope_success_checks_and_cleanup_required(self): + _, original = self.v3_proof_fixture() + proof = original["browser-proof.json"] + paths = [("passed",), ("phase",), ("wrapper_outcome",), ("schema",)] + paths += [(group, key) for group in ("scope", "checks", "cleanup", "outer_cleanup") for key in proof[group]] + for path in paths: + selected, records = self.v3_proof_fixture(); parent = records["browser-proof.json"] + for key in path[:-1]: parent = parent[key] + old = parent[path[-1]] + parent[path[-1]] = not old if type(old) is bool else "invalid" + with self.subTest(path=path), self.assertRaises(ValueError): self.check_v3_proof(selected, records) + + def test_v3_capture_interface_is_explicit_and_cannot_replace_legacy_capture(self): + arguments = [str(self.root / str(index)) for index in range(8)] + for action, args in (("capture-v3", arguments), ("capture", arguments[:6]), ("capture-v2", arguments[:6])): + calls = [] + def record(*paths, **kwargs): + calls.append((paths, kwargs)); return {"captured": False} + with patch.dict(W["main"].__globals__, capture_v3=record, capture=record), \ + patch.object(W["sys"], "argv", ["bundle", action, *args]), redirect_stdout(io.StringIO()): + W["main"]() + self.assertEqual(calls, [(tuple(map(Path, args)), {} if action == "capture-v3" + else {"version": 2 if action == "capture-v2" else 1})]) + + def test_v3_actual_inert_capture_freezes_current_bytes_without_executing_them(self): + weather, app, static, browser, firefox = (self.root / name for name in + ("weather", "app", "static", "browser", "firefox")) + for root in (weather, app, static, browser, firefox): root.mkdir(mode=0o700) + sources = [(weather, W["WEATHER_FILES_V3"] - {W["OVERMORROW_PROOF"]}), + (app, W["CLIENT_FILES_V3"]), (static, W["STATIC_REQUIRED_V3"]), + (browser, W["BROWSER_FILES"]), (firefox, W["RUNTIME_FILES"])] + for root, names in sources: + for name in names: + path = root / name; path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + path.write_bytes(self.raw); path.chmod(0o600) + for name in W["EXECUTABLES"]: + if name.startswith(W["STAGE"]): (firefox / name[len(W["STAGE"]):]).chmod(0o700) + node = self.root / "node"; node.write_bytes(self.raw); node.chmod(0o700) + decoder = weather / "build/eccodes/eccodes-build/bin/grib_ls" + decoder.parent.mkdir(parents=True, mode=0o700); decoder.write_bytes(self.raw); decoder.chmod(0o700) + proof = self.root / "retained-proof.json"; proof.write_bytes(self.raw); proof.chmod(0o600) + for directory in self.root.rglob("*"): + if directory.is_dir(): directory.chmod(0o700) + archive = self.root / "captured.tar" + proof_calls = [] + with patch.dict(W["capture_v3"].__globals__, overmorrow_source_proof=lambda *args: proof_calls.append(args)): + result = W["capture_v3"](weather, app, static, browser, firefox, node, proof, archive) + self.assertEqual(len(proof_calls), 1) + self.assertEqual(proof_calls[0][:2], (weather, firefox)) + self.assertEqual(proof_calls[0][3], proof) + self.assertTrue(result["historical_source_proofs_only"]) + self.assertFalse(result["current_origin_acquisition_proven"]) + self.assertFalse(result["joined_delivery_proven"]) + expected = result["manifest_sha256"] + self.assertEqual(W["checked_archive"](archive, expected, runtime=True, report_version=True)["bundle_version"], 3) + frozen = self.root / "frozen" + W["checked_archive"](archive, expected, frozen) + self.assertEqual((frozen / W["WEATHER"] / W["OVERMORROW_PROOF"]).read_bytes(), self.raw) + self.assertEqual(proof.read_bytes(), self.raw, "historical original must remain untouched") + self.assertEqual(W["verify_tree"](frozen, expected)["runtime_executed"], False) + def test_v2_runtime_gate_checks_complete_archive_and_preserves_existing_roots(self): self.files = self.profile_files(2) archive, expected = self.archive(version=2) diff --git a/tests/integration/test-weather-products-delivery-shell.py b/tests/integration/test-weather-products-delivery-shell.py index 8885484b4..5027b1f8a 100644 --- a/tests/integration/test-weather-products-delivery-shell.py +++ b/tests/integration/test-weather-products-delivery-shell.py @@ -160,8 +160,8 @@ def test_verified_closed_dispatch_before_any_v1_work(self): helper_root = self.root / "tests/integration" helper_root.mkdir(parents=True) (helper_root / "weather-products-delivery.sh").write_text( - 'weather_products_delivery_run() { printf "v2\\n"; }\n') - for version in (1, 2, 3, "2"): + 'weather_products_delivery_run() { printf "v%s\\n" "$weather_profile_version"; }\n') + for version in (1, 2, 3, 4, "2", "3", True): with self.subTest(version=version, value_type=type(version)): import json value = json.dumps({"bundle_version": version}) @@ -174,7 +174,7 @@ def test_verified_closed_dispatch_before_any_v1_work(self): fail() { exit 77; } run_selected '''.replace("JSON", value), source=source + "\nrun_selected() {\n" + head + '\nprintf "v1\\n"\n}\n') - self.assertEqual(result.returncode, 0 if type(version) is int and version in (1, 2) else 77) + self.assertEqual(result.returncode, 0 if type(version) is int and version in (1, 2, 3) else 77) if result.returncode == 0: self.assertEqual(result.stdout, f"v{version}\n") self.assertLess(head.index("weather_client_runtime_profile"), head.index(".bundle_version")) self.assertLess(head.index(".bundle_version"), head.index(' . "$source_directory')) @@ -185,12 +185,12 @@ def test_profile_owner_boundary_and_helper_failure_are_fail_closed(self): helper_root = self.root / "tests/integration" helper_root.mkdir(parents=True) (helper_root / "weather-products-delivery.sh").write_text( - 'weather_products_delivery_run() { printf "v2\\n"; }\n') + 'weather_products_delivery_run() { printf "v%s\\n" "$weather_profile_version"; }\n') scenarios = (("root", 0, 0, 0), ("vpci", 1000, 0, 0), ("other", 985, 0, 77), ("stat-failure", 1000, 0, 77), ("missing-account", 1000, 0, 77), ("runuser-failure", 1000, 0, 77), ("helper-failure", 0, 1, 77), ("helper-failure", 1000, 1, 77)) - for version in (1, 2): + for version in (1, 2, 3): for scenario, owner, helper_status, expected_status in scenarios: with self.subTest(version=version, scenario=scenario, owner=owner): result = self.call(''' @@ -248,6 +248,29 @@ def test_profile_owner_boundary_and_helper_failure_are_fail_closed(self): self.assertEqual(observed, expected) if calls.exists(): calls.unlink() + def test_two_product_driver_paths_are_closed_and_keep_the_same_deadline(self): + source = SOURCE.read_text() + head = source.split("weather_products_delivery_run() {", 1)[1].split( + " PHASE=weather-products-delivery-prepare", 1)[0] + for version in ("2", "3", "1", "4", "", "../../../arbitrary"): + with self.subTest(version=version): + result = self.call(''' +weather_client_bundle=inert +weather_client_manifest_sha256=inert +weather_profile_version='VERSION' +fail() { exit 77; } +run_selected +'''.replace("VERSION", version), source=source + "\nrun_selected() {\n" + head + + '\nprintf "%s\\n" "$weather_products_driver"\n}\n') + if version in ("2", "3"): + self.ok(result) + consumer = "drizzli" if version == "2" else "overmorrow" + self.assertEqual(result.stdout.strip(), f"integrations/{consumer}/smoke_core_browser.py") + else: self.assertEqual(result.returncode, 77) + self.assertIn('"$WORK/state-client/air-quality-cache" "$weather_profile_version"', source) + self.assertIn('python3 -B "$weather_client_directory/volparossa-weather/$weather_products_driver"', source) + self.assertEqual(source.count('weather_products_remaining 270'), 1) + def test_publication_cleanup_independent_captures_and_retained_proof_order(self): source = SOURCE.read_text() run = source.split("weather_products_delivery_run() {", 1)[1] diff --git a/tests/integration/test-weather-products-delivery.py b/tests/integration/test-weather-products-delivery.py index 7fe17f61b..4183f46e1 100644 --- a/tests/integration/test-weather-products-delivery.py +++ b/tests/integration/test-weather-products-delivery.py @@ -88,7 +88,7 @@ def failure_fixture(work, raw=None): return marker -def report(): +def report(version=2): legacy = OLD["report"](); expected = legacy["expected"] result = dict(schema=2, passed=True, phase="complete", source=legacy["source"], expected={key: expected[key] for key in ("expected_sha256", "core_sha256", "display_sha256")}, @@ -101,6 +101,9 @@ def report(): publication = selected()["products"][key]["publication"] result["products"][key] = dict(expected=dict(cold_receipt_sha256=SHA, display_sha256=SHA, forecast_sha256=publication["sha256"], manifest_id=publication["manifest_id"]), checks=copy.deepcopy(legacy["checks"])) + if version == 3: + result["schema"] = 3 + result["scope"]["actual_overmorrow_ui"] = result["scope"].pop("actual_drizzli_ui") return result @@ -108,7 +111,7 @@ def save(path, value, mode=0o600): A["write_new"](path, json.dumps(value).encode(), mode) -def joined_fixture(work): +def joined_fixture(work, version=2): f = OLD["FILTER"]["fixture"]() selection = selected() common = {"input": selection, "acquisition": acquisition(), @@ -142,17 +145,22 @@ def joined_fixture(work): files[node + "-serve"] = dict(f["serves"][node], publications=4 if key == "weather" else 5) for suffix, value in files.items(): save(A["artifact"](work, suffix, key), value) B = A["B"] - items = {name: dict(bytes=1, mode=B["mode"](name), sha256=SHA) for name in B["FIXED_V2"] | {B["STATIC"]+"index.html"}} + profile = A["ui_profile"](version) + fixed = B["FIXED_V2"] if version == 2 else B["FIXED_V3"] + static_files = {"index.html"} if version == 2 else B["STATIC_REQUIRED_V3"] + items = {name: dict(bytes=1, mode=B["mode"](name), sha256=SHA) + for name in fixed | {profile["static"] + name for name in static_files}} for name, digest in (("runtime/node", B["NODE_SHA"]), ("decoder/grib_ls", B["DECODER_SHA"]), (B["WEATHER"]+"third_party/eccodes-sources.json", B["PIN_SHA"]), (B["WEATHER"]+"scripts/build_eccodes.py", B["BUILD_SHA"])): items[name]["sha256"] = digest - manifest = B["encode"](dict(version=2, kind=B["KIND"], files=items)); digest = hashlib.sha256(manifest).hexdigest() + manifest = B["encode"](dict(version=version, kind=B["KIND"], files=items)); digest = hashlib.sha256(manifest).hexdigest() A["write_new"](A["artifact"](work, "manifest"), manifest, 0o400) for phase in ("before", "after"): common["tree-"+phase] = dict(version=1, action="verify-tree", manifest_sha256=digest, files=len(items), runtime_executed=False) common["cache-"+phase] = dict(schema=2, products=caches) - browser = report(); browser["source"]["bundle_manifest_sha256"] = digest - browser["source"]["built_tree_sha256"] = hashlib.sha256(json.dumps({"index.html": SHA}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + browser = report(version); browser["source"]["bundle_manifest_sha256"] = digest + browser["source"]["built_tree_sha256"] = hashlib.sha256(json.dumps( + {name: SHA for name in static_files}, sort_keys=True, separators=(",", ":")).encode()).hexdigest() common["original-report"] = browser save(work / "a01-expected-peers.json", f["expected_peers"]); save(work / "content-provider-layout.json", f["layout"]) socket = work / "runtime-client/control/agent.sock"; socket.parent.mkdir(parents=True) @@ -168,6 +176,48 @@ def setUp(self): clock = patch.dict(A["prepare"].__globals__, observed_ms=lambda: CLOCK) clock.start(); self.addCleanup(clock.stop) + def test_profiles_and_browser_exports_never_accept_other_consumers(self): + for version, consumer in ((2, "drizzli"), (3, "overmorrow")): + good = report(version) + A["closed_report"](good) + self.assertEqual(A["ui_profile"](version)["ui"], "actual_"+consumer+"_ui") + for other in (1, 5, "3", True, 5-version): + bad = copy.deepcopy(good); bad["schema"] = other + with self.subTest(version=version, other=other), self.assertRaises(ValueError): A["closed_report"](bad) + bad = copy.deepcopy(good); bad["scope"]["arbitrary_ui"] = True + with self.assertRaises(ValueError): A["closed_report"](bad) + for version in (1, 4, "3", True, None): + with self.assertRaises(ValueError): A["ui_profile"](version) + + def test_overmorrow_full_join_is_bound_to_profile_three_and_cold_products(self): + mutations = [None, ("schema",), ("scope", "actual_overmorrow_ui"), + ("source", "driver_sha256"), ("source", "preparer_sha256"), ("source", "built_tree_sha256"), + ("products", "weather", "checks", "exact_native_times_and_values"), + ("products", "air_quality", "checks", "refresh_dom_display_sha256"), + ("products", "air_quality", "expected", "cold_receipt_sha256")] + for mutation in mutations: + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); digest, caches = joined_fixture(work, 3) + if mutation: + path = A["artifact"](work, "original-report"); value = A["read"](path); target = value + for key in mutation[:-1]: target = target[key] + target[mutation[-1]] = (2 if mutation == ("schema",) else + False if type(target[mutation[-1]]) is bool else "f"*64) + if mutation == ("schema",): + value["scope"]["actual_drizzli_ui"] = value["scope"].pop("actual_overmorrow_ui") + path.unlink(); save(path, value) + with patch.dict(A["evidence"].__globals__, cache_state=lambda key, *args: caches[key]): + if mutation: + with self.assertRaises(ValueError): A["evidence"](work, digest, SHA) + else: + proof = A["evidence"](work, digest, SHA); A["validate_evidence_export"](proof) + self.assertEqual(proof["schema"], 3) + self.assertTrue(proof["scope"]["actual_overmorrow_ui"]) + self.assertNotIn("actual_drizzli_ui", proof["scope"]) + self.assertFalse(proof["scope"]["browser_triggered_cold_fetch"]) + bad = copy.deepcopy(proof); bad["schema"] = 2 + with self.assertRaises(ValueError): A["validate_evidence_export"](bad) + def test_acquisition_diagnoses_every_stage_without_error_details(self): for failed_stage in A["ACQUISITION_STAGES"]: with self.subTest(stage=failed_stage), tempfile.TemporaryDirectory() as temporary: @@ -430,6 +480,28 @@ def execute(command, cwd, maximum, timeout): A["prepare"](base / "source", viewer, base / "binary", base / "agent.sock", base / "state-client/weather-cache", base / "state-client/air-quality-cache") + def test_overmorrow_preparer_selects_only_its_source_and_schema(self): + for expected_version in (2, 3, "3", True): + with self.subTest(expected_version=expected_version), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); viewer = base / "weather-viewer"; viewer.mkdir(mode=0o700) + selection, forecasts = prepare_fixture(viewer) + def execute(command, cwd, maximum, timeout): + self.assertEqual(command[2], str(base / "source" / A["B"]["WEATHER"] / + "integrations/overmorrow/prepare_core_browser.mjs")) + self.assertNotIn("drizzli", command[2]) + expected = dict(schema=expected_version, core_sha256=SHA, display_sha256=SHA, + products={key: dict(publication=value["publication"], forecast=forecasts[key]) + for key, value in selection["products"].items()}) + raw = A["B"]["encode"](expected); A["write_new"](viewer / "expected.json", raw, 0o400) + return json.dumps(dict(prepared=True, expected_sha256=hashlib.sha256(raw).hexdigest(), display_sha256=SHA)).encode() + with patch.dict(A["prepare"].__globals__, run=execute), patch.dict(A["B"], fingerprint=lambda *args: {"sha256": SHA}): + args = (base / "source", viewer, base / "binary", base / "agent.sock", + base / "state-client/weather-cache", base / "state-client/air-quality-cache", 3) + if type(expected_version) is int and expected_version == 3: + self.assertTrue(A["prepare"](*args)["prepared"]) + else: + with self.assertRaises(ValueError): A["prepare"](*args) + def test_partial_browser_report_is_retainable_but_never_success(self): failed = report(); failed.update(passed=False, phase="provider", checks={}, cleanup={}) failed["scope"].update(actual_drizzli_ui=False, real_core_adapter=False, separate_native_runs=False) diff --git a/tests/integration/weather-client-bundle.py b/tests/integration/weather-client-bundle.py index ab1b3fbfd..29bc6fb61 100644 --- a/tests/integration/weather-client-bundle.py +++ b/tests/integration/weather-client-bundle.py @@ -72,6 +72,33 @@ FIXED_V2 = frozenset({*(WEATHER + name for name in WEATHER_FILES_V2), *(BROWSER + name for name in BROWSER_FILES), *(STAGE + name for name in RUNTIME_FILES), *(UI_STAGE + name for name in CLIENT_FILES_V2), "runtime/node", "decoder/grib_ls"}) +# A distinct consumer profile, not a relaxation or reinterpretation of v1/v2. +UI_STAGE_V3 = WEATHER + "build/overmorrow-app/" +STATIC_V3 = WEATHER + "build/overmorrow-web/" +OVERMORROW_OVERLAY = frozenset(("lib/api_key.dart", "lib/volparossa/products.dart", + "lib/volparossa/profile.dart", "lib/volparossa/provider.dart", "lib/volparossa/views.dart", + "test/volparossa_products_test.dart", "test/volparossa_provider_test.dart", + "test/volparossa_ui_test.dart", "web/flutter_bootstrap.js")) +OVERMORROW_BRIDGE = PROVEN_BRIDGE_MODULES | PROVEN_WEATHER_MODULES | AQ_MODULES | {"src/air-quality-core.mjs"} +OVERMORROW_FIXTURE = FIXTURE_MODULES | {"integrations/overmorrow/test-fixtures.mjs", "integrations/drizzli/browser-fixture.mjs"} +OVERMORROW_PROOF = "integrations/overmorrow/browser-proof.json" +OVERMORROW_FILES = frozenset(("upstream.json", "upstream.patch", "build-verification.json", "deploy.py", + "smoke_browser.py", "smoke_core_browser.py", "prepare_core_browser.mjs", "browser-fixture.mjs", "test-fixtures.mjs")) +WEATHER_FILES_V3 = frozenset({*(name for name in WEATHER_FILES_V2 if not name.startswith("integrations/")), + *("integrations/drizzli/" + name for name in ("smoke_browser.py", "smoke_core_browser.py", + "prepare-core-browser.mjs", "browser-fixture.mjs")), + *("integrations/overmorrow/" + name for name in OVERMORROW_FILES), + *("integrations/overmorrow/overlay/" + name for name in OVERMORROW_OVERLAY), OVERMORROW_PROOF}) +CLIENT_FILES_V3 = OVERMORROW_OVERLAY | {"LICENSE", "LICENSE.VOLPAROSSA", "pubspec.lock"} +FIXED_V3 = frozenset({*(WEATHER + name for name in WEATHER_FILES_V3), + *(BROWSER + name for name in BROWSER_FILES), *(STAGE + name for name in RUNTIME_FILES), + *(UI_STAGE_V3 + name for name in CLIENT_FILES_V3), "runtime/node", "decoder/grib_ls"}) +STATIC_REQUIRED_V3 = frozenset(("index.html", "version.json", "flutter.js", "main.dart.js", "favicon.png", + "manifest.json", "flutter_bootstrap.js", "canvaskit/canvaskit.js", "canvaskit/canvaskit.wasm", + "canvaskit/chromium/canvaskit.js", "canvaskit/chromium/canvaskit.wasm", "assets/NOTICES", + "assets/AssetManifest.bin", "LICENSE.txt", "LICENSE.VOLPAROSSA.txt")) +STATIC_ASSET_EXTENSIONS_V3 = frozenset((".png", ".jpg", ".jpeg", ".webp", ".svg", ".ttf", ".otf", + ".woff", ".woff2", ".json", ".bin", ".frag")) EXECUTABLES = frozenset({"runtime/node", "decoder/grib_ls", *(STAGE + name for name in ("firefox-esr", "firefox-bin", "crashhelper", "crashreporter", "glxtest", "vaapitest", "pingsender"))}) MAX_FILES = 512 @@ -92,31 +119,41 @@ def mode(name): def bound(name): if name.startswith(STAGE) or name in ("runtime/node", "decoder/grib_ls"): return MAX_FILE - return 8 * 1024 * 1024 if name.startswith(STATIC) else 1024 * 1024 + return 8 * 1024 * 1024 if name.startswith((STATIC, STATIC_V3)) else 1024 * 1024 -def static_name(name): - return name.startswith(STATIC) and all(re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.@+-]{0,254}", part) - and part not in (".", "..") for part in name[len(STATIC):].split("/")) +def static_name(name, version=1): + prefix = STATIC_V3 if version == 3 else STATIC + safe = name.startswith(prefix) and all(re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.@+-]{0,254}", part) + and part not in (".", "..") for part in name[len(prefix):].split("/")) + if not safe or version != 3: + return safe + relative = name[len(prefix):] + return (relative in STATIC_REQUIRED_V3 + or relative.startswith("assets/") and Path(relative).suffix in STATIC_ASSET_EXTENSIONS_V3 + or relative.startswith("icons/") and Path(relative).suffix == ".png") def manifest(raw, expected): require(hex64(expected) and 0 < len(raw) <= MAX_MANIFEST and hashlib.sha256(raw).hexdigest() == expected) value = json.loads(raw, object_pairs_hook=B["unique_pairs"]) require(type(value) is dict and set(value) == {"version", "kind", "files"} - and type(value["version"]) is int and value["version"] in (1, 2) and value["kind"] == KIND + and type(value["version"]) is int and value["version"] in (1, 2, 3) and value["kind"] == KIND and type(value["files"]) is dict and 1 <= len(value["files"]) <= MAX_FILES and encode(value) == raw) - fixed = FIXED if value["version"] == 1 else FIXED_V2 - require(fixed <= set(value["files"]) and all(name in fixed or static_name(name) for name in value["files"]) - and STATIC + "index.html" in value["files"]) + version = value["version"] + fixed = {1: FIXED, 2: FIXED_V2, 3: FIXED_V3}[version] + prefix = STATIC_V3 if version == 3 else STATIC + required_static = STATIC_REQUIRED_V3 if version == 3 else {"index.html"} + require(fixed <= set(value["files"]) and all(name in fixed or static_name(name, version) for name in value["files"]) + and {prefix + name for name in required_static} <= set(value["files"])) total = static_bytes = 0 for name, item in value["files"].items(): require(type(item) is dict and set(item) == {"bytes", "mode", "sha256"} and type(item["bytes"]) is int and 0 < item["bytes"] <= bound(name) and type(item["mode"]) is int and item["mode"] == mode(name) and hex64(item["sha256"])) total += item["bytes"] - static_bytes += item["bytes"] if name.startswith(STATIC) else 0 + static_bytes += item["bytes"] if name.startswith(prefix) else 0 require(total <= MAX_TOTAL and static_bytes <= MAX_STATIC) for name, expected_sha in (("runtime/node", NODE_SHA), ("decoder/grib_ls", DECODER_SHA), (WEATHER + "third_party/eccodes-sources.json", PIN_SHA), (WEATHER + "scripts/build_eccodes.py", BUILD_SHA)): @@ -207,6 +244,12 @@ def source_proof(weather, browser, ui, stage, selected, version=1): and metadata["autoconfig_sha256"] == selected[STAGE + "volparossa.cfg"]["sha256"]) for relative, digest in runtime.items(): require(selected[STAGE + relative]["sha256"] == digest) + decoder_proof(weather) + if version == 2: + extended_source_proof(weather, selected, proof) + + +def decoder_proof(weather): build = small_json(weather / "build/eccodes/build-receipt.json") require(build["compiled"] is True and build["embedded_definitions"] is True and build["required_environment"] == {} and build["source_pin_sha256"] == PIN_SHA @@ -220,8 +263,6 @@ def source_proof(weather, browser, ui, stage, selected, version=1): artifacts[item["path"]] = item require(artifacts.get("build/eccodes/eccodes-build/bin/grib_ls") == dict(path="build/eccodes/eccodes-build/bin/grib_ls", bytes=31328216, sha256=DECODER_SHA)) - if version == 2: - extended_source_proof(weather, selected, proof) def extended_source_proof(weather, selected, proof): @@ -275,6 +316,112 @@ def extended_source_proof(weather, selected, proof): require(selected[WEATHER + relative]["sha256"] == digest) +def overmorrow_source_proof(weather, stage, selected, proof_path): + """Bind the current real UI proof. Origin receipts remain historical only. + +This does not assert that the current acquisition wrappers have run against +origins: the joined runner must acquire and validate new products itself. +Neither historical receipt is changed or used as current-wrapper authority. +""" + proof = small_json(proof_path) + pin = small_json(weather / "integrations/overmorrow/upstream.json") + build = small_json(weather / "integrations/overmorrow/build-verification.json") + require(type(proof["schema"]) is int and proof["schema"] == 1 and proof["passed"] is True + and proof["phase"] == "complete" and proof["wrapper_outcome"] == "completed") + scope = dict(actual_overmorrow_ui=True, actual_http_bridge=True, synthetic_weather=True, + synthetic_air_quality=True, core_executed=False, overlay_proven=False, + upstream_origin_proven=False, native_mobile_proven=False) + require(proof["scope"] == scope and all(type(value) is bool for value in proof["scope"].values())) + for group, names in (("cleanup", ("browser_exited", "fixture_exited", "private_profile_removed", + "normal_browser_stop", "normal_fixture_stop")), + ("outer_cleanup", ("namespace_wrapper_exited", "private_profile_removed", "host_netns_unchanged"))): + require(set(proof[group]) == set(names) and all(proof[group][name] is True for name in names)) + checks = proof["checks"] + for name in ("actual_canvas_and_semantics", "independent_native_timelines", "native_values_visible", + "no_fabricated_current_or_aqi", "fixed_public_area", "source_attribution", "actual_refresh", + "weather_failure_and_stale_preserve_air_quality", "weather_idle_expiry_without_request", + "air_quality_failure_and_stale_preserve_weather", "air_quality_idle_expiry_without_request", + "existing_aq_detail_without_new_request", "same_origin_loaded_resources"): + require(checks[name] is True) + for name in ("provider_requests", "air_quality_provider_requests"): + require(type(checks[name]) is int and checks[name] == 10) + require(all(hex64(checks[name]) for name in ("initial_semantics_sha256", "final_semantics_sha256"))) + source = proof["source"] + require(set(source) == {"upstream", "tree", "patch_sha256", "overlay_sha256", "deployed_tree_sha256", + "deployed_files", "deployed_bytes", "bridge_modules", "fixture_modules", "browser_helpers", + "isolation_helper_sha256", "driver_sha256", "fixture_sha256"}) + require(source["upstream"] == pin["commit"] == build["upstream_commit"] + and source["tree"] == pin["tree"] and source["patch_sha256"] == pin["upstream_patch_sha256"] + == build["upstream_patch_sha256"] == selected[WEATHER + "integrations/overmorrow/upstream.patch"]["sha256"] + and selected[UI_STAGE_V3 + "LICENSE"]["sha256"] == pin["license_sha256"] + == selected[STATIC_V3 + "LICENSE.txt"]["sha256"] + and selected[UI_STAGE_V3 + "pubspec.lock"]["sha256"] == pin["pubspec_lock_sha256"] == build["pubspec_lock_sha256"] + and selected[UI_STAGE_V3 + "LICENSE.VOLPAROSSA"]["sha256"] == selected[WEATHER + "LICENSE"]["sha256"] + == selected[STATIC_V3 + "LICENSE.VOLPAROSSA.txt"]["sha256"]) + for relative, digest in proven_hashes(source["overlay_sha256"], OVERMORROW_OVERLAY).items(): + require(selected[WEATHER + "integrations/overmorrow/overlay/" + relative]["sha256"] + == selected[UI_STAGE_V3 + relative]["sha256"] == digest) + for group, names, prefix in (("bridge_modules", OVERMORROW_BRIDGE, WEATHER), + ("fixture_modules", OVERMORROW_FIXTURE, WEATHER), + ("browser_helpers", PROVEN_BROWSER_HELPERS, BROWSER)): + for relative, digest in proven_hashes(source[group], names).items(): + require(selected[prefix + relative]["sha256"] == digest) + for field, relative in (("isolation_helper_sha256", "integrations/drizzli/smoke_browser.py"), + ("driver_sha256", "integrations/overmorrow/smoke_browser.py"), + ("fixture_sha256", "integrations/overmorrow/browser-fixture.mjs")): + require(hex64(source[field]) and selected[WEATHER + relative]["sha256"] == source[field]) + hashes = {name[len(STATIC_V3):]: item["sha256"] for name, item in selected.items() if name.startswith(STATIC_V3)} + tree_sha = hashlib.sha256(json.dumps(hashes, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + deployment = build["deployment"] + require(tree_sha == source["deployed_tree_sha256"] == deployment["tree_sha256"] + and type(source["deployed_files"]) is int and source["deployed_files"] == len(hashes) == deployment["files"] + and type(source["deployed_bytes"]) is int and source["deployed_bytes"] == deployment["bytes"] + == sum(item["bytes"] for name, item in selected.items() if name.startswith(STATIC_V3)) + and deployment["static_profile"] == "flutter-canvaskit") + for field, name in (("main_js_sha256", "main.dart.js"), ("bootstrap_sha256", "flutter_bootstrap.js"), + ("notices_sha256", "assets/NOTICES")): + require(deployment[field] == hashes[name]) + require(type(build["schema"]) is int and build["schema"] == 1 and build["actual_release_web_build"] is True + and build["profile"] == "VOLPAROSSA_WEATHER=true" and build["flutter"] == pin["flutter"] + and build["flutter_commit"] == pin["flutter_commit"] and build["synthetic_products"] is True + and build["browser_interaction_verified"] is True and build["core_delivery_verified"] is False + and build["native_android_ios_verified"] is False and build["forecast_accuracy_verified"] is False + and build["browser_trial"]["passed"] is True + and build["browser_trial"]["report_sha256"] == selected[WEATHER + OVERMORROW_PROOF]["sha256"] + and build["request_lifecycle"]["provider_source_sha256"] == source["overlay_sha256"]["lib/volparossa/provider.dart"] + and build["request_lifecycle"]["cold_core_delivery_verified"] is False) + metadata = small_json(stage / "volparossa-staging.json") + runtime = proven_hashes(metadata["runtime_sha256"], PROVEN_RUNTIME_FILES) + require(runtime == proven_hashes(proof["runtime"]["sha256"], PROVEN_RUNTIME_FILES) + and proof["runtime"]["node_sha256"] == NODE_SHA + and metadata["defaults_sha256"] == selected[BROWSER + "defaults/privacy.json"]["sha256"] + and metadata["autoconfig_sha256"] == selected[STAGE + "volparossa.cfg"]["sha256"]) + for relative, digest in runtime.items(): + require(selected[STAGE + relative]["sha256"] == digest) + decoder_proof(weather) + # Retain known separate historical source evidence without claiming its + # hashes describe these current wrappers (notably the newer NOAA cleanup). + for filename, kind, group in (("noaa-gefs-smoke.json", "direct-noaa-native-particulate-forecast", NOAA_MODULES), + ("ecmwf-precipitation-smoke.json", "direct-ecmwf-native-precipitation-forecast", ECMWF_MODULES)): + historical = small_json(weather / "third_party" / filename) + require(type(historical["version"]) is int and historical["version"] == 1 and historical["kind"] == kind + and historical["success"] is True and historical["node_sha256"] == NODE_SHA + and historical["decoder_sha256"] == DECODER_SHA) + proven_hashes(historical["source_modules"], group) + if filename.startswith("noaa"): + require(historical["origin"] == "https://noaa-gefs-pds.s3.amazonaws.com" + and historical["scope"]["actual_origin_acquisition"] is True + and historical["scope"]["actual_pinned_native_decoder"] is True + and all(historical["scope"][key] is False for key in + ("core_executed", "overlay_proven", "browser_executed", "accuracy_proven"))) + else: + require(historical["source"] == "https://data.ecmwf.int/forecasts" + and historical["native_accumulation_windows_preserved"] is True + and historical["hourly_rates_inferred"] is False + and historical["core_overlay_roundtrip_proven"] is False + and historical["browser_execution_proven"] is False) + + def capture(weather, ui, browser, stage, node, archive, version=1): require(type(version) is int and version in (1, 2)) for root in (weather, ui, browser, stage): @@ -292,6 +439,32 @@ def capture(weather, ui, browser, stage, node, archive, version=1): raw = encode(dict(version=version, kind=KIND, files=selected)); expected = hashlib.sha256(raw).hexdigest() manifest(raw, expected) source_proof(weather, browser, ui, stage, selected, version) + return write_archive(sources, selected, raw, expected, archive) + + +def capture_v3(weather, ui, static_root, browser, stage, node, proof, archive): + for root in (weather, ui, static_root, browser, stage): + B["canonical"](root) + inventory(stage, exact=RUNTIME_FILES) + require(set(inventory(weather / "integrations/overmorrow/overlay")) == OVERMORROW_OVERLAY) + sources = {**{WEATHER + name: weather / name for name in WEATHER_FILES_V3 if name != OVERMORROW_PROOF}, + WEATHER + OVERMORROW_PROOF: proof, + **{BROWSER + name: browser / name for name in BROWSER_FILES}, + **{STAGE + name: stage / name for name in RUNTIME_FILES}, + **{UI_STAGE_V3 + name: ui / name for name in CLIENT_FILES_V3}, + **{STATIC_V3 + name: path for name, path in inventory(static_root).items()}, + "runtime/node": node, "decoder/grib_ls": weather / "build/eccodes/eccodes-build/bin/grib_ls"} + selected = {name: dict(**fingerprint(path, bound(name)), mode=mode(name)) for name, path in sorted(sources.items())} + for name in EXECUTABLES: + require(sources[name].stat().st_mode & 0o100) + raw = encode(dict(version=3, kind=KIND, files=selected)); expected = hashlib.sha256(raw).hexdigest() + manifest(raw, expected) + overmorrow_source_proof(weather, stage, selected, proof) + return dict(**write_archive(sources, selected, raw, expected, archive), + historical_source_proofs_only=True, current_origin_acquisition_proven=False, joined_delivery_proven=False) + + +def write_archive(sources, selected, raw, expected, archive): new_path(archive) with os.fdopen(os.open(archive, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "wb") as output: for name in ("INPUTS.json", *sorted(selected)): @@ -322,9 +495,9 @@ def checked_archive(archive, expected, root=None, *, runtime=False, report_versi raw = read_exact(source, item.size) require(read_exact(source, -item.size % 512) == b"\0" * (-item.size % 512)) selected = manifest(raw, expected) - # Both closed profiles have distinct joined-runtime dispatch. Preserve + # All closed profiles have explicit joined-runtime dispatch. Preserve # this explicit gate when future archive-only profiles are introduced. - require(not runtime or selected["version"] in (1, 2)) + require(not runtime or selected["version"] in (1, 2, 3)) if root is not None: new_path(root); root.mkdir(mode=0o700) for name, record in sorted(selected["files"].items()): @@ -374,6 +547,8 @@ def main(): action, *args = sys.argv[1:] if action in ("capture", "capture-v2") and len(args) == 6: result = capture(*map(Path, args), version=2 if action == "capture-v2" else 1) + elif action == "capture-v3" and len(args) == 8: + result = capture_v3(*map(Path, args)) elif action == "verify-runtime" and len(args) == 2: result = checked_archive(Path(args[0]), args[1], runtime=True) elif action == "runtime-version" and len(args) == 2: diff --git a/tests/integration/weather-client-delivery.sh b/tests/integration/weather-client-delivery.sh index 1e67b2ba9..658429488 100644 --- a/tests/integration/weather-client-delivery.sh +++ b/tests/integration/weather-client-delivery.sh @@ -118,11 +118,11 @@ weather_client_delivery_run() { # legacy profile must keep its original driver and evidence contract. weather_profile=$(weather_client_runtime_profile) || fail WEATHER_CLIENT_PROFILE_INVALID weather_profile_version=$(printf '%s\n' "$weather_profile" | \ - jq -er '.bundle_version | select(type == "number" and (. == 1 or . == 2))') \ + jq -er '.bundle_version | select(type == "number" and (. == 1 or . == 2 or . == 3))') \ || fail WEATHER_CLIENT_PROFILE_INVALID case $weather_profile_version in 1) ;; - 2) + 2|3) . "$source_directory/tests/integration/weather-products-delivery.sh" weather_products_delivery_run return ;; diff --git a/tests/integration/weather-products-delivery.py b/tests/integration/weather-products-delivery.py index 870ee21e7..e2b4c91bc 100644 --- a/tests/integration/weather-products-delivery.py +++ b/tests/integration/weather-products-delivery.py @@ -2,9 +2,9 @@ # SPDX-License-Identifier: GPL-3.0-only """Join two real source products to independent cold transfers and native UI. -Version one is deliberately untouched. This closed version-two fixture shares -its process, ownership and packet-boundary checks, never its single-product -evidence. Coordinates and raw receipts stay inside the disposable guest. +Version one is deliberately untouched. Closed version-two (Drizz.li) and +version-three (Overmorrow) UI profiles share two-product acquisition and transfer, +not each other's UI evidence. Coordinates and raw receipts stay inside the guest. """ import datetime import hashlib @@ -51,6 +51,18 @@ SAFE_ERRORS = (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError) +def ui_profile(version): + # A version selects a fixed consumer, never a caller-supplied executable. + require(type(version) is int and version in (2, 3)) + if version == 2: + return dict(ui="actual_drizzli_ui", static=B["STATIC"], + preparer="integrations/drizzli/prepare-core-browser.mjs", + driver="integrations/drizzli/smoke_core_browser.py") + return dict(ui="actual_overmorrow_ui", static=B["WEATHER"] + "build/overmorrow-web/", + preparer="integrations/overmorrow/prepare_core_browser.mjs", + driver="integrations/overmorrow/smoke_core_browser.py") + + class AcquisitionFailure(ValueError): """Carry only a closed classification, never a subprocess error or argv.""" def __init__(self, stage, error): @@ -169,7 +181,7 @@ def acquisition(source, manifest_sha, root, parent, client, group): boundary(parent, client, group, True) stage = "tree" B["verify_tree"](source, manifest_sha) - require(B["manifest"](raw_file(source / "INPUTS.json", B["MAX_MANIFEST"], 0o400), manifest_sha)["version"] == 2) + ui_profile(B["manifest"](raw_file(source / "INPUTS.json", B["MAX_MANIFEST"], 0o400), manifest_sha)["version"]) deadline = time.monotonic() + 720 stage = "decoder" require(run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10).strip() == b"ecCodes Version 2.44.0") @@ -293,7 +305,8 @@ def cold(key, binary, control, cache, viewer, parent, client, group): cold_cache_requested=True, observed_unix_seconds=int(time.time())) -def prepare(source, viewer, binary, control, weather_cache, air_quality_cache): +def prepare(source, viewer, binary, control, weather_cache, air_quality_cache, version=2): + profile = ui_profile(version) private(viewer) selected = validate_selection(read(viewer / "selection.json", mode=0o400)) core_sha = B["fingerprint"](binary, 256*1024**2)["sha256"] @@ -307,7 +320,7 @@ def core_config(key, cache): airQuality=dict(core=core_config("air_quality", air_quality_cache), variables=["pm2_5", "pm10"])) write_new(viewer / "config.json", B["encode"](config)) raw = run([str(source / "runtime/node"), "--max-old-space-size=256", - str(source / B["WEATHER"] / "integrations/drizzli/prepare-core-browser.mjs"), + str(source / B["WEATHER"] / profile["preparer"]), "--forecast", str(viewer / "forecast.json"), "--receipt", str(viewer / "cold-receipt.json"), "--air-quality", str(viewer / "air-quality.json"), "--air-quality-receipt", str(viewer / "air-quality-cold-receipt.json"), "--config", str(viewer / "config.json"), "--core-sha256", core_sha, "--output", str(viewer / "expected.json")], viewer, 4096, 30) @@ -316,7 +329,8 @@ def core_config(key, cache): require(result["prepared"] is True and hex64(result["expected_sha256"]) and hex64(result["display_sha256"])) expected_raw = raw_file(viewer / "expected.json", MAX_EXPECTED, 0o400) expected = json.loads(expected_raw, object_pairs_hook=B["B"]["unique_pairs"]) - require(set(expected) == {"schema", "core_sha256", "products", "display_sha256"} and expected["schema"] == 2 + require(set(expected) == {"schema", "core_sha256", "products", "display_sha256"} + and type(expected["schema"]) is int and expected["schema"] == version and expected["core_sha256"] == core_sha and set(expected["products"]) == set(PRODUCTS) and hashlib.sha256(expected_raw).hexdigest() == result["expected_sha256"] and expected["display_sha256"] == result["display_sha256"]) @@ -394,7 +408,7 @@ def validate_acquisition(value): def closed_report(report): require(type(report) is dict and set(report) <= {"schema", "passed", "phase", "source", "expected", "products", "scope", "checks", "cleanup", "outer_cleanup", "failure"}) - same(report["schema"], 2); require(type(report["passed"]) is bool) + profile = ui_profile(report["schema"]); require(type(report["passed"]) is bool) require(report["phase"] in {"inputs", "sandbox", "bridge_start", "provider", "browser_start", "hydration", "refresh", "shutdown", "complete"}) if "failure" in report: require(report["failure"] in ("interrupted", "check_or_runtime_failed")) source = report["source"] @@ -415,7 +429,7 @@ def closed_report(report): for key, v in item["checks"].items(): require(hex64(v) if key.endswith("sha256") else type(v) is int and 0 <= v <= 2 if key in {"browser_api_requests", "provider_requests_checked"} else type(v) is bool) - require(set(report["scope"]) == {"actual_drizzli_ui", "actual_http_bridge", "real_core_adapter", "synthetic_weather", + require(set(report["scope"]) == {profile["ui"], "actual_http_bridge", "real_core_adapter", "synthetic_weather", "synthetic_air_quality", "separate_native_runs", "standalone_origin_proven", "standalone_overlay_proven", "browser_triggered_cold_fetch"} and all(type(v) is bool for v in report["scope"].values())) require(set(report["checks"]) <= {"no_external_resources", "provider_requests_checked", "browser_api_requests", @@ -483,7 +497,7 @@ def validate_cold_phase(work, manifest_sha, binary_sha): publisher = work / "client-fixtures/weather-publisher" require(not publisher.exists() and not publisher.is_symlink()) manifest = B["manifest"](raw_file(artifact(work, "manifest"), B["MAX_MANIFEST"], 0o400), manifest_sha) - same(manifest["version"], 2) + ui_profile(manifest["version"]) same(read(artifact(work, "tree-before")), dict(version=1, action="verify-tree", manifest_sha256=manifest_sha, files=len(manifest["files"]), runtime_executed=False)) return selected, acquired, colds, manifest, caches @@ -537,6 +551,8 @@ def cold_evidence(work, manifest_sha, binary_sha): def evidence(work, manifest_sha, binary_sha): selected, acquired, colds, manifest, before = validate_cold_phase(work, manifest_sha, binary_sha) + version = manifest["version"] + profile = ui_profile(version) retained = read(artifact(work, "cold-evidence")); validate_cold_export(retained) same(retained, cold_summary(selected, acquired, colds, manifest_sha, binary_sha)) same(read(artifact(work, "input-cleanup")), dict(directory_removed=True, files_removed=True)) @@ -549,19 +565,20 @@ def evidence(work, manifest_sha, binary_sha): same(read(artifact(work, "tree-" + phase)), dict(version=1, action="verify-tree", manifest_sha256=manifest_sha, files=len(manifest["files"]), runtime_executed=False)) report = closed_report(read(artifact(work, "original-report"), 16384)) + same(report["schema"], version) same(read(artifact(work, "prepared")), dict(prepared=True, expected_sha256=report["expected"]["expected_sha256"], display_sha256=report["expected"]["display_sha256"])) require(report["passed"] is True and report["phase"] == "complete" and "failure" not in report) - same(report["scope"], dict(actual_drizzli_ui=True, actual_http_bridge=True, real_core_adapter=True, + same(report["scope"], dict({profile["ui"]: True}, actual_http_bridge=True, real_core_adapter=True, synthetic_weather=False, synthetic_air_quality=False, separate_native_runs=True, standalone_origin_proven=False, standalone_overlay_proven=False, browser_triggered_cold_fetch=False)) require(report["source"]["bundle_manifest_sha256"] == manifest_sha and report["expected"]["core_sha256"] == binary_sha) - for field, relative in (("driver_sha256", "integrations/drizzli/smoke_core_browser.py"), - ("preparer_sha256", "integrations/drizzli/prepare-core-browser.mjs")): + for field, relative in (("driver_sha256", profile["driver"]), ("preparer_sha256", profile["preparer"])): require(report["source"][field] == manifest["files"][B["WEATHER"] + relative]["sha256"]) for relative, digest in report["source"]["bridge_modules"].items(): require(digest == manifest["files"][B["WEATHER"] + relative]["sha256"]) require(report["source"]["node_sha256"] == B["NODE_SHA"]) - static = {name[len(B["STATIC"]):]: item["sha256"] for name, item in manifest["files"].items() if name.startswith(B["STATIC"])} + static = {name[len(profile["static"]):]: item["sha256"] for name, item in manifest["files"].items() + if name.startswith(profile["static"])} require(report["source"]["built_tree_sha256"] == hashlib.sha256(json.dumps(static, sort_keys=True, separators=(",", ":")).encode()).hexdigest()) for key in PRODUCTS: item = report["products"][key]; publication = selected["products"][key]["publication"] @@ -583,12 +600,12 @@ def evidence(work, manifest_sha, binary_sha): # validating/exporting this historical evidence later. observed_at = observed_ms() for key in PRODUCTS: future_query(key, selected["products"][key]["query"], observed_at) - result = dict(schema=2, success=True, kind="volparossa-weather-products-delivery", bundle_manifest_sha256=manifest_sha, + result = dict(schema=version, success=True, kind="volparossa-weather-products-delivery", bundle_manifest_sha256=manifest_sha, core_sha256=binary_sha, decoder_sha256=B["DECODER_SHA"], products=retained["products"], browser_report_sha256=hashlib.sha256(raw_file(artifact(work, "original-report"))).hexdigest(), display_sha256=report["expected"]["display_sha256"], - scope=dict(actual_acquisition=True, same_publications_joined=True, independent_product_captures=True, - actual_drizzli_ui=True, actual_http_bridge=True, actual_ui_refresh=True, end_to_end_proven=True, + scope=dict({profile["ui"]: True}, actual_acquisition=True, same_publications_joined=True, independent_product_captures=True, + actual_http_bridge=True, actual_ui_refresh=True, end_to_end_proven=True, browser_triggered_cold_fetch=False, browser_cache_only_proven=False, two_subflow_bulk_aggregation=False, upstream_signature_attested=False, production_service_complete=False), cleanup=dict(publisher_sources_removed_before_fetch=True, browser_private_state_removed=True, @@ -601,16 +618,17 @@ def validate_evidence_export(value): require(set(value) == {"schema", "success", "kind", "bundle_manifest_sha256", "core_sha256", "decoder_sha256", "products", "browser_report_sha256", "display_sha256", "scope", "cleanup"}) same(value["kind"], "volparossa-weather-products-delivery") + profile = ui_profile(value["schema"]) require(hex64(value["browser_report_sha256"]) and hex64(value["display_sha256"])) cold = {k: v for k, v in value.items() if k not in {"browser_report_sha256", "display_sha256", "scope", "cleanup"}} - cold.update(kind="volparossa-weather-products-protected-cold", scope=dict(actual_acquisition=True, + cold.update(schema=2, kind="volparossa-weather-products-protected-cold", scope=dict(actual_acquisition=True, acquisition_outside_overlay=True, same_publications_joined=True, independent_product_captures=True, protected_cold_transfer=True, actual_drizzli_ui=False, actual_http_bridge=False, actual_ui_refresh=False, end_to_end_proven=False, browser_triggered_cold_fetch=False, two_subflow_bulk_aggregation=False, upstream_signature_attested=False, production_service_complete=False), cleanup=dict(publisher_sources_removed_before_fetch=True)) validate_cold_export(cold) - same(value["scope"], dict(actual_acquisition=True, same_publications_joined=True, independent_product_captures=True, - actual_drizzli_ui=True, actual_http_bridge=True, actual_ui_refresh=True, end_to_end_proven=True, + same(value["scope"], dict({profile["ui"]: True}, actual_acquisition=True, same_publications_joined=True, independent_product_captures=True, + actual_http_bridge=True, actual_ui_refresh=True, end_to_end_proven=True, browser_triggered_cold_fetch=False, browser_cache_only_proven=False, two_subflow_bulk_aggregation=False, upstream_signature_attested=False, production_service_complete=False)) same(value["cleanup"], dict(publisher_sources_removed_before_fetch=True, browser_private_state_removed=True, @@ -727,6 +745,7 @@ def main(): elif action == "select" and len(args) == 2: result = select(*map(Path, args)) elif action == "cold" and len(args) == 8: result = cold(args[0], *map(Path, args[1:5]), args[5], args[6], int(args[7])) elif action == "prepare" and len(args) == 6: result = prepare(*map(Path, args)) + elif action == "prepare" and len(args) == 7: result = prepare(*map(Path, args[:6]), int(args[6])) elif action in {"publisher-cleanup", "user-cleanup"} and len(args) == 1: result = cleanup(Path(args[0]), action == "publisher-cleanup") elif action == "cache-state" and len(args) == 4: result = cache_state(args[0], Path(args[1]), int(args[2]), int(args[3])) elif action == "cache-summary" and len(args) == 2: result = cache_summary(Path(args[0]), args[1]) diff --git a/tests/integration/weather-products-delivery.sh b/tests/integration/weather-products-delivery.sh index e4b0b56fd..c8c1629e4 100644 --- a/tests/integration/weather-products-delivery.sh +++ b/tests/integration/weather-products-delivery.sh @@ -102,8 +102,15 @@ weather_client_delivery_copy_artifacts() { weather_products_delivery_run() { [ -n "$weather_client_bundle" ] && [ -n "$weather_client_manifest_sha256" ] \ || fail WEATHER_PRODUCTS_INPUTS_MISSING + # Selected by complete immutable-archive validation in the enclosing + # dispatcher. Never accept a caller-supplied driver path or fallback UI. + case ${weather_profile_version:-} in + 2) weather_products_driver=integrations/drizzli/smoke_core_browser.py ;; + 3) weather_products_driver=integrations/overmorrow/smoke_core_browser.py ;; + *) fail WEATHER_CLIENT_PROFILE_INVALID ;; + esac PHASE=weather-products-delivery-prepare - printf '%s\n' 'Weather v2 trial: acquire public ECMWF precipitation/weather and NOAA particulate fields in the disposable guest; publish separately to two existing peers, remove both originals and publisher identity, cold-fetch each product with independent captures, then check both products and refresh in the real isolated client. No host network or user-data changes.' + printf '%s\n' 'Weather two-product trial: acquire public ECMWF precipitation/weather and NOAA particulate fields in the disposable guest; publish separately to two existing peers, remove both originals and publisher identity, cold-fetch each product with independent captures, then check both products and refresh in the exact bundled isolated client. No host network or user-data changes.' [ "$WORKER_UID" = 985 ] && [ "$site_control_gid" != "$AGENT_GID" ] \ || fail WEATHER_PRODUCTS_USER_INVALID weather_client_directory=$WORK/client-fixtures/weather-client-source @@ -264,7 +271,8 @@ weather_products_delivery_run() { --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ -- python3 -B "$WORK/bin/weather-products-delivery.py" prepare "$weather_client_directory" "$weather_viewer" \ "$binary_directory/volparossa" "$WORK/runtime-client/control/agent.sock" "$WORK/state-client/weather-cache" \ - "$WORK/state-client/air-quality-cache" >"$WORK/weather-products-delivery-prepared.json" || fail WEATHER_PRODUCTS_EXPECTED_FAILED + "$WORK/state-client/air-quality-cache" "$weather_profile_version" \ + >"$WORK/weather-products-delivery-prepared.json" || fail WEATHER_PRODUCTS_EXPECTED_FAILED PHASE=weather-products-delivery-browser weather_browser_status=0 weather_command_limit=$(weather_products_remaining 270) || fail WEATHER_PRODUCTS_DEADLINE @@ -272,7 +280,7 @@ weather_products_delivery_run() { --reuid="$WORKER_UID" --regid="$site_control_gid" --clear-groups \ --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ -- env -i PATH=/usr/bin:/bin LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONDONTWRITEBYTECODE=1 \ - python3 -B "$weather_client_directory/volparossa-weather/integrations/drizzli/smoke_core_browser.py" \ + python3 -B "$weather_client_directory/volparossa-weather/$weather_products_driver" \ --execute --bundle-root "$weather_client_directory" --bundle-manifest-sha256 "$weather_client_manifest_sha256" \ --config "$weather_viewer/config.json" --expected "$weather_viewer/expected.json" \ --output "$weather_viewer/browser-output" --parent-netns "$site_parent_ns" --client-netns "$site_client_ns" \ From c2fbcd8c7f14a4c917c2c7bfe721a0650e8d20c9 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:46:43 +0200 Subject: [PATCH 32/49] test: admit exact late-provider capture interfaces --- docs/IMPLEMENTATION_STATUS.md | 50 +++++++++++++++---- tests/integration/kvm-alpha-topology.sh | 2 +- .../test-content-provider-smoke.py | 41 ++++++++++++++- 3 files changed, 81 insertions(+), 12 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 4c2e62514..ef28d7edd 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,14 +29,14 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Resolve the observed fixture I/O pressure, prove concurrent three-provider bulk transfer, then exercise late complementary-provider recovery. | The standalone run on `7ad108c0` reconstructs native and HTTPS adaptive objects sequentially. Both budget reports identify I/O pressure as the sole pressure/unavailability flag, holding admission at one worker despite memory and FD headroom. Identify and reduce the actual storage bottleneck or establish bounded readiness; do not override pressure protection or weaken the overlap gate. The separate CI primary-Exit failure remains unresolved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Exercise late complementary-provider recovery after the now-proven concurrent three-provider transfers. | The standalone run on `a5e698cc` passes native and HTTPS bulk-overlap checks after bounded readiness, without overriding resource protection. It then stops because the control observer rejects the new late-provider test interfaces before fetching. Correct that fixture allowlist and repeat the real trial; late recovery and requester cancellation remain unproven. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR8 aligns client/bridge deadlines and cancellation. The new closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows; component tests pass. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the standalone -provider proof, which needs no weather downloads, after investigating the failed -concurrency requirement; successful reconstruction does not prove late discovery +provider proof, which needs no weather downloads, after correcting the late-phase +capture setup; successful concurrent reconstruction does not prove late discovery convergence. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM @@ -103,13 +103,43 @@ capacity is genuinely available, without competing heavy jobs or reduced limits. The retained runner log is `.git/ci-evidence/content-provider-readiness-01.X2JNlh/runner.log`. -The following readiness attempt uses clean source -`a5e698ccb20c8d4a5288ca9d49e2286b1ff37019` in a separate worktree, retaining -the same guest and host-memory limits. It starts after capacity becomes -available; its private evidence directory is -`.git/ci-evidence/content-provider-readiness-02.WgVnio`. -No result is claimed while the trial runs. Overmorrow changes are excluded -from that snapshot, and competing heavy builds remain deferred. +The following real readiness run on clean source +`a5e698ccb20c8d4a5288ca9d49e2286b1ff37019` passes the separate adaptive +evidence validator. Native and cooperative-HTTPS transfers each reconstruct +15,728,640 bytes in sixty chunks from three providers, with no origin body bytes +or range requests. Their three-provider bulk windows overlap by +`4,082,921,445 ns` and `4,552,924,112 ns` respectively. Both worker-budget +receipts record three concurrent leases, no pressure flags or denied leases, +and zero leases remaining on return. Each preparation reaches three consecutive +acceptable PSI samples after about twelve seconds; the unchanged live admission +policy and packet-overlap requirements still decide the result. + +The overall run remains **failed**: it stops at +`content-provider-late-recovery-register-a` with +`CONTENT_PROVIDER_LATE_CONTROL_CAPTURE_FAILED`. The observer's closed interface +list accepts `cp0/cp1` and `ac0/ac1/ac2`, but omits the new `lc0/lc1` pair. +It rejects startup before the late fetch begins, so neither late recovery nor +requester cancellation is established. The aggregate report conservatively +retains false completion flags; the successful adaptive result is separate. + +Evidence is retained under +`.git/ci-evidence/content-provider-readiness-02.WgVnio/output/`: +`content-provider-adaptive-evidence.json` has SHA-256 +`bde143d0f81a8b846a98edec6be3b668bfd3091ba60c3e97cac7c50f3e315e77`; +the failed `content-provider-smoke.json` has SHA-256 +`ee54a50e5ec025a3c0dce877c437bf407e03a22de09eae7524eaa5cb47ef8047`. +Cleanup leaves zero owned objects and unchanged guest-root network state. +The VM and scratch overlay are removed; development-host DNS and IPv4/IPv6 +route hashes match their pre-run values. Host-firewall comparison remains +unavailable. No weather origin was queried, and Overmorrow changes are excluded +from this exact-source run. + +The capture parser now also accepts the exact `lc0/lc1` interface pair. +Regression checks continue rejecting duplicate, reordered, mixed or extra +interfaces, malformed endpoints and foreign protocol/peer traffic. Nineteen +provider checks, ten inert cancellation checks, seven late-recovery checks and +eleven observer checks pass. This fixture correction still needs a new live +late-recovery result; it changes neither production admission nor privacy policy. The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index a7903917e..558b48768 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -4667,7 +4667,7 @@ content_control_pairs = {} if role == "content-control" and content_provider_mode: control_argument = interfaces.pop(0) if interfaces else "" control_addresses = control_argument.removeprefix("--content-control=").split(",") - allowed_interfaces = (["cp0", "cp1"], ["ac0", "ac1", "ac2"]) + allowed_interfaces = (["cp0", "cp1"], ["ac0", "ac1", "ac2"], ["lc0", "lc1"]) if not control_argument.startswith("--content-control=") or interfaces not in allowed_interfaces \ or len(control_addresses) != len(interfaces) + 1 \ or len(set(control_addresses)) != len(control_addresses): diff --git a/tests/integration/test-content-provider-smoke.py b/tests/integration/test-content-provider-smoke.py index 743dd090b..2b2619bd4 100644 --- a/tests/integration/test-content-provider-smoke.py +++ b/tests/integration/test-content-provider-smoke.py @@ -703,9 +703,13 @@ def test_actual_control_classifier_rejects_app_traffic_and_wrong_endpoints(self) for target in node.targets)) parser = compile(ast.Module(body=nodes[start:end], type_ignores=[]), "actual-control-arguments", "exec") for interfaces, accepted in ((["cp0", "cp1"], True), (["ac0", "ac1", "ac2"], True), + (["lc0", "lc1"], True), (["cp0"], False), (["cp0", "ac1"], False), (["ac0", "ac1"], False), (["ac0", "ac1", "ac3"], False), - (["cp0", "cp1", "cp2"], False)): + (["cp0", "cp1", "cp2"], False), + (["lc0"], False), (["lc0", "lc0"], False), + (["lc1", "lc0"], False), (["lc0", "lc2"], False), + (["lc0", "cp1"], False), (["lc0", "lc1", "lc2"], False)): addresses = ["42.158.0.1", "48.164.4.1", "49.165.5.1", "50.166.6.1"][:len(interfaces) + 1] environment = dict(socket=socket, sys=SimpleNamespace(argv=["observer", "content-control", "out", "ready", "--content-providers", "--content-control=" + ",".join(addresses), *interfaces])) @@ -716,6 +720,21 @@ def test_actual_control_classifier_rejects_app_traffic_and_wrong_endpoints(self) else: with self.assertRaises(SystemExit): exec(parser, environment) + # The live late-recovery invocation uses this exact two-interface family; + # adding it must not admit malformed, repeated or incomplete endpoints. + for control_argument in ( + "42.158.0.1,49.165.5.1,50.166.6.1", + "--content-control=42.158.0.1,49.165.5.1", + "--content-control=42.158.0.1,49.165.5.1,50.166.6.1,48.164.4.1", + "--content-control=42.158.0.1,49.165.5.1,49.165.5.1", + "--content-control=42.158.0.1,42.158.0.1,50.166.6.1", + "--content-control=42.158.0.1,not-an-address,50.166.6.1", + "--content-control=42.158.0.1,,50.166.6.1", + ): + environment = dict(socket=socket, sys=SimpleNamespace(argv=["observer", "content-control", "out", "ready", + "--content-providers", control_argument, "lc0", "lc1"])) + with self.subTest(control_argument=control_argument), self.assertRaises((SystemExit, OSError)): + exec(parser, environment) function = next(node for node in nodes if isinstance(node, ast.FunctionDef) and node.name == "record_provider_control") environment = dict(socket=socket, content_control_pairs={"cp0": ["42.158.0.1", "49.165.5.1"]}, @@ -734,6 +753,26 @@ def test_actual_control_classifier_rejects_app_traffic_and_wrong_endpoints(self) ): record(*args) self.assertEqual(environment["unexpected_provider_control_packets"], 4) + + # Late recovery still observes only the exact broker/provider UDP pairs, + # not provider application traffic or another family's interfaces. + pairs = {f"lc{index}": ["42.158.0.1", CHECK["PUBLIC_IPS"][node]] + for index, node in enumerate(("relay4", "relay5"))} + environment = dict(socket=socket, content_control_pairs=pairs, + content_control_packets={interface: dict(inbound=0, outbound=0) for interface in pairs}, + unexpected_provider_control_packets=0) + exec(compile(ast.Module(body=[function], type_ignores=[]), "actual-late-control", "exec"), environment) + record = environment["record_provider_control"] + for interface, (source, destination) in pairs.items(): + record(interface, socket.IPPROTO_UDP, source, 41000, destination, 41000) + record(interface, socket.IPPROTO_UDP, destination, 41000, source, 41000) + self.assertEqual(environment["content_control_packets"][interface], dict(inbound=1, outbound=1)) + for args in (("lc0", socket.IPPROTO_TCP, "42.158.0.1", 32000, "49.165.5.1", 18080), + ("lc0", socket.IPPROTO_UDP, "42.158.0.1", 41000, "50.166.6.1", 41000), + ("lc2", socket.IPPROTO_UDP, "42.158.0.1", 41000, "49.165.5.1", 41000), + ("ac0", socket.IPPROTO_UDP, "42.158.0.1", 41000, "49.165.5.1", 41000)): + record(*args) + self.assertEqual(environment["unexpected_provider_control_packets"], 4) # The additive family has three exact endpoint pairs, not a wildcard control allowance. pairs = {f"ac{index}": ["42.158.0.1", CHECK["PUBLIC_IPS"][node]] for index, node in enumerate(("relay3", "relay4", "relay5"))} From e1754649fd2855e272d3ba0af8fbb0d4ad5ae712 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:52:10 +0200 Subject: [PATCH 33/49] docs: record merged Overmorrow preparation and live cache follow-up --- docs/IMPLEMENTATION_STATUS.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index ef28d7edd..7faef618d 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,9 +29,9 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Exercise late complementary-provider recovery after the now-proven concurrent three-provider transfers. | The standalone run on `a5e698cc` passes native and HTTPS bulk-overlap checks after bounded readiness, without overriding resource protection. It then stops because the control observer rejects the new late-provider test interfaces before fetching. Correct that fixture allowlist and repeat the real trial; late recovery and requester cancellation remain unproven. The separate CI primary-Exit failure remains unresolved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Exercise late complementary-provider recovery after the now-proven concurrent three-provider transfers. | The standalone run on `a5e698cc` passes native and HTTPS bulk-overlap checks after bounded readiness, without overriding resource protection. It then stops because the control observer rejects the new late-provider test interfaces before fetching. The narrow fixture correction is in `c2fbcd8c` and a new standalone trial is running; late recovery and requester cancellation remain unproven. The separate CI primary-Exit failure remains unresolved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR8 aligns client/bridge deadlines and cancellation. The new closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows; component tests pass. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9 is merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows; an exact immutable input archive passes capture and verification. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the standalone @@ -141,6 +141,12 @@ provider checks, ten inert cancellation checks, seven late-recovery checks and eleven observer checks pass. This fixture correction still needs a new live late-recovery result; it changes neither production admission nor privacy policy. +A new standalone trial uses clean source +`c2fbcd8c7f14a4c917c2c7bfe721a0650e8d20c9`, with evidence reserved at +`.git/ci-evidence/content-provider-late-02.2GyayM`. It retains the four-GiB guest, +two-GiB host reserve and all existing acceptance gates, and makes no weather +source requests. Its result is pending; do not restart an active run. + The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first completed, single-offer discovery. A temporary provider-side gate is bounded to @@ -194,6 +200,26 @@ trial. A prior cold core fetch followed by UI use is not browser-triggered first-cold delivery. No SDK rebuild, renderer CDN, new scheduler, resource-limit increase or extra source request is introduced by this candidate. +Weather [PR9](https://github.com/VOLPAROSSA/volparossa-weather/pull/9) is merged +at `fe6eb4b92984ed5569bb2db7026414593fc392b8`. Its CI passes 120 core-contract, +13 Drizz.li, seven Overmorrow Node and 47 Overmorrow Python checks, plus CodeQL. +A stale-data test previously sampled the clock twice, occasionally exceeding +the thirty-minute validity limit by one millisecond. Its corrected single-clock +fixture retains both strict expiry rejection and stale-result rejection; no +production freshness rule changes. + +The actual profile-three archive is retained at +`.git/ci-evidence/weather-overmorrow-inputs-v3.2mYCGu/inputs.tar`: +176 files, 455,024,116 payload bytes and 455,191,552 archive bytes. +Its manifest SHA-256 is +`da01dec6735c3084c3affb596531e798ed9ae21cbe1d52cf146bfda4dc8c4f29`; +archive SHA-256 is +`1f7cb371d04c546e17267f6ff94a113534cd58e4057c415756efb1725dc0fe63`. +Capture, runtime eligibility, unpacking and exact-tree verification pass against +private, byte-verified source copies and the existing pinned decoder. No runtime +or origin executes during capture. This establishes reproducible trial inputs, +not successful joined delivery or mobile execution. + Weather [PR8](https://github.com/VOLPAROSSA/volparossa-weather/pull/8) is merged into `main` at `ce1e28696d47808bd2908242325d1da478cbf0a1`. Each Overmorrow product now has one 195-second deadline including local abort/drain, with up to five bounded From a03c3c8fc865b8e1dfbf3caab745bcc27918fd4c Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:17:14 +0200 Subject: [PATCH 34/49] test: repair late provider gate and retain failure stage --- docs/IMPLEMENTATION_STATUS.md | 86 +++++++++++++++++-- .../content-provider-late-recovery.py | 55 +++++++++++- tests/integration/run-alpha-topology-vm.sh | 2 +- .../test-content-provider-late-recovery.py | 74 ++++++++++++++-- .../test-content-provider-smoke.py | 11 ++- 5 files changed, 206 insertions(+), 22 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 7faef618d..9e4b9bda0 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,14 +29,14 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Exercise late complementary-provider recovery after the now-proven concurrent three-provider transfers. | The standalone run on `a5e698cc` passes native and HTTPS bulk-overlap checks after bounded readiness, without overriding resource protection. It then stops because the control observer rejects the new late-provider test interfaces before fetching. The narrow fixture correction is in `c2fbcd8c` and a new standalone trial is running; late recovery and requester cancellation remain unproven. The separate CI primary-Exit failure remains unresolved. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9 is merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows; an exact immutable input archive passes capture and verification. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Exercise late complementary-provider recovery after the now-proven concurrent three-provider transfers. | The latest standalone run on `c2fbcd8c` again passes native and HTTPS bulk-overlap checks without overriding resource protection. It reaches the corrected control observer, then fails before starting the late fetch: the temporary gate has invalid nftables syntax. The gate separator and failure-stage retention are now corrected and pass targeted checks; repeat the actual trial. Late recovery and requester cancellation remain unproven; the earlier hosted-CI primary-Exit failure has not been revalidated on current source. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Weather PR10 fixes cleanup of rejected HTTP responses without retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows. Browser08 passes actual synthetic UI checks with the corrected ECMWF module, and a new immutable archive verifies against that proof. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the standalone provider proof, which needs no weather downloads, after correcting the late-phase -capture setup; successful concurrent reconstruction does not prove late discovery +gate setup; successful concurrent reconstruction does not prove late discovery convergence. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM @@ -141,11 +141,40 @@ provider checks, ten inert cancellation checks, seven late-recovery checks and eleven observer checks pass. This fixture correction still needs a new live late-recovery result; it changes neither production admission nor privacy policy. -A new standalone trial uses clean source -`c2fbcd8c7f14a4c917c2c7bfe721a0650e8d20c9`, with evidence reserved at -`.git/ci-evidence/content-provider-late-02.2GyayM`. It retains the four-GiB guest, -two-GiB host reserve and all existing acceptance gates, and makes no weather -source requests. Its result is pending; do not restart an active run. +The next standalone trial on clean source +`c2fbcd8c7f14a4c917c2c7bfe721a0650e8d20c9` also passes the separate adaptive +validator: each transfer reconstructs 15,728,640 bytes in sixty chunks from three +providers without origin body bytes or range requests. Native bulk overlap is +4,424,226,702 ns; cooperative-HTTPS overlap is 4,574,166,917 ns. The corrected +late-provider observer starts, but the run then fails at +`content-provider-late-recovery-fetch` with `CONTENT_PROVIDER_LATE_FETCH_FAILED`. +No late fetch process or gate-installed receipt is produced. The exact generated +gate is rejected by the nftables parser: the chain's closing brace has no +separator before the table's closing brace. This is a fixture setup failure, +not evidence that late rediscovery succeeded or failed during transfer. + +Evidence is retained under `.git/ci-evidence/content-provider-late-02.2GyayM/output/`. +The separate adaptive report has SHA-256 +`c54d9aac85194fa6a3894978c368e39e7ad7114056596766e7d8e46a8ed468a9`; +the failed aggregate report has SHA-256 +`ba0104fcf4c84387bc3695fb72c3a640298f62be5da19506c32b250c20e9aa6f`. +Cleanup leaves zero owned objects and unchanged guest-root network state. +The runner, VM and owned scratch overlay are gone; development-host DNS and +IPv4/IPv6 route hashes match their baseline. Host-firewall comparison remains +unavailable. No weather source was queried. Late recovery and requester +cancellation remain open. + +The gate now separates the nested closing braces. Its exact old bytes fail the +real nftables userspace parser; the corrected bytes pass parsing, then encounter +the expected unprivileged netlink refusal. That check installs no rule and does +not prove the live gate. Nine focused coordinator tests pass. Failures retain a +closed stage/category and bounded exit code before cleanup, never raw exception +text, commands or child stderr. The emergency collector also prioritizes that +one fixed JSON filename within its unchanged size and privacy limits. Nineteen +provider checks and ten inert cancellation checks pass, including bounded +retention and rejected oversized/foreign-scenario evidence. The six-second gate, +three-hundred-second transfer, gate-first cleanup and production behavior are +unchanged; the actual late-provider trial is still required. The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first @@ -208,6 +237,24 @@ the thirty-minute validity limit by one millisecond. Its corrected single-clock fixture retains both strict expiry rejection and stale-result rejection; no production freshness rule changes. +Weather [PR10](https://github.com/VOLPAROSSA/volparossa-weather/pull/10) is merged +at `88e5405e0fdd23a22a0e69999f6926d177a0b84e`. ECMWF now aborts its owned request +and cancels unread response bodies after rejected status or headers; a stalled +stream-cancellation promise cannot prolong the request deadline. Exact status, +range and size checks, source permissions and the thirty-second deadline remain +unchanged. Fifty-two focused ECMWF, wind and precipitation tests pass, alongside +independent review and all PR checks. No new source request was made to test this +cleanup correction. + +The actual synthetic Overmorrow browser trial now also passes with that updated +module. `build/overmorrow-browser-08/report.json` has SHA-256 +`b5efb8726a4e0215b08958f65f214a475b303d1b2ebf19bd8a49a47d14c41875`. +It retains the same sixty-one deployed files and exercises ten requests per +product, pointer refresh, independent missing/stale/expiry states and complete +browser/fixture cleanup. This is a new UI proof using the existing release build, +not a new SDK build, real-origin acquisition, core-cache delivery or native-mobile +result. Browser07 and its historical proof remain intact. + The actual profile-three archive is retained at `.git/ci-evidence/weather-overmorrow-inputs-v3.2mYCGu/inputs.tar`: 176 files, 455,024,116 payload bytes and 455,191,552 archive bytes. @@ -219,6 +266,24 @@ Capture, runtime eligibility, unpacking and exact-tree verification pass against private, byte-verified source copies and the existing pinned decoder. No runtime or origin executes during capture. This establishes reproducible trial inputs, not successful joined delivery or mobile execution. +This archive predates PR10 and binds Browser07's older ECMWF module. Do not +rewrite it or reuse its proof as evidence for the corrected module; a new capture +must bind Browser08 and the corresponding current verification record. + +That updated record is merged in Weather +[PR11](https://github.com/VOLPAROSSA/volparossa-weather/pull/11), at +`513e01c641f988b6c3a5d9879acdad7e7ca7e309`, after passing all CI checks. +It changes only the report and two semantic hashes from the actual Browser08 +trial; original build facts and unproven-delivery flags remain unchanged. +The new immutable archive is +`.git/ci-evidence/weather-overmorrow-inputs-v3-cleanup.b8FXQv/inputs.tar`: +176 files, 455,024,391 payload bytes and 455,192,064 archive bytes. +Its manifest SHA-256 is +`b724a30c21f3a3915d44841b1baa7f19f48e05e7e46acbc4f443bd6ac1676148`; +archive SHA-256 is +`a7fa0c2bf4f179babc77adefb650a4d0b9b3da4f354428598106e6350ed705b9`. +Capture, runtime eligibility, unpacking and exact-tree verification all pass; +none executes the runtime or establishes new origin or joined delivery evidence. Weather [PR8](https://github.com/VOLPAROSSA/volparossa-weather/pull/8) is merged into `main` at `ce1e28696d47808bd2908242325d1da478cbf0a1`. Each Overmorrow product now @@ -743,6 +808,9 @@ provider-index propagation. Neither a longer readiness wait nor substituting the second Exit is an established fix. The previous pre-VM fixture failure in [run 38162824767](https://github.com/VOLPAROSSA/volparossa/actions/runs/38162824767) is fixed by `c8b57d94`; that correction does not resolve either live failure. +This is a historical hosted result on `c8b57d94`, not a check of the current +branch head. Later local runs reach the provider inventory, but do not establish +that the hosted environment now passes it. The candidate now retries completed capability snapshots when verified chunks are missing: at most three generic discovery rounds and sixteen unique provider diff --git a/tests/integration/content-provider-late-recovery.py b/tests/integration/content-provider-late-recovery.py index 5bb609cb5..3325fca0e 100644 --- a/tests/integration/content-provider-late-recovery.py +++ b/tests/integration/content-provider-late-recovery.py @@ -26,6 +26,38 @@ SHA = hashlib.sha256(b"".join(bytes([16+i])*262144 for i in range(8)) + bytes([25])*123).hexdigest() EVENTS = ("CONTENT_DISCOVERY_COMPLETED", "CONTENT_DISCOVERY_RECEIVED_OFFERS", "CONTENT_DISCOVERY_RECEIVED_EMPTY", "CONTENT_PROVIDER_QUERY_STARTED", "CONTENT_PROVIDER_OFFER_VERIFIED") +FAILURE_STAGES = ("gate-install", "gate-inspect", "fetch-start", "first-snapshot", "activate-b", + "gate-release", "fetch-wait", "final-snapshot", "output") +FAILURE_CATEGORIES = ("command-failed", "timeout", "interrupted", "os-error", "invalid-evidence") + + +class FixtureCommandError(ValueError): + def __init__(self, returncode): + super().__init__("bounded fixture command failed") + self.returncode = returncode + + +def failure_evidence(stage, error): + category, code = "invalid-evidence", None + if isinstance(error, FixtureCommandError): category, code = "command-failed", error.returncode + elif isinstance(error, subprocess.TimeoutExpired): category = "timeout" + elif isinstance(error, InterruptedError): category = "interrupted" + elif isinstance(error, OSError): category = "os-error" + value = dict(schema_version=1, kind="content-provider-late-recovery-failure", success=False, + stage=stage, category=category, command_exit_code=code) + validate_failure(value) + return value + + +def validate_failure(value): + require(set(value) == {"schema_version", "kind", "success", "stage", "category", "command_exit_code"} + and type(value["schema_version"]) is int and value["schema_version"] == 1 + and value["kind"] == "content-provider-late-recovery-failure" and value["success"] is False + and value["stage"] in FAILURE_STAGES and value["category"] in FAILURE_CATEGORIES, + "invalid fixture failure diagnostic") + code = value["command_exit_code"] + require((value["category"] == "command-failed" and type(code) is int and -255 <= code <= 255 and code != 0) + or (value["category"] != "command-failed" and code is None), "invalid command failure status") def path(work, suffix, extension="json"): @@ -72,13 +104,14 @@ def command(args, deadline, input_data=None): require(remaining > 0, "fixture deadline exceeded") result = subprocess.run(args, input=input_data, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=min(remaining, 2), check=False) - require(result.returncode == 0 and len(result.stdout) <= LIMIT, "bounded fixture command failed") + if result.returncode != 0: raise FixtureCommandError(result.returncode) + require(len(result.stdout) <= LIMIT, "bounded fixture command output too large") return result.stdout def gate_rules(): return (f"table inet {TABLE} {{ chain input {{ type filter hook input priority -30; policy accept; " - 'iifname "r4x" ip saddr 46.162.3.1 ip daddr 49.165.5.1 tcp dport 18080 counter drop; } }\n').encode() + 'iifname "r4x" ip saddr 46.162.3.1 ip daddr 49.165.5.1 tcp dport 18080 counter drop; }\n}\n').encode() def validate_gate(value): @@ -200,12 +233,15 @@ def snapshot(baseline, deadline): gate_start = time.monotonic_ns(); deadline = gate_start + GATE_NS write(path(work, "gate", "state"), dict(namespace=ns)) process, success = None, False + stage = "gate-install" def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") handlers = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} try: command(["ip", "netns", "exec", ns["name"], "nft", "-f", "-"], deadline, gate_rules()) + stage = "gate-inspect" installed = json.loads(command(["ip", "netns", "exec", ns["name"], "nft", "-j", "list", "table", "inet", TABLE], deadline)) validate_gate(installed); write(path(work, "gate-installed"), installed) + stage = "fetch-start" started = time.monotonic_ns() with path(work, "fetch").open("xb") as stdout, path(work, "fetch", "err").open("xb") as stderr: os.fchmod(stdout.fileno(), 0o600); os.fchmod(stderr.fileno(), 0o600) @@ -213,6 +249,7 @@ def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") "--publisher-key", publication["publisher_hex"], "--cache", str(native / "cache"), "--output", str(native / "object.bin")), stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr) write(path(work, "fetch", "state"), process_identity(process.pid)) + stage = "first-snapshot" while True: require(process.poll() is None, "fetch ended before late activation") initial = snapshot(baseline, deadline) @@ -222,6 +259,7 @@ def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") time.sleep(0.02) observed = time.monotonic_ns() write(path(work, "first-snapshot"), initial) + stage = "activate-b" activated = time.monotonic_ns() serve = json.loads(command(cli("relay5", "content", "serve", "--manifest", str(native / "manifest.bin"), "--publisher-key", publication["publisher_hex"], "--cache", str(work / "state-relay5/content-late-recovery/cache"), @@ -229,13 +267,17 @@ def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") require(serve["serving"] is True and serve["publications"] == 1 and serve["replication_enabled"] is False, "late service did not acknowledge registration") acknowledged = time.monotonic_ns(); write(path(work, "relay5-serve"), serve) + stage = "gate-release" command(["ip", "netns", "exec", ns["name"], "nft", "delete", "table", "inet", TABLE], deadline) released = time.monotonic_ns(); require(released <= deadline, "gate exceeded six seconds") - require(process.wait(timeout=max(0.001, (started+300_000_000_000-time.monotonic_ns())/1e9)) == 0, - "native late recovery failed") + stage = "fetch-wait" + status = process.wait(timeout=max(0.001, (started+300_000_000_000-time.monotonic_ns())/1e9)) + if status != 0: raise FixtureCommandError(status) completed = time.monotonic_ns() + stage = "final-snapshot" final = snapshot(baseline, time.monotonic_ns()+5_000_000_000) write(path(work, "final-snapshot"), final) + stage = "output" output = native / "object.bin" require(output.is_file() and not output.is_symlink() and output.stat().st_size == BYTES, "wrong output") write(path(work, "object"), dict(bytes=output.stat().st_size, sha256=hashlib.sha256(output.read_bytes()).hexdigest(), @@ -244,6 +286,11 @@ def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") late_activation_acknowledged_ns=acknowledged, gate_released_ns=released, fetch_completed_ns=completed, cold_cache=True, consumer_provider_hints=False, origin_fallback=False)) success = True + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: + # Retain only fixed classifications before cleanup can raise a second error. + # No arbitrary command, path, exception text or child stderr is exported. + write(path(work, "failure"), failure_evidence(stage, error)) + raise finally: gate_removed = False try: diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index f7defc87a..e4dbdce62 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1115,7 +1115,7 @@ WEATHER_PRODUCTS_FAILURE_NAMES = {f"weather-products-delivery-{suffix}.json" for "acquisition-failure", "export-status", "tree-before", "input-cleanup")} WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", *WEATHER_CLIENT_HELPERS) LATE_PROVIDER_NAMES = {f"content-provider-late-recovery-{suffix}.json" for suffix in ( - "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence")} + "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure")} def joined_client_helper(work, names, entrypoint): diff --git a/tests/integration/test-content-provider-late-recovery.py b/tests/integration/test-content-provider-late-recovery.py index 13cfe05b4..091e71e19 100644 --- a/tests/integration/test-content-provider-late-recovery.py +++ b/tests/integration/test-content-provider-late-recovery.py @@ -5,7 +5,10 @@ import json from pathlib import Path import runpy +import shutil +import subprocess import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -134,6 +137,47 @@ def test_gate_can_only_drop_exact_exit_to_a_tcp_content_not_udp_discovery(self): target[keys[-1]] = changed with self.assertRaises(ValueError): CHECK["validate_gate"](value) + def test_gate_rules_separate_nested_blocks_and_pass_read_only_nft_parser(self): + rules = CHECK["gate_rules"]() + self.assertTrue(rules.endswith(b"counter drop; }\n}\n")) + nft = shutil.which("nft", path="/usr/sbin:/usr/bin:/sbin:/bin") + if nft is None: self.skipTest("nft userspace parser is not installed") + broken = rules.replace(b"counter drop; }\n}\n", b"counter drop; } }\n") + def parse(raw): + # Check-only never commits a rule. An unprivileged host may refuse + # netlink initialization after parsing; that is not installed-gate proof. + return subprocess.run([nft, "--check", "--file", "-"], input=raw, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3, check=False, + env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL": "C"}) + before, after = parse(broken), parse(rules) + self.assertNotEqual(before.returncode, 0) + self.assertIn(b"syntax error", before.stderr) + self.assertNotIn(b"syntax error", after.stderr) + if after.returncode: + self.assertEqual(after.stderr.strip(), b"netlink: Error: cache initialization failed: Operation not permitted") + + def test_failure_evidence_is_closed_and_never_exports_exception_or_command_text(self): + cases = ((CHECK["FixtureCommandError"](2), "command-failed", 2), + (subprocess.TimeoutExpired(["private-command"], 2, stderr=b"secret"), "timeout", None), + (InterruptedError("secret"), "interrupted", None), (OSError("secret"), "os-error", None), + (ValueError("secret"), "invalid-evidence", None)) + for error, category, code in cases: + value = CHECK["failure_evidence"]("gate-install", error) + self.assertEqual(value, dict(schema_version=1, kind="content-provider-late-recovery-failure", + success=False, stage="gate-install", category=category, command_exit_code=code)) + self.assertNotIn("secret", json.dumps(value)); self.assertNotIn("private-command", json.dumps(value)) + original = CHECK["failure_evidence"]("gate-install", CHECK["FixtureCommandError"](2)) + for field, changed in (("schema_version", True), ("success", True), ("stage", "arbitrary-path"), + ("category", "raw-error"), ("command_exit_code", True), ("command_exit_code", 0), + ("command_exit_code", 256), ("command_exit_code", -256), ("stderr", "secret")): + value = {**original, field: changed} + with self.assertRaises(ValueError): CHECK["validate_failure"](value) + with patch.object(CHECK["subprocess"], "run", return_value=SimpleNamespace(returncode=2, stdout=b"secret")): + with self.assertRaises(CHECK["FixtureCommandError"]) as caught: + CHECK["command"](["unused-inert-command"], CHECK["time"].monotonic_ns() + 1_000_000_000) + self.assertEqual(caught.exception.returncode, 2) + self.assertNotIn("secret", str(caught.exception)) + def test_log_snapshots_are_bounded_and_ignore_preexisting_or_noncontent_events(self): raw = b"1000 INFO event=CONTENT_DISCOVERY_COMPLETED\n1001 INFO event=CONTENT_DISCOVERY_RECEIVED_OFFERS\n" self.assertEqual(CHECK["event_rows"](raw, 1000), [dict(unix_ms=1001, event="CONTENT_DISCOVERY_RECEIVED_OFFERS")]) @@ -155,7 +199,7 @@ def gate(_work): events.append("gate"); raise ValueError("gate failed") with tempfile.TemporaryDirectory() as temporary: CHECK["cleanup"](Path(temporary)) def test_coordinator_activates_b_only_after_snapshot_and_always_releases_owned_gate(self): - for failure in (None, "snapshot", "activation"): + for failure in (None, "gate-install", "gate-inspect", "snapshot", "activation"): with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary: work = Path(temporary); native = work / "state-client/content-late-recovery"; native.mkdir(parents=True) binary = work / "volparossa"; binary.touch() @@ -185,8 +229,13 @@ def spawn(args, **kwargs): def now(): clock[0] += 100_000_000; return clock[0] def execute(args, deadline, input_data=None): if args[-2:] == ["list", "tables"]: return b'{"nftables":[]}' - if args[-2:] == ["-f", "-"]: calls.append("gate-install"); return b"" - if "list" in args and "table" in args: return json.dumps(gate_fixture()).encode() + if args[-2:] == ["-f", "-"]: + calls.append("gate-install") + if failure == "gate-install": raise CHECK["FixtureCommandError"](2) + return b"" + if "list" in args and "table" in args: + if failure == "gate-inspect": raise ValueError("secret malformed gate") + return json.dumps(gate_fixture()).encode() if "delete" in args: calls.append("gate-release"); return b"" if "logs" in args: label = "client" if "runtime-client" in args[2] else "broker" @@ -201,10 +250,20 @@ def execute(args, deadline, input_data=None): if failure == "activation": raise ValueError("registration failed") return json.dumps(value["providers"]["relay5"]["serve"]).encode() self.fail(f"unexpected command {args}") + def release(_work, **_kwargs): + calls.append("cleanup-gate") + diagnostic = CHECK["path"](work, "failure") + self.assertEqual(diagnostic.exists(), failure is not None) + if failure: + record = CHECK["read"](diagnostic); CHECK["validate_failure"](record) + stage = {"snapshot": "first-snapshot", "activation": "activate-b"}.get(failure, failure) + self.assertEqual(record["stage"], stage) + self.assertEqual(record["command_exit_code"], 2 if failure == "gate-install" else None) + self.assertNotIn("secret", diagnostic.read_text()) with patch.dict(CHECK["run"].__globals__, guest=lambda work: None, namespace=lambda work: dict(name="va-00000000-r4", identity="1:2"), command=execute, process_identity=lambda pid: dict(pid=pid, start_ticks=100), - remove_gate=lambda work, **kwargs: calls.append("cleanup-gate")), \ + remove_gate=release), \ patch.object(CHECK["subprocess"], "Popen", side_effect=spawn), \ patch.object(CHECK["time"], "monotonic_ns", side_effect=now), \ patch.object(CHECK["time"], "time_ns", return_value=1_000_000_000), \ @@ -213,14 +272,17 @@ def execute(args, deadline, input_data=None): with self.assertRaises(ValueError): CHECK["run"](work, binary, "va-00000000-r4") else: CHECK["run"](work, binary, "va-00000000-r4") self.assertIn("cleanup-gate", calls) - self.assertIsNotNone(process.returncode) + if failure in ("gate-install", "gate-inspect"): + self.assertNotIn("fetch-start", calls) + else: self.assertIsNotNone(process.returncode) cleanup = CHECK["read"](CHECK["path"](work, "run-cleanup")) self.assertEqual(cleanup, dict(fetch_reaped=True, gate_removed=True, operation_completed=failure is None)) if failure == "snapshot": self.assertNotIn("activate-b", calls) elif failure is None: self.assertLess(calls.index("activate-b"), calls.index("gate-release")) self.assertLess(calls.index("gate-release"), calls.index("fetch-wait")) - if failure: self.assertLess(calls.index("cleanup-gate"), calls.index("fetch-terminate")) + if failure in ("snapshot", "activation"): + self.assertLess(calls.index("cleanup-gate"), calls.index("fetch-terminate")) def test_builder_requires_exact_original_snapshots_and_disconnected_route(self): with tempfile.TemporaryDirectory() as temporary: diff --git a/tests/integration/test-content-provider-smoke.py b/tests/integration/test-content-provider-smoke.py index 2b2619bd4..3bfb6a458 100644 --- a/tests/integration/test-content-provider-smoke.py +++ b/tests/integration/test-content-provider-smoke.py @@ -257,8 +257,8 @@ def test_late_emergency_metadata_survives_bounds_without_claiming_success(self): api = {"__name__": "inert_late_collector"} exec(compile(embedded, "late-collector", "exec"), api) self.assertEqual(api["LATE_PROVIDER_NAMES"], {f"content-provider-late-recovery-{suffix}.json" - for suffix in ("first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence")}) - for scenario in ("content-provider", "content"): + for suffix in ("first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure")}) + for scenario, oversized_failure in (("content-provider", False), ("content-provider", True), ("content", False)): with tempfile.TemporaryDirectory() as directory: base = Path(directory) home, opt, proc, cgroups = (base / name for name in ("home", "opt", "proc", "cgroups")) @@ -270,6 +270,10 @@ def test_late_emergency_metadata_survives_bounds_without_claiming_success(self): # scan before the later-named recovery proof was reached. for index in range(60): (work / f"content-before-{index:02}.json").write_text("{}") for name in api["LATE_PROVIDER_NAMES"]: (work / name).write_text('{"observed":true}') + failure_name = "content-provider-late-recovery-failure.json" + failure_raw = json.dumps(dict(schema_version=1, kind="content-provider-late-recovery-failure", + success=False, stage="gate-install", category="command-failed", command_exit_code=2)).encode() + (work / failure_name).write_bytes(b" " * (api["FILE_LIMIT"] + 1) if oversized_failure else failure_raw) oversized = "content-provider-late-recovery-evidence.json" (work / oversized).write_bytes(b" " * (api["FILE_LIMIT"] + 1)) alias = "content-provider-late-recovery-cleanup.json" @@ -283,7 +287,10 @@ def test_late_emergency_metadata_survives_bounds_without_claiming_success(self): files = {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()} selected = {name.rsplit("/", 1)[-1] for name in files if "late-recovery" in name} expected = api["LATE_PROVIDER_NAMES"] - {oversized, alias} if scenario == "content-provider" else set() + if oversized_failure: expected.discard(failure_name) self.assertEqual(selected, expected) + if failure_name in selected: + self.assertEqual(files[f"work-1/{failure_name}"], failure_raw) self.assertFalse(any(b"PRIVATE" in value for value in files.values())) summary = json.loads(files["vm-incomplete.json"]) self.assertIs(summary["success"], False) From 987c64c41d98eb0f50d86b631da0e13a507a549b Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:19:23 +0200 Subject: [PATCH 35/49] docs: record corrected late provider trial in progress --- docs/IMPLEMENTATION_STATUS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 9e4b9bda0..b04c771b7 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -176,6 +176,13 @@ retention and rejected oversized/foreign-scenario evidence. The six-second gate, three-hundred-second transfer, gate-first cleanup and production behavior are unchanged; the actual late-provider trial is still required. +A new standalone trial is running on clean source +`a03c3c8fc865b8e1dfbf3caab745bcc27918fd4c`, with evidence at +`.git/ci-evidence/content-provider-late-03.nV7eQv`. Its resource preflight passes +without reducing the four-GiB guest or two-GiB host reserve; the pinned VM is +provisioning. No weather source acquisition or joined Weather bundle is included. +Await the retained terminal result and cleanup before starting another heavy job. + The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first completed, single-offer discovery. A temporary provider-side gate is bounded to From 23512882a94b21c829bad19a907c74f551bdb070 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:06:18 +0200 Subject: [PATCH 36/49] test: accept disjoint late provider discovery snapshots --- .../content-provider-late-recovery.py | 5 ++- .../test-content-provider-late-recovery.py | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/integration/content-provider-late-recovery.py b/tests/integration/content-provider-late-recovery.py index 3325fca0e..cccb3e34b 100644 --- a/tests/integration/content-provider-late-recovery.py +++ b/tests/integration/content-provider-late-recovery.py @@ -340,8 +340,11 @@ def validate(value): validate_gate(value["gate"]) require(first_snapshot(value["first_snapshot"]), "first generic snapshot was not one verified offer") final = value["final_snapshot"] + # Snapshots need not repeat A: one verified A followed by one verified B is + # sufficient. The activation, exact useful suppliers and shard/hash checks + # independently prove recovery; an offer count alone never proves a new peer. require(2 <= count(final["client"], EVENTS[0]) <= 3 and count(final["client"], EVENTS[1]) >= 2 - and 2 <= count(final["broker"], EVENTS[3]) <= 3 and 3 <= count(final["broker"], EVENTS[4]) <= 48, + and 2 <= count(final["broker"], EVENTS[3]) <= 3 and 2 <= count(final["broker"], EVENTS[4]) <= 48, "bounded genuine rediscovery not proven") for label in ("client", "broker"): require(final[label][:len(value["first_snapshot"][label])] == value["first_snapshot"][label], "snapshot lineage changed") diff --git a/tests/integration/test-content-provider-late-recovery.py b/tests/integration/test-content-provider-late-recovery.py index 091e71e19..ae5f190e5 100644 --- a/tests/integration/test-content-provider-late-recovery.py +++ b/tests/integration/test-content-provider-late-recovery.py @@ -102,6 +102,15 @@ def raw_files(value): return files +def disjoint_snapshot_fixture(): + value = fixture() + # The broker records generic verified-offer events, not peer identifiers. + # Keep the two actual suppliers and complementary payload proof, but model + # only one offer in each completed snapshot: first {A}, then {B}, not {A,B}. + value["final_snapshot"]["broker"].pop() + return value + + class LateRecovery(unittest.TestCase): def test_exact_late_phase_is_independent_of_payload_overlap(self): for broker in ("relay0", "relay1", "relay2"): @@ -110,6 +119,38 @@ def test_exact_late_phase_is_independent_of_payload_overlap(self): CHECK["validate"](value) self.assertNotEqual(CHECK["SHA"], CHECK["BASE"]["SHA"]) + def test_disjoint_first_a_then_b_snapshots_do_not_require_a_repeated_offer(self): + value = disjoint_snapshot_fixture() + self.assertEqual(CHECK["count"](value["first_snapshot"]["broker"], CHECK["EVENTS"][4]), 1) + self.assertEqual(CHECK["count"](value["final_snapshot"]["broker"], CHECK["EVENTS"][4]), 2) + self.assertEqual(CHECK["count"](value["final_snapshot"]["broker"], CHECK["EVENTS"][3]), 2) + self.assertEqual(CHECK["count"](value["final_snapshot"]["client"], CHECK["EVENTS"][0]), 2) + CHECK["validate"](value) + + def test_two_completed_snapshots_with_only_one_verified_offer_still_fail(self): + value = disjoint_snapshot_fixture() + self.assertEqual(value["final_snapshot"]["broker"].pop()["event"], CHECK["EVENTS"][4]) + self.assertEqual(CHECK["count"](value["final_snapshot"]["broker"], CHECK["EVENTS"][4]), 1) + with self.assertRaisesRegex(ValueError, "bounded genuine rediscovery not proven"): + CHECK["validate"](value) + + def test_two_offers_still_require_actual_late_b_and_complementary_delivery(self): + original = disjoint_snapshot_fixture() + mutations = [ + (("fetch", "provider_peer_ids"), [original["expected_peers"]["relay4"]]*2), + (("privacy", "exit", "provider_application", "relay5", "response_payload_bytes"), 0), + (("providers", "relay5", "serve", "serving"), False), + (("output", "sha256"), "0"*64), + (("output", "late_activation_started_ns"), 1), + (("output", "fetch_completed_ns"), 400_000_000_000), + ] + for keys, changed in mutations: + with self.subTest(keys=keys): + value = copy.deepcopy(original); target = value + for key in keys[:-1]: target = target[key] + target[keys[-1]] = changed + with self.assertRaises(ValueError): CHECK["validate"](value) + def test_rejects_no_rediscovery_early_b_warm_cache_unprotected_paths_and_bad_cleanup(self): mutations = [(("first_snapshot", "broker"), []), (("final_snapshot", "client"), fixture()["first_snapshot"]["client"]), (("output", "cold_cache"), False), (("output", "consumer_provider_hints"), True), From ebf0d077f410042d8160c627fe6bc27fe9ceda8a Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:06:38 +0200 Subject: [PATCH 37/49] fix: resume complementary cache coverage after resource pressure --- .../volparossa-agent/src/content/parallel.rs | 170 +++++++-- .../src/content/parallel/adaptive_tests.rs | 18 +- .../parallel/adaptive_tests/continuation.rs | 333 ++++++++++++++++++ .../src/content/parallel/continuation.rs | 100 ++++++ .../src/content/rediscovery/tests.rs | 8 +- .../src/content/worker_budget.rs | 2 +- .../src/content/worker_budget/diagnostics.rs | 6 +- .../src/transfer/parallel.rs | 149 +++++++- .../src/transfer/parallel/adaptive.rs | 3 +- .../src/transfer/parallel/tests/adaptive.rs | 60 +++- .../parallel/tests/adaptive/retirement.rs | 197 +++++++++++ docs/IMPLEMENTATION_STATUS.md | 78 +++- 12 files changed, 1068 insertions(+), 56 deletions(-) create mode 100644 crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs create mode 100644 crates/volparossa-agent/src/content/parallel/continuation.rs create mode 100644 crates/volparossa-content/src/transfer/parallel/tests/adaptive/retirement.rs diff --git a/crates/volparossa-agent/src/content/parallel.rs b/crates/volparossa-agent/src/content/parallel.rs index 44ecf0ae9..82fc8d94f 100644 --- a/crates/volparossa-agent/src/content/parallel.rs +++ b/crates/volparossa-agent/src/content/parallel.rs @@ -11,6 +11,7 @@ use tokio::time::{Instant, timeout_at}; #[cfg(test)] mod adaptive_tests; +mod continuation; mod drivers; #[cfg(test)] mod tests; @@ -20,7 +21,7 @@ use volparossa_content::{ provider::{ProviderError, pull_publication_worker}, transfer::{ TransferError, TransferLimits, TransferProgress, - parallel::{ChunkWorker, ParallelDownload}, + parallel::{ChunkWorker, ParallelDownload, ResourceRetired}, }, }; @@ -159,6 +160,17 @@ pub(super) async fn pull_indexed_with_budget( pull_inner(context, manifest, store, policy, sources, Some(deadline)).await } +fn distinct_sources(providers: Vec) -> Result, ContentError> { + if providers.len() > 16 { + return Err(ContentError::Invalid); + } + let mut seen = BTreeSet::new(); + Ok(providers + .into_iter() + .filter(|source| seen.insert(source.provider.peer_id)) + .collect()) +} + async fn pull_inner( context: &ControlContext, manifest: &VerifiedManifest, @@ -167,14 +179,7 @@ async fn pull_inner( providers: Vec, deadline: Option, ) -> Result { - if providers.len() > 16 { - return Err(ContentError::Invalid); - } - let mut seen = BTreeSet::new(); - let providers = providers - .into_iter() - .filter(|source| seen.insert(source.provider.peer_id)) - .collect::>(); + let providers = distinct_sources(providers)?; if providers.is_empty() || complete(manifest, store)? { return Ok(IndexedReceipt::default()); } @@ -213,14 +218,37 @@ async fn pull_inner( ) }) .collect(); - let (progress, measurements, timed_out) = Box::pin(receive_workers( - download, - store, - drivers, - || budget.allowance(active.load(Ordering::Acquire)), + let (mut progress, measurements, mut timed_out, retired) = + Box::pin(receive_workers_retired( + download, + store, + drivers, + || budget.allowance(active.load(Ordering::Acquire)), + deadline, + )) + .await?; + let resumed = continuation::RetiredBatch { + manifest, + progress: &mut progress, + measurements: &measurements, + retired, deadline, - )) + } + .resume( + store, + &ProtectedContinuations { + context, + policy, + manifest, + providers: &providers, + budget: &budget, + active: &active, + deadline, + scope, + }, + ) .await?; + timed_out |= resumed.timed_out; let elapsed = observed_at.elapsed(); if timed_out { content_event(context, "CONTENT_PROVIDER_TRANSFER_FAILED").await; @@ -238,7 +266,9 @@ async fn pull_inner( observed_at, observed_wall, elapsed, - valid: !timed_out && same_route, + // Serial recovery is useful authenticated progress, not an ordinary clean + // concurrent-batch cost sample or a new throughput improvement claim. + valid: !timed_out && same_route && !resumed.attempted, }, ) .await @@ -249,6 +279,80 @@ async fn pull_inner( result } +struct ProtectedContinuations<'a> { + context: &'a ControlContext, + policy: &'a volparossa_policy::VerifiedManifest, + manifest: &'a VerifiedManifest, + providers: &'a [ProviderSource], + budget: &'a super::worker_budget::BatchBudget<'a>, + active: &'a AtomicUsize, + deadline: Instant, + scope: Option, +} + +impl continuation::Sources for ProtectedContinuations<'_> { + fn available(&self) -> bool { + self.budget.available_workers() > 0 + } + + fn transport(&self, index: usize) -> &VerifiedManifest { + self.providers[index] + .transport + .as_ref() + .unwrap_or(self.manifest) + } + + async fn receive( + &self, + index: usize, + download: ParallelDownload, + worker: ChunkWorker, + store: &mut ChunkStore, + ) -> Result<(TransferProgress, bool), ContentError> { + if self.scope.is_none() + || RecentProviderScope::for_route(self.context, self.policy).await != self.scope + { + return Err(ContentError::Policy); + } + let driver = drivers::assigned( + worker, + self.deadline, + self.active, + || self.budget.try_acquire(), + |mut worker| async move { + // The clean original session consumed the first transport attempt. + // This single continuation spends the last one; no nested reconnect. + Ok(matches!( + attempt( + self.context, + self.policy, + &self.providers[index].provider, + &mut worker, + ) + .await?, + Attempt::Complete + )) + }, + ); + let (progress, _, timed_out, retired) = receive_workers_retired( + download, + store, + vec![driver], + || self.budget.allowance(self.active.load(Ordering::Acquire)), + self.deadline, + ) + .await?; + if RecentProviderScope::for_route(self.context, self.policy).await != self.scope { + return Err(ContentError::Policy); + } + debug_assert!( + retired.is_empty(), + "continuations never renew retirement authority" + ); + Ok((progress[0], timed_out)) + } +} + fn prepare_download( manifest: &VerifiedManifest, store: &mut ChunkStore, @@ -396,6 +500,7 @@ async fn remember_measurement( } } +#[cfg(test)] async fn receive_workers>>( download: ParallelDownload, store: &mut ChunkStore, @@ -403,21 +508,42 @@ async fn receive_workers usize, deadline: Instant, ) -> Result<(Vec, Vec, bool), ContentError> { + receive_workers_retired(download, store, drivers, allowance, deadline) + .await + .map(|(progress, measurements, expired, _)| (progress, measurements, expired)) +} + +async fn receive_workers_retired>>( + download: ParallelDownload, + store: &mut ChunkStore, + drivers: Vec, + allowance: impl Fn() -> usize, + deadline: Instant, +) -> Result< + ( + Vec, + Vec, + bool, + Vec, + ), + ContentError, +> { let mut progress = vec![TransferProgress::default(); drivers.len()]; let write = timeout_at( deadline, - download.run_with_allowance(store, &mut progress, allowance), + download.run_with_allowance_retired(store, &mut progress, allowance), ); // Every future remains owned here: cancellation drops both writer and stream owners. // A failed peer is collected independently, so verified siblings can still finish. let (written, outcomes) = tokio::join!(write, drivers::collect(drivers)); - let timed_out = match written { - Ok(Ok(())) => false, - Ok(Err(TransferError::Timeout)) | Err(_) => true, + let (retired, expired) = match written { + Ok(Ok(retired)) => (retired, false), + Ok(Err(TransferError::Timeout)) | Err(_) => (Vec::new(), true), Ok(Err(_)) => return Err(ContentError::Invalid), - } || Instant::now() >= deadline; + }; + let timed_out = expired || Instant::now() >= deadline; let measurements = outcomes.into_iter().collect::, _>>()?; - Ok((progress, measurements, timed_out)) + Ok((progress, measurements, timed_out, retired)) } /// A transport/close failure cannot cancel a useful sibling or erase bytes the single diff --git a/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs b/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs index 621e27e89..02e38e668 100644 --- a/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs +++ b/crates/volparossa-agent/src/content/parallel/adaptive_tests.rs @@ -1,5 +1,7 @@ //! Actual v1 selector/chunk streams through the production assignment/lease driver gate. +mod continuation; + use std::{ sync::atomic::{AtomicUsize, Ordering}, time::Duration, @@ -27,6 +29,10 @@ struct Fixture { } fn fixture() -> Fixture { + fixture_with_disjoint(false) +} + +fn fixture_with_disjoint(disjoint: bool) -> Fixture { let directory = tempfile::tempdir().unwrap(); let limits = CacheLimits { max_bytes: 12 * CHUNK_BYTES as u64, @@ -59,10 +65,13 @@ fn fixture() -> Fixture { .unwrap(); let manifest = envelope.verify(&signer.verifying_key(), now()).unwrap(); let mut registries = Vec::new(); - for provider in 0..4 { + for provider in 0..if disjoint { 3 } else { 4 } { let root = directory.path().join(format!("provider-{provider}")); let mut store = ChunkStore::create(&root, limits).unwrap(); - for chunk in manifest.chunks() { + for (index, chunk) in manifest.chunks().iter().enumerate() { + if disjoint && index % 3 != provider { + continue; + } store .put_verified(*chunk.id(), &source.get(chunk.id()).unwrap().unwrap()) .unwrap(); @@ -272,9 +281,10 @@ async fn adaptive_worker_interruption_releases_observed_leases_before_batch_summ &active, || budget.try_acquire(), |worker| async { - let _worker = worker; started.notify_one(); - std::future::pending::>().await + let result = std::future::pending::>().await; + drop(worker); + result }, ) }) diff --git a/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs b/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs new file mode 100644 index 000000000..6cdf9a0b5 --- /dev/null +++ b/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs @@ -0,0 +1,333 @@ +//! The production serial coordinator and lease driver over actual disjoint v1 streams. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use tokio::{ + io::duplex, + sync::Notify, + time::{Duration, Instant}, +}; +use volparossa_content::{ + ChunkStore, VerifiedManifest, + provider::{pull_publication_worker, serve_publication}, + reassemble, + transfer::{ + TransferLimits, TransferProgress, + parallel::{ChunkWorker, ParallelDownload, ResourceRetired}, + }, +}; + +use super::{Fixture, fixture_with_disjoint}; +use crate::content::{ + ContentError, now, + parallel::{ + continuation::{RetiredBatch, Sources}, + drivers, receive_workers_retired, + }, + worker_budget::{BatchBudget, WorkerBudget}, +}; + +struct Prepared { + fixture: Fixture, + progress: Vec, + measurements: Vec, + retired: Vec, + deadline: Instant, +} + +async fn prepare() -> Prepared { + let mut fixture = fixture_with_disjoint(true); + let budget = WorkerBudget::for_test(3 * 32 * 8 * 1024 * 1024, 3 * 4 * 8, false); + let batch = budget.for_test_batch(); + let active = AtomicUsize::new(0); + let opened = AtomicUsize::new(0); + let allowance = AtomicUsize::new(3); + let limits = TransferLimits { + max_requests: 12, + max_bytes: fixture.bytes.len() as u64, + exchange_timeout: Duration::from_secs(5), + session_timeout: Duration::from_secs(10), + }; + let deadline = Instant::now() + limits.session_timeout; + let (download, workers) = + ParallelDownload::new_adaptive(&fixture.manifest, 3, &mut fixture.cache, limits, 3) + .unwrap(); + let opened = &opened; + let allowance = &allowance; + let tasks = workers + .into_iter() + .zip(&fixture.registries) + .map(|(worker, registry)| { + drivers::assigned( + worker, + deadline, + &active, + || batch.try_acquire(), + move |mut worker| async move { + if opened.fetch_add(1, Ordering::SeqCst) == 2 { + allowance.store(1, Ordering::SeqCst); + } + let (mut client, mut server) = duplex(4096); + let (received, sent) = tokio::join!( + pull_publication_worker(&mut client, &mut worker), + serve_publication(&mut server, registry, limits), + ); + received.map_err(|_| ContentError::Unavailable)?; + sent.map_err(|_| ContentError::Unavailable)?; + Ok(true) + }, + ) + }) + .collect(); + let (progress, measurements, expired, retired) = receive_workers_retired( + download, + &mut fixture.cache, + tasks, + || { + batch + .allowance(active.load(Ordering::Acquire)) + .min(allowance.load(Ordering::SeqCst)) + }, + deadline, + ) + .await + .unwrap(); + assert!(!expired); + assert_eq!(opened.load(Ordering::SeqCst), 3); + assert_eq!( + active.load(Ordering::SeqCst), + 0, + "join original leases before continuation" + ); + assert!(!retired.is_empty()); + assert!(progress.iter().map(|part| part.bytes).sum::() < fixture.bytes.len() as u64); + assert!( + measurements + .iter() + .all(|measurement| measurement.elapsed.is_some()) + ); + assert_eq!(batch.snapshot_for_test().unwrap()["peak_leases"], 3); + Prepared { + fixture, + progress, + measurements, + retired, + deadline, + } +} + +#[derive(Clone, Copy)] +enum Mode { + Complete, + Failed, + Cancel, +} + +struct ActualSources<'a> { + fixture_manifest: &'a VerifiedManifest, + registries: &'a [volparossa_content::provider::PublicationRegistry], + budget: &'a BatchBudget<'a>, + deadline: Instant, + active: AtomicUsize, + opened: [AtomicUsize; 3], + mode: Mode, + started: Notify, +} + +impl Sources for ActualSources<'_> { + fn available(&self) -> bool { + self.budget.available_workers() > 0 + } + fn transport(&self, _: usize) -> &VerifiedManifest { + self.fixture_manifest + } + async fn receive( + &self, + index: usize, + download: ParallelDownload, + worker: ChunkWorker, + store: &mut ChunkStore, + ) -> Result<(TransferProgress, bool), ContentError> { + assert_eq!(self.active.load(Ordering::SeqCst), 0, "serial continuation"); + let driver = drivers::assigned( + worker, + self.deadline, + &self.active, + || self.budget.try_acquire(), + |mut worker| async { + assert_eq!( + self.opened[index].fetch_add(1, Ordering::SeqCst), + 0, + "single-use retirement" + ); + assert_eq!(self.active.load(Ordering::SeqCst), 1); + if matches!(self.mode, Mode::Cancel) { + self.started.notify_one(); + let result = std::future::pending::>().await; + drop(worker); + return result; + } + let (mut client, mut server) = duplex(4096); + if matches!(self.mode, Mode::Failed) { + drop(server); + return pull_publication_worker(&mut client, &mut worker) + .await + .map(|()| true) + .map_err(|_| ContentError::Unavailable); + } + let (received, sent) = tokio::join!( + pull_publication_worker(&mut client, &mut worker), + serve_publication( + &mut server, + &self.registries[index], + TransferLimits::default() + ), + ); + received.map_err(|_| ContentError::Unavailable)?; + sent.map_err(|_| ContentError::Unavailable)?; + Ok(true) + }, + ); + let (progress, _, expired, retired) = receive_workers_retired( + download, + store, + vec![driver], + || self.budget.allowance(self.active.load(Ordering::Acquire)), + self.deadline, + ) + .await?; + assert!( + retired.is_empty(), + "a continuation cannot authorize another one" + ); + assert_eq!(self.active.load(Ordering::SeqCst), 0); + Ok((progress[0], expired)) + } +} + +#[derive(Clone, Copy)] +enum Case { + Complete, + NoResources, + Failed, + Expired, + Unclean, + Cancel, +} + +async fn exercise(case: Case) { + let Prepared { + mut fixture, + mut progress, + mut measurements, + retired, + deadline, + } = prepare().await; + let initial_bytes = progress.iter().map(|part| part.bytes).sum::(); + let retired_count = retired.len(); + let capacity = usize::from(!matches!(case, Case::NoResources)) as u64; + let budget = WorkerBudget::for_test(capacity * 32 * 8 * 1024 * 1024, capacity * 4 * 8, true); + let batch = budget.for_test_batch(); + if matches!(case, Case::Unclean) { + for measurement in &mut measurements { + measurement.elapsed = None; + } + } + let sources = ActualSources { + fixture_manifest: &fixture.manifest, + registries: &fixture.registries, + budget: &batch, + deadline, + active: AtomicUsize::new(0), + opened: std::array::from_fn(|_| AtomicUsize::new(0)), + mode: match case { + Case::Failed => Mode::Failed, + Case::Cancel => Mode::Cancel, + _ => Mode::Complete, + }, + started: Notify::new(), + }; + let retired = RetiredBatch { + manifest: &fixture.manifest, + progress: &mut progress, + measurements: &measurements, + retired, + deadline: if matches!(case, Case::Expired) { + Instant::now() + } else { + deadline + }, + }; + let mut work = Box::pin(retired.resume(&mut fixture.cache, &sources)); + if matches!(case, Case::Cancel) { + tokio::select! { + () = sources.started.notified() => {}, + _ = &mut work => panic!("continuation must remain owned until cancellation"), + } + assert_eq!(sources.active.load(Ordering::SeqCst), 1); + drop(work); + } else { + let outcome = work.as_mut().await.unwrap(); + assert_eq!( + outcome.attempted, + matches!(case, Case::Complete | Case::Failed) + ); + assert_eq!(outcome.timed_out, matches!(case, Case::Expired)); + drop(work); + } + assert_eq!(sources.active.load(Ordering::SeqCst), 0); + let opened = sources + .opened + .iter() + .map(|value| value.load(Ordering::SeqCst)) + .sum::(); + assert!(opened <= retired_count); + let diagnosis = batch.snapshot_for_test().unwrap(); + assert_eq!(diagnosis["active_leases"], 0); + assert!(diagnosis["peak_leases"].as_u64().unwrap() <= 1); + let mut output = Vec::new(); + let result = reassemble( + &fixture.manifest, + &mut [&mut fixture.cache], + now(), + &mut output, + ); + if matches!(case, Case::Complete) { + result.unwrap(); + assert_eq!(output, fixture.bytes); + assert_eq!( + progress.iter().map(|part| part.bytes).sum::(), + fixture.bytes.len() as u64 + ); + assert_eq!(progress.iter().map(|part| part.chunks).sum::(), 12); + assert_eq!(opened, retired_count); + } else { + assert!(result.is_err()); + assert_eq!( + progress.iter().map(|part| part.bytes).sum::(), + initial_bytes + ); + } +} + +#[tokio::test] +async fn production_continuation_completes_disjoint_coverage_after_dynamic_pressure() { + tokio::time::timeout(Duration::from_secs(15), exercise(Case::Complete)) + .await + .unwrap(); +} + +#[tokio::test] +async fn production_continuation_never_renews_failed_unclean_or_unavailable_work() { + for case in [ + Case::NoResources, + Case::Failed, + Case::Expired, + Case::Unclean, + Case::Cancel, + ] { + tokio::time::timeout(Duration::from_secs(15), exercise(case)) + .await + .unwrap(); + } +} diff --git a/crates/volparossa-agent/src/content/parallel/continuation.rs b/crates/volparossa-agent/src/content/parallel/continuation.rs new file mode 100644 index 000000000..189575d8f --- /dev/null +++ b/crates/volparossa-agent/src/content/parallel/continuation.rs @@ -0,0 +1,100 @@ +//! Single-use serial coverage after locally pressure-retired streams have drained. + +use std::future::Future; + +use tokio::time::Instant; +use volparossa_content::{ + ChunkStore, VerifiedManifest, + transfer::{ + TransferError, TransferProgress, + parallel::{ChunkWorker, ParallelDownload, ResourceRetired}, + }, +}; + +use super::{ContentError, complete, drivers}; + +pub(super) trait Sources { + fn available(&self) -> bool; + fn transport(&self, index: usize) -> &VerifiedManifest; + fn receive( + &self, + index: usize, + download: ParallelDownload, + worker: ChunkWorker, + store: &mut ChunkStore, + ) -> impl Future> + Send; +} + +#[derive(Default)] +pub(super) struct Outcome { + pub attempted: bool, + pub timed_out: bool, +} + +pub(super) struct RetiredBatch<'a> { + pub manifest: &'a VerifiedManifest, + pub progress: &'a mut [TransferProgress], + pub measurements: &'a [drivers::Measurement], + pub retired: Vec, + pub deadline: Instant, +} + +impl RetiredBatch<'_> { + // The caller must join the original batch before entering this coordinator. Each + // awaited receive owns its writer/driver/lease until they all terminate; no two + // continuations overlap and no failed/partial continuation is queued again. + pub async fn resume( + self, + store: &mut ChunkStore, + sources: &impl Sources, + ) -> Result { + let mut outcome = Outcome::default(); + for retirement in self.retired { + if Instant::now() >= self.deadline { + outcome.timed_out = true; + break; + } + if complete(self.manifest, store)? { + break; + } + let index = retirement.worker_index(); + // A transport retry may spend requests invisible to the single writer. + // Only a clean first-attempt close preserves the token's budget proof. + if self.measurements[index].elapsed.is_none() || !sources.available() { + continue; + } + let (download, worker) = match retirement.prepare_continuation( + self.manifest, + sources.transport(index), + store, + ) { + Ok(prepared) => prepared, + Err(TransferError::Timeout) => { + outcome.timed_out = true; + break; + } + Err(_) => return Err(ContentError::Invalid), + }; + outcome.attempted = true; + let (part, expired) = sources.receive(index, download, worker, store).await?; + let progress = &mut self.progress[index]; + progress.bytes = progress + .bytes + .checked_add(part.bytes) + .ok_or(ContentError::Invalid)?; + progress.chunks = progress + .chunks + .checked_add(part.chunks) + .ok_or(ContentError::Invalid)?; + progress.missing = progress + .missing + .checked_add(part.missing) + .ok_or(ContentError::Invalid)?; + if expired { + outcome.timed_out = true; + break; + } + } + Ok(outcome) + } +} diff --git a/crates/volparossa-agent/src/content/rediscovery/tests.rs b/crates/volparossa-agent/src/content/rediscovery/tests.rs index 9a114b894..08e414016 100644 --- a/crates/volparossa-agent/src/content/rediscovery/tests.rs +++ b/crates/volparossa-agent/src/content/rediscovery/tests.rs @@ -48,6 +48,8 @@ impl Drop for Held { } } +// Independent fault-injection switches exercise combinations, not one exclusive state. +#[allow(clippy::struct_excessive_bools)] struct Fixture { _directory: tempfile::TempDir, manifest: VerifiedManifest, @@ -358,7 +360,7 @@ async fn duplicate_recent_hints_and_snapshots_do_not_repeat_provider_work() { async fn repeated_same_and_empty_snapshots_are_bounded_and_not_full_object_success() { for duplicate in [false, true] { let mut source = fixture(2); - for _ in 0..MAX_ROUNDS + 1 { + for _ in 0..=MAX_ROUNDS { source.snapshots.push_back(Ok(if duplicate { vec![source.providers[0].clone()] } else { @@ -430,10 +432,10 @@ async fn authority_and_discovery_errors_remain_fatal_or_preserve_only_verified_p let result = run(&mut source, deadline()).await; match error { ContentDiscoveryError::Invalid => { - assert!(matches!(result, Err(ContentError::Invalid))) + assert!(matches!(result, Err(ContentError::Invalid))); } ContentDiscoveryError::Invalidated => { - assert!(matches!(result, Err(ContentError::Policy))) + assert!(matches!(result, Err(ContentError::Policy))); } _ if partial => assert_eq!(result.unwrap().1, CHUNK_BYTES as u64), _ => assert!(matches!(result, Err(ContentError::Unavailable))), diff --git a/crates/volparossa-agent/src/content/worker_budget.rs b/crates/volparossa-agent/src/content/worker_budget.rs index 8c05e051b..ac30f31f5 100644 --- a/crates/volparossa-agent/src/content/worker_budget.rs +++ b/crates/volparossa-agent/src/content/worker_budget.rs @@ -458,7 +458,7 @@ mod tests { } let causes = value["causes"].as_object().unwrap(); assert!(causes.iter().all(|(key, value)| value - == &matches!( + == matches!( key.as_str(), "cpu_pressure" | "memory_unavailable" | "io_pressure" ))); diff --git a/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs b/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs index 0100f7cf9..7ff2c76c7 100644 --- a/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs +++ b/crates/volparossa-agent/src/content/worker_budget/diagnostics.rs @@ -16,6 +16,8 @@ pub(super) enum Outcome { } #[derive(Clone, Copy, Default, Serialize)] +// Independent, cumulative observations in a closed diagnostic schema, not exclusive states. +#[allow(clippy::struct_excessive_bools)] struct Causes { cpu_pressure: bool, memory_pressure: bool, @@ -27,7 +29,7 @@ struct Causes { lock_unavailable: bool, } -/// Counts saturate at u32::MAX; this never clamps the actual resource decision. +/// Counts saturate at `u32::MAX`; this never clamps the actual resource decision. #[derive(Clone, Serialize)] struct Summary { schema_version: u8, @@ -197,7 +199,7 @@ mod tests { assert_eq!(value["initial_allowance"], 0); let causes = value["causes"].as_object().unwrap(); assert_eq!(causes.len(), 8); - assert!(causes.iter().all(|(key, value)| value == &(key == reason))); + assert!(causes.iter().all(|(key, value)| value == (key == reason))); } } diff --git a/crates/volparossa-content/src/transfer/parallel.rs b/crates/volparossa-content/src/transfer/parallel.rs index 95a04de13..c942608d6 100644 --- a/crates/volparossa-content/src/transfer/parallel.rs +++ b/crates/volparossa-content/src/transfer/parallel.rs @@ -44,6 +44,84 @@ struct Peer { state: adaptive::PeerState, probing: bool, coverage_only: bool, + resource_retired: bool, + transport_id: [u8; 32], +} + +enum RetirementMode { + Capture, + Suppressed, +} + +/// One locally pressure-retired worker's remaining, single-use coverage authority. +/// +/// This is not a successful transfer receipt. The caller must first join every original +/// worker, require this worker's clean first-attempt close, and acquire a fresh resource +/// lease. A failed transport/retry must never use this token: writer accounting cannot +/// observe extra requests made by a reconnecting transport. +pub struct ResourceRetired { + index: usize, + expected_id: [u8; 32], + transport_id: [u8; 32], + attempted: BTreeSet, + limits: TransferLimits, + deadline: Instant, +} + +impl ResourceRetired { + /// Index in the original bounded provider batch. + pub fn worker_index(&self) -> usize { + self.index + } + + /// Prepare one serial coverage continuation, never a renewed peer budget. + /// + /// Verified hits and this provider's previous requests (including explicit misses) + /// are excluded. The original absolute deadline and both manifest identities remain + /// binding. The returned coordinator cannot issue another retirement token. + /// + /// # Errors + /// Rejects changed authority, expiry/deadline, invalid limits or local cache corruption. + pub fn prepare_continuation( + self, + expected: &VerifiedManifest, + transport: &VerifiedManifest, + store: &mut ChunkStore, + ) -> Result<(ParallelDownload, ChunkWorker), TransferError> { + if *expected.manifest_id() != self.expected_id + || *transport.manifest_id() != self.transport_id + { + return Err(TransferError::Protocol); + } + let mut limits = self.limits; + limits.session_timeout = self.deadline.saturating_duration_since(Instant::now()); + if limits.session_timeout.is_zero() { + return Err(TransferError::Timeout); + } + limits.exchange_timeout = limits.exchange_timeout.min(limits.session_timeout); + let (mut download, mut workers) = if self.expected_id == self.transport_id { + ParallelDownload::new_adaptive(expected, 1, store, limits, 1)? + } else { + ParallelDownload::new_adaptive_with_transport_manifests( + expected, + &[transport], + store, + limits, + 1, + )? + }; + download.deadline = self.deadline; + download.retirement_mode = RetirementMode::Suppressed; + for chunk in &mut download.chunks { + if self.attempted.contains(chunk.chunk.id()) { + chunk.attempted.insert(0); + } + } + let mut worker = workers.pop().ok_or(TransferError::Protocol)?; + worker.deadline = self.deadline; + worker.session.deadline = self.deadline; + Ok((download, worker)) + } } /// A bounded writer/coordinator; workers never own or reopen the destination cache. @@ -57,6 +135,8 @@ pub struct ParallelDownload { measurement: adaptive::Measurement, probe_baseline: Option, growth_blocked: bool, + limits: TransferLimits, + retirement_mode: RetirementMode, } /// One provider's original transport index for coordinator-assigned chunks; fields are sealed. @@ -208,6 +288,8 @@ impl ParallelDownload { measurement: adaptive::Measurement::new(), probe_baseline: None, growth_blocked: false, + limits, + retirement_mode: RetirementMode::Capture, }, workers, )) @@ -240,11 +322,30 @@ impl ParallelDownload { /// # Errors /// Rejects a mismatched progress slice, local storage failure, expiry or the original deadline. pub async fn run_with_allowance( - mut self, + self, store: &mut ChunkStore, progress: &mut [TransferProgress], allowance: impl Fn() -> usize, ) -> Result<(), TransferError> { + self.run_with_allowance_retired(store, progress, allowance) + .await + .map(|_| ()) + } + + /// As [`Self::run_with_allowance`], retaining explicit local-pressure retirements. + /// + /// Exhausted, failed and ordinarily dormant workers produce no continuation authority. + /// Returned tokens require the caller's independently observed clean first-attempt + /// worker closure and serial reacquisition; they never authorize extra concurrency. + /// + /// # Errors + /// Same failures as [`Self::run_with_allowance`]. + pub async fn run_with_allowance_retired( + mut self, + store: &mut ChunkStore, + progress: &mut [TransferProgress], + allowance: impl Fn() -> usize, + ) -> Result, TransferError> { if progress.len() != self.peers.len() { return Err(TransferError::Protocol); } @@ -301,7 +402,49 @@ impl ParallelDownload { let _ = peer.requests.try_send(Work::Finish); } } - Ok(()) + Ok(self.resource_retirements()) + } + + fn resource_retirements(&self) -> Vec { + if matches!(self.retirement_mode, RetirementMode::Suppressed) { + return Vec::new(); + } + self.peers + .iter() + .enumerate() + .filter(|(index, peer)| { + peer.resource_retired + && peer.state == adaptive::PeerState::Finished + && self.eligible(*index) + }) + .filter_map(|(index, peer)| { + let attempted = self + .chunks + .iter() + .filter(|chunk| chunk.attempted.contains(&index)) + .collect::>(); + let bytes = attempted.iter().try_fold(0_u64, |sum, chunk| { + sum.checked_add(u64::from(chunk.chunk.length())) + })?; + let mut limits = self.limits; + limits.max_requests = limits.max_requests.checked_sub(attempted.len())?; + limits.max_bytes = limits.max_bytes.checked_sub(bytes)?; + if limits.max_requests == 0 || limits.max_bytes == 0 { + return None; + } + Some(ResourceRetired { + index, + expected_id: *self.manifest.manifest_id(), + transport_id: peer.transport_id, + attempted: attempted + .into_iter() + .map(|chunk| *chunk.chunk.id()) + .collect(), + limits, + deadline: self.deadline, + }) + }) + .collect() } fn assign_ready(&mut self) -> Result<(), TransferError> { @@ -457,6 +600,8 @@ fn peer( }, probing: false, coverage_only: false, + resource_retired: false, + transport_id: *manifest.manifest_id(), }, ChunkWorker { manifest: manifest.clone(), diff --git a/crates/volparossa-content/src/transfer/parallel/adaptive.rs b/crates/volparossa-content/src/transfer/parallel/adaptive.rs index b2e32bb5a..4c18d9e61 100644 --- a/crates/volparossa-content/src/transfer/parallel/adaptive.rs +++ b/crates/volparossa-content/src/transfer/parallel/adaptive.rs @@ -62,6 +62,7 @@ impl ParallelDownload { break; } if self.peers[index].state == PeerState::Active && self.peers[index].pending.is_none() { + self.peers[index].resource_retired = true; self.finish_peer(index); active -= 1; } @@ -128,7 +129,7 @@ impl ParallelDownload { .count() } - fn eligible(&self, index: usize) -> bool { + pub(super) fn eligible(&self, index: usize) -> bool { self.chunks .iter() .any(|chunk| !chunk.complete && !chunk.attempted.contains(&index)) diff --git a/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs b/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs index 91cbe7873..de53eda4e 100644 --- a/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs +++ b/crates/volparossa-content/src/transfer/parallel/tests/adaptive.rs @@ -5,6 +5,7 @@ use std::sync::{ atomic::{AtomicUsize, Ordering}, }; +mod retirement; mod stream; use super::{ @@ -23,11 +24,22 @@ enum Case { NoResources, Many, Pressure, + PressureComplementary, RetiredCoverage, RetiredDuplicate, PressureCoverage, } +impl Case { + fn allowance(self) -> usize { + match self { + Self::NoResources => 0, + Self::Serial | Self::Many => 1, + _ => 3, + } + } +} + struct Fixture { _directory: tempfile::TempDir, manifest: VerifiedManifest, @@ -274,6 +286,16 @@ async fn hard_pressure_preserves_partial_progress_but_does_not_claim_full_object .expect("hard resource pressure takes precedence over complementary coverage"); } +#[tokio::test] +async fn dynamic_pressure_preserves_disjoint_provider_coverage_with_serial_continuation() { + tokio::time::timeout( + Duration::from_secs(10), + transfer(Case::PressureComplementary), + ) + .await + .expect("bounded disjoint coverage after three streams drain to one"); +} + fn reject_speed_probe(download: &mut ParallelDownload) { // Fix only the performance history, not any content/transport response: // a third admitted probe did not improve the preceding aggregate rate. @@ -342,11 +364,7 @@ async fn transfer(case: Case) { mut output, bytes, } = fixture(case); - let allowance = match case { - Case::NoResources => 0, - Case::Serial | Case::Many => 1, - _ => 3, - }; + let allowance = case.allowance(); let limits = TransferLimits { session_timeout: Duration::from_secs(8), ..TransferLimits::default() @@ -387,7 +405,12 @@ async fn transfer(case: Case) { Ordering::SeqCst, ); let mut drivers = Vec::new(); - for (index, (worker, provider)) in workers.into_iter().zip(providers).enumerate() { + let providers = providers.into_iter().map(Arc::new).collect::>(); + for (index, (worker, provider)) in workers + .into_iter() + .zip(providers.iter().cloned()) + .enumerate() + { drivers.push(tokio::spawn(drive( worker, provider, @@ -398,8 +421,8 @@ async fn transfer(case: Case) { ))); } let mut progress = vec![TransferProgress::default(); transports.len()]; - download - .run_with_allowance(&mut output, &mut progress, || { + let retired = download + .run_with_allowance_retired(&mut output, &mut progress, || { counters.allowance.load(Ordering::SeqCst) }) .await @@ -408,6 +431,21 @@ async fn transfer(case: Case) { for driver in drivers { sent += driver.await.unwrap().bytes; } + if matches!(case, Case::PressureComplementary) { + assert!(!retired.is_empty()); + assert!(progress.iter().map(|part| part.chunks).sum::() < 9); + sent += retirement::continue_streams( + retired, + &manifest, + &transports, + &providers, + &mut output, + &mut progress, + &counters, + ) + .await; + assert!(counters.opened.load(Ordering::SeqCst) <= 6); + } check_result( case, &counters, @@ -447,7 +485,7 @@ fn check_result( assert!(matches!(result, Err(crate::Error::MissingChunk(_)))); return; } - if matches!(case, Case::Pressure) { + if matches!(case, Case::Pressure | Case::PressureComplementary) { assert_eq!(counters.maximum.load(Ordering::SeqCst), 3); assert_eq!(counters.allowance.load(Ordering::SeqCst), 1); assert!( @@ -478,7 +516,7 @@ fn check_result( async fn drive( mut worker: ChunkWorker, - provider: PublicationRegistry, + provider: Arc, counters: Counters, limits: TransferLimits, case: Case, @@ -496,7 +534,7 @@ async fn drive( counters.maximum.fetch_max(running, Ordering::SeqCst); counters.opened.fetch_add(1, Ordering::SeqCst); let (client, mut server) = tokio::io::duplex(1024); - if matches!(case, Case::Pressure) && running == 3 { + if matches!(case, Case::Pressure | Case::PressureComplementary) && running == 3 { counters.allowance.store(1, Ordering::SeqCst); } let mut client = stream::MeasuredStream { diff --git a/crates/volparossa-content/src/transfer/parallel/tests/adaptive/retirement.rs b/crates/volparossa-content/src/transfer/parallel/tests/adaptive/retirement.rs new file mode 100644 index 000000000..a8b3fa05b --- /dev/null +++ b/crates/volparossa-content/src/transfer/parallel/tests/adaptive/retirement.rs @@ -0,0 +1,197 @@ +//! Sealed continuation authority: reason, original identity/time and spent work remain binding. + +use super::{Case, Fixture, fixture}; +use crate::transfer::{ + TransferError, TransferLimits, + parallel::{ParallelDownload, ResourceRetired, adaptive::PeerState}, +}; +use tokio::time::Instant; + +pub(super) async fn continue_streams( + retired: Vec, + manifest: &super::VerifiedManifest, + transports: &[super::VerifiedManifest], + providers: &[super::Arc], + output: &mut super::ChunkStore, + progress: &mut [super::TransferProgress], + counters: &super::Counters, +) -> u64 { + let mut sent = 0; + for token in retired { + assert_eq!(counters.active.load(super::Ordering::SeqCst), 0); + let index = token.worker_index(); + let deadline = token.deadline; + let remaining_requests = token.limits.max_requests; + let remaining_bytes = token.limits.max_bytes; + let attempted = token.attempted.clone(); + let (download, worker) = token + .prepare_continuation(manifest, &transports[index], output) + .unwrap(); + assert_eq!(download.deadline, deadline); + assert_eq!(worker.deadline, deadline); + assert_eq!(worker.session.deadline, deadline); + assert_eq!(worker.limits.max_requests, remaining_requests); + assert_eq!(worker.limits.max_bytes, remaining_bytes); + for chunk in &download.chunks { + assert_eq!( + chunk.attempted.contains(&0), + attempted.contains(chunk.chunk.id()) + ); + } + let limits = worker.limits; + let driver = tokio::spawn(super::drive( + worker, + super::Arc::clone(&providers[index]), + counters.clone(), + limits, + Case::PressureComplementary, + index, + )); + let mut part = [super::TransferProgress::default()]; + let repeated = download + .run_with_allowance_retired(output, &mut part, || 1) + .await + .unwrap(); + assert!( + repeated.is_empty(), + "retirement cannot mint a second continuation" + ); + sent += driver.await.unwrap().bytes; + progress[index].bytes += part[0].bytes; + progress[index].chunks += part[0].chunks; + progress[index].missing += part[0].missing; + } + sent +} + +fn prepared() -> (Fixture, ParallelDownload) { + let mut fixture = fixture(Case::Complementary); + let limits = TransferLimits { + max_requests: 9, + max_bytes: 9 * super::CHUNK_BYTES as u64, + ..TransferLimits::default() + }; + let refs = fixture.transports.iter().collect::>(); + let (mut download, workers) = ParallelDownload::new_adaptive_with_transport_manifests( + &fixture.manifest, + &refs, + &mut fixture.output, + limits, + 3, + ) + .unwrap(); + // A completed explicit miss consumes a request and requested-byte budget, even + // though it inserted no payload. Resource closure is a distinct terminal reason. + download.chunks[0].attempted.insert(1); + download.peers[1].resource_retired = true; + download.peers[1].state = PeerState::Finished; + drop(workers); + (fixture, download) +} + +fn token(download: &ParallelDownload) -> ResourceRetired { + let mut retired = download.resource_retirements(); + assert_eq!(retired.len(), 1); + retired.pop().unwrap() +} + +#[test] +fn pressure_token_preserves_misses_budgets_deadline_and_original_transport() { + let (mut fixture, download) = prepared(); + let token = token(&download); + assert_eq!(token.worker_index(), 1); + assert_eq!(token.limits.max_requests, 8); + assert_eq!(token.limits.max_bytes, 8 * super::CHUNK_BYTES as u64); + let original_deadline = download.deadline; + let known_miss = download.chunks[0].chunk.id(); + let (mut continued, worker) = token + .prepare_continuation( + &fixture.manifest, + &fixture.transports[1], + &mut fixture.output, + ) + .unwrap(); + assert_eq!(continued.deadline, original_deadline); + assert_eq!(worker.deadline, original_deadline); + assert_eq!(worker.session.deadline, original_deadline); + assert_eq!(worker.session.limits.max_requests, 8); + assert_eq!( + worker.session.limits.max_bytes, + 8 * super::CHUNK_BYTES as u64 + ); + assert_eq!( + worker.manifest().manifest_id(), + fixture.transports[1].manifest_id() + ); + let miss = continued + .chunks + .iter() + .find(|chunk| chunk.chunk.id() == known_miss) + .unwrap(); + assert!(miss.attempted.contains(&0)); + continued.assign_ready().unwrap(); + assert_ne!( + continued.peers[0].pending, + Some(0), + "known misses receive no new request" + ); + continued.peers[0].pending = None; + continued.peers[0].resource_retired = true; + continued.peers[0].state = PeerState::Finished; + assert!( + continued.resource_retirements().is_empty(), + "single-use even if pressure flaps again" + ); +} + +#[test] +fn pressure_token_rejects_different_authority_and_expired_original_deadline() { + let (mut fixture, download) = prepared(); + assert!(matches!( + token(&download).prepare_continuation( + &fixture.manifest, + &fixture.transports[0], + &mut fixture.output, + ), + Err(TransferError::Protocol) + )); + assert!(matches!( + token(&download).prepare_continuation( + &fixture.transports[1], + &fixture.transports[1], + &mut fixture.output, + ), + Err(TransferError::Protocol) + )); + let mut expired = token(&download); + expired.deadline = Instant::now(); + assert!(matches!( + expired.prepare_continuation( + &fixture.manifest, + &fixture.transports[1], + &mut fixture.output, + ), + Err(TransferError::Timeout) + )); +} + +#[test] +fn failed_exhausted_active_and_budget_spent_peers_never_get_pressure_tokens() { + let (_, mut download) = prepared(); + download.peers[1].resource_retired = false; + assert!(download.resource_retirements().is_empty()); + download.peers[1].resource_retired = true; + download.peers[1].state = PeerState::Active; + assert!(download.resource_retirements().is_empty()); + download.peers[1].state = PeerState::Finished; + download.limits.max_requests = 1; + assert!(download.resource_retirements().is_empty()); + download.limits.max_requests = 9; + download.limits.max_bytes = super::CHUNK_BYTES as u64; + assert!(download.resource_retirements().is_empty()); + download.limits.max_bytes = 9 * super::CHUNK_BYTES as u64; + for chunk in &mut download.chunks { + chunk.attempted.insert(1); + } + assert!(download.resource_retirements().is_empty()); +} diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index b04c771b7..77e7c36f6 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,15 +29,16 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Exercise late complementary-provider recovery after the now-proven concurrent three-provider transfers. | The latest standalone run on `c2fbcd8c` again passes native and HTTPS bulk-overlap checks without overriding resource protection. It reaches the corrected control observer, then fails before starting the late fetch: the temporary gate has invalid nftables syntax. The gate separator and failure-stage retention are now corrected and pass targeted checks; repeat the actual trial. Late recovery and requester cancellation remain unproven; the earlier hosted-CI primary-Exit failure has not been revalidated on current source. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Preserve complementary chunks when pressure reduces concurrency, then prove late-provider recovery and requester cancellation. | Earlier runs on `a5e698cc` and `c2fbcd8c` pass native and HTTPS three-provider reconstruction and overlap checks. The corrected-source run on `a03c3c8f` fails earlier after I/O pressure reduces allowance from twelve to one. A deterministic regression reproduces lost complementary coverage; bounded serial continuation passes 36 focused Rust checks and strict scoped Clippy. Exercise the correction on the protected network next. Late recovery and cancellation remain unproven; the earlier hosted-CI primary-Exit failure has not been revalidated on current source. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Weather PR10 fixes cleanup of rejected HTTP responses without retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows. Browser08 passes actual synthetic UI checks with the corrected ECMWF module, and a new immutable archive verifies against that proof. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Continue the standalone -provider proof, which needs no weather downloads, after correcting the late-phase -gate setup; successful concurrent reconstruction does not prove late discovery -convergence. The Drizz.li/Weather candidate and frozen input +provider proof, which needs no weather downloads, after addressing loss of +complementary coverage under pressure. Successful concurrent reconstruction in +an earlier run does not prove pressure recovery or late discovery convergence. +The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; @@ -176,12 +177,69 @@ retention and rejected oversized/foreign-scenario evidence. The six-second gate, three-hundred-second transfer, gate-first cleanup and production behavior are unchanged; the actual late-provider trial is still required. -A new standalone trial is running on clean source -`a03c3c8fc865b8e1dfbf3caab745bcc27918fd4c`, with evidence at -`.git/ci-evidence/content-provider-late-03.nV7eQv`. Its resource preflight passes -without reducing the four-GiB guest or two-GiB host reserve; the pinned VM is -provisioning. No weather source acquisition or joined Weather bundle is included. -Await the retained terminal result and cleanup before starting another heavy job. +The corrected-source trial on `a03c3c8fc865b8e1dfbf3caab745bcc27918fd4c` +fails earlier, at `content-provider-adaptive-fetch`, with +`CONTENT_PROVIDER_ADAPTIVE_RUNTIME_FETCH_FAILED`. The native request returns +`CONTENT_UNAVAILABLE`, not a timeout. Its resource preflight and PSI readiness +pass without reducing the four-GiB guest or two-GiB host reserve. Three provider +leases are admitted concurrently, with a 4.388-second overlap in their observed +bulk-payload windows. During the transfer, I/O pressure reduces worker allowance +from twelve to one; no lease is denied, and all leases are released on return. +Three discovery rounds each verify three offers, but only one worker batch runs. +This is overlapping partial traffic, not complete reconstruction. + +Code inspection identifies a matching failure mode: pressure permanently closes +idle surplus workers even when they have unattempted chunks, while subsequent +discovery excludes those already-attempted providers. The trace supports this +explanation but does not retain individual closure reasons or the final verified +chunk count. A deterministic regression now reproduces the coverage loss over +real chunk streams: three disjoint providers begin concurrently, allowance drops +to one, and the existing coordinator receives only four of nine unique chunks. +The new bounded-continuation path reconstructs all nine chunks in that regression. + +The earlier two-provider, HTTPS, site and filter checks pass and their component +reports rebuild identically from the retained raw evidence. Adaptive HTTPS, +late-provider recovery and cancellation are not reached. The failed aggregate +report is retained under `.git/ci-evidence/content-provider-late-03.nV7eQv/output/`, +SHA-256 `ec1694f4cd4e9133fbf84bd5a1b45de0f0a14c5011698990d1a17d3828fa07e3`. +Cleanup leaves zero owned objects and unchanged guest-root network state. The +runner, VM and scratch overlay are gone; development-host DNS and IPv4/IPv6 route +hashes match the baseline. Host-firewall comparison remains unavailable. No +weather source acquisition or joined Weather bundle is included. + +Locally pressure-retired providers now retain a sealed, single-use continuation. +Only a clean first-attempt close qualifies. All original workers and leases must +drain before serial continuation; current resource admission, protected-route +context, original manifest identities and the absolute deadline remain binding. +Known misses and spent request/byte budgets carry forward. Each eligible source +gets at most one new connection, without recursive continuation or an additional +transport retry. Resumed bytes are verified by the same cache writer and do not +become a concurrent-batch performance sample. Production-coordinator duplex tests +also cover unavailable resources, an expired deadline, failed or unclean streams +and cancellation. This correction still needs the real protected-network trial. + +Independent final checks pass: thirteen content parallel-transfer tests, nine +agent parallel-transfer tests, seven rediscovery tests and seven worker-budget +tests. Strict Clippy passes for both changed crates with all targets/features; +workspace formatting and diff checks also pass. The earlier PR227 Quality failure +was nine narrow lint findings, now corrected without changing admission thresholds +or the closed diagnostic schema. CodeQL passed on that earlier revision; the new +revision still requires hosted checks. + +Use the hosted disposable `content-provider` proof for the next exact revision +while the development host has insufficient available memory for the unchanged +four-GiB guest plus two-GiB reserve. Do not reclaim the user's application memory +or lower that reserve. This standalone scenario makes no Weather source request; +the immutable joined Weather inputs remain ready for their separate trial. + +The late-provider validator now accepts two verified offers across separate +snapshots: first A, then B. Discovery does not require A to reappear alongside B. +The actual late activation, two distinct useful suppliers, complementary chunks, +object hash, protected traffic, original deadline and bounded query count remain +mandatory. Twelve focused late-recovery tests and nineteen aggregate tests pass, +alongside ten inert cancellation checks. Negative cases still reject a missing B, +one total offer, early activation, invalid payload/hash and deadline overflow. +This removes a false rejection; it is not a live late-recovery result. The late-provider fixture uses a fresh publication, route and consumer cache. It registers the second complementary provider only after observing the first From 24463595b4dcd3e9a490c87490188f08c17b7477 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:44:42 +0200 Subject: [PATCH 38/49] test: isolate cancellation proof from prior bootstrap cooldown --- .github/workflows/alpha-topology.yml | 1 + docs/IMPLEMENTATION_STATUS.md | 68 ++++++- .../integration/content-cancellation-smoke.py | 131 +++++++++++- tests/integration/content-provider-smoke.py | 6 +- .../test-content-cancellation-smoke.py | 191 ++++++++++++++++++ .../test-content-provider-smoke.py | 5 + .../test-kvm-alpha-topology-contract.sh | 31 ++- 7 files changed, 415 insertions(+), 18 deletions(-) create mode 100644 tests/integration/test-content-cancellation-smoke.py diff --git a/.github/workflows/alpha-topology.yml b/.github/workflows/alpha-topology.yml index fe049dd88..eb75f6564 100644 --- a/.github/workflows/alpha-topology.yml +++ b/.github/workflows/alpha-topology.yml @@ -180,6 +180,7 @@ jobs: sh -n tests/integration/content-provider-filter-smoke.sh python3 -B tests/integration/test-content-provider-filter-smoke.py python3 -B tests/integration/test-content-provider-smoke.py + python3 -B tests/integration/test-content-cancellation-smoke.py sh -n tests/integration/content-provider-late-recovery.sh python3 -B tests/integration/test-content-provider-late-recovery.py sh -n tests/integration/content-provider-https-smoke.sh diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 77e7c36f6..a85513ba6 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,21 +29,79 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Preserve complementary chunks when pressure reduces concurrency, then prove late-provider recovery and requester cancellation. | Earlier runs on `a5e698cc` and `c2fbcd8c` pass native and HTTPS three-provider reconstruction and overlap checks. The corrected-source run on `a03c3c8f` fails earlier after I/O pressure reduces allowance from twelve to one. A deterministic regression reproduces lost complementary coverage; bounded serial continuation passes 36 focused Rust checks and strict scoped Clippy. Exercise the correction on the protected network next. Late recovery and cancellation remain unproven; the earlier hosted-CI primary-Exit failure has not been revalidated on current source. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete requester cancellation, then join Weather delivery to the verified cache path. | Hosted run `38190664934` on `ebf0d077` passes native and HTTPS three-provider reconstruction and actual late-provider recovery. Cancellation stops before admission because its eight-second prerequisite overlaps the previous bootstrap's thirty-second cooldown. The fixture correction passes focused tests and independent review; repeat the protected trial without relaxing cancellation deadlines. Pressure-retirement continuation passes deterministic tests but was not triggered in this healthy live run; a protected pressure-specific result remains open. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Weather PR10 fixes cleanup of rejected HTTP responses without retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows. Browser08 passes actual synthetic UI checks with the corrected ECMWF module, and a new immutable archive verifies against that proof. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the -joined trial or routing requests through other peers. Continue the standalone -provider proof, which needs no weather downloads, after addressing loss of -complementary coverage under pressure. Successful concurrent reconstruction in -an earlier run does not prove pressure recovery or late discovery convergence. +joined trial or routing requests through other peers. Finish the standalone +provider proof, which needs no weather downloads, at its remaining cancellation +boundary. The latest run proves concurrent reconstruction and late discovery, +but does not exercise the pressure-retirement continuation itself. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. +The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38190664934) +remains **failed overall**, at `content-provider-requester-cancellation`. +Independent reconstruction from its raw evidence validates the earlier native +and cooperative-HTTPS transfers: each reconstructs 15,728,640 bytes in sixty +chunks from three providers, with triple bulk overlap of 4.455 and 4.798 seconds. +Neither transfer fetches origin body bytes. Both worker budgets stay at twelve +with no resource-pressure flags, so these results do not establish live pressure +recovery. + +Late recovery also passes: a second complementary provider activates after the +first completed one-offer snapshot; two queries and three total verified offers +lead to the exact 2,097,275-byte, nine-chunk object. Transfer takes 6.700 seconds; +the temporary gate lasts 0.474 seconds. The original deadlines, protected paths, +privacy captures and cleanup checks pass. This is recovery, not a late-provider +concurrency or speed claim. + +The cancellation request never reaches the required pending bootstrap. Eight +attempts report `PRESELECTION_OWNER_BUSY` while all disconnect probes report idle. +The previous successful bootstrap starts a thirty-second admission cooldown; +route disconnection does not clear it. The fixture starts its eight-second +pending check inside that cooldown. The helper is resumed and fixture outputs +are removed on failure; requester cancellation and shutdown are not yet proved. + +The corrected fixture waits a full thirty seconds on both production clocks +after confirming the original services and idle client. Initial and final +identity/status checks share one thirty-five-second absolute budget. Only then +may it pause the helper; the actual eight-second pending-bootstrap, +four-second cache-release and two-second control-shutdown gates are unchanged. +Current cancellation receipts use schema 2 with the explicit prerequisite; +old schema 1 receipts remain readable only through historical validation. +Thirteen prerequisite tests, ten inert cancellation checks, nineteen aggregate +checks and twelve late-recovery checks pass. Independent review confirms the +clock, process-identity, cleanup and version boundaries. This is fixture +verification, not yet a successful live cancellation result. + +For the same `ebf0d077` source, [CodeQL passes](https://github.com/VOLPAROSSA/volparossa/actions/runs/38190611286). +[Quality](https://github.com/VOLPAROSSA/volparossa/actions/runs/38190613877) +passes dependency/license checks, strict workspace Clippy, workspace tests and +the namespace-backed step, but fails its final non-mutating harness check. +That test extracted the embedded guest argument parser instead of the host +dispatch after the case-arm order changed. A test-only correction now selects +the unique host boundary, rejects four ambiguous/embedded variants, and passes +the affected static contract script, syntax and ShellCheck. The product runner +and argument/hash assertions are unchanged; hosted verification of the new +fixture revision is still required. + +Evidence is retained at `.git/ci-evidence/content-provider-hosted-ebf.dCSta4/`. +The independently rebuilt adaptive and late reports have SHA-256 +`61cda7ef39469d86eb7cf12e997aee951afbe547fb438caed251a025c880d8f4` and +`30279523964150d7af888f983f8ff334231002d5b6b2af98d66b738927611ac5`. +The failed aggregate has SHA-256 +`3b538fddf1ed1e5bf86c7cc26cfcb2293867b3c315a2556a628e56db6a3712c0`. +Cleanup leaves zero owned objects. Guest-root network snapshots are byte-identical, +SHA-256 `8952811858ef8aec046d64c94d6c0c656e4eb98eb8ec143ece8b8f82ebef0f65`. +This hosted result also passes the primary-Exit inventory prerequisite that failed +in an older run; it does not diagnose that older failure. No Weather bundle or +Weather source acquisition is included. + Worker-budget diagnostics are now opt-in through one dedicated debug target. Each provider batch retains only bounded allowance and reservation counts, resource-capacity minima and closed pressure/unavailability flags from the same diff --git a/tests/integration/content-cancellation-smoke.py b/tests/integration/content-cancellation-smoke.py index d733be82f..36fad9747 100644 --- a/tests/integration/content-cancellation-smoke.py +++ b/tests/integration/content-cancellation-smoke.py @@ -20,6 +20,9 @@ AGENT = "volparossa-alpha-agent@client.service" NAME = "disposable-native-network-publication" LIMIT = 262144 +COOLDOWN_MS = 30000 +COOLDOWN_DEADLINE_MS = 35000 +COOLDOWN_SCOPE = "fixture-idle-preselection-cooldown-not-route-readiness" def require(ok, message): @@ -55,19 +58,19 @@ def identity(pid): return dict(pid=pid, start_ticks=int(fields[19])), fields[0] -def properties(unit): +def properties(unit, timeout=5): result = subprocess.run(["systemctl", "show", unit, "--property=MainPID,InvocationID,ActiveState"], - capture_output=True, timeout=5, check=True) + capture_output=True, timeout=timeout, check=True) value = dict(line.split("=", 1) for line in result.stdout.decode().splitlines()) require(value["ActiveState"] == "active" and int(value["MainPID"]) > 1 and re.fullmatch(r"[0-9a-f]{32}", value["InvocationID"]), "unit not active") return value -def owned_helper(record): +def owned_helper(record, timeout=5): current, state = identity(record["process"]["pid"]) require(current == record["process"], "helper PID was replaced") - unit = properties(HELPER) + unit = properties(HELPER, timeout) require(int(unit["MainPID"]) == current["pid"] and unit["InvocationID"] == record["invocation_id"], "helper unit was replaced") namespace = os.stat(f"/proc/{current['pid']}/ns/net") @@ -78,6 +81,90 @@ def owned_helper(record): return state +def cooldown_record(): + return dict(schema_version=1, scope=COOLDOWN_SCOPE, required_ms=COOLDOWN_MS, + deadline_ms=COOLDOWN_DEADLINE_MS, elapsed_monotonic_ms=0, + elapsed_wall_ms=0, total_monotonic_ms=0, before_confirmed=False, after_confirmed=False, + outcome="not-confirmed") + + +def validate_cooldown(record): + require(type(record) is dict and set(record) == set(cooldown_record()), + "wrong fixture cooldown fields") + require(type(record["schema_version"]) is int and record["schema_version"] == 1 + and record["scope"] == COOLDOWN_SCOPE + and type(record["required_ms"]) is int and record["required_ms"] == COOLDOWN_MS + and type(record["deadline_ms"]) is int and record["deadline_ms"] == COOLDOWN_DEADLINE_MS + and type(record["elapsed_monotonic_ms"]) is int + and 0 <= record["elapsed_monotonic_ms"] < 2 ** 63 + and type(record["total_monotonic_ms"]) is int + and record["elapsed_monotonic_ms"] <= record["total_monotonic_ms"] < 2 ** 63 + and type(record["elapsed_wall_ms"]) is int + and -(2 ** 63) < record["elapsed_wall_ms"] < 2 ** 63 + and type(record["before_confirmed"]) is bool + and type(record["after_confirmed"]) is bool + and type(record["outcome"]) is str + and record["outcome"] in {"not-confirmed", "elapsed", "deadline", "clock-invalid"}, + "invalid fixture cooldown receipt") + if record["outcome"] == "elapsed": + require(record["before_confirmed"] and record["after_confirmed"] + and COOLDOWN_MS <= record["elapsed_monotonic_ms"] < COOLDOWN_DEADLINE_MS + and record["total_monotonic_ms"] < COOLDOWN_DEADLINE_MS + and record["elapsed_wall_ms"] >= COOLDOWN_MS, + "fixture cooldown was not confirmed on both clocks") + return record + + +def await_fixture_cooldown(record, confirm, *, monotonic_ns=None, wall_ns=None, sleep=None): + """Fixture-only separation from an earlier bootstrap; never grants route admission. + + Successful production preselection cools for 30 seconds on both clocks, even + after Disconnect reports idle. Do not spend the separate eight-second pending + bootstrap assertion inside that earlier cooldown. No product timer is changed. + `confirm` rechecks the original services and disconnected state without mutation. + """ + monotonic_ns = monotonic_ns or time.monotonic_ns + wall_ns = wall_ns or time.time_ns + sleep = sleep or time.sleep + started, wall_started = monotonic_ns(), wall_ns() + absolute_started = started + deadline = started + COOLDOWN_DEADLINE_MS * 1000000 + previous, previous_wall = started, wall_started + + def observe(): + nonlocal previous, previous_wall + current, wall_current = monotonic_ns(), wall_ns() + record["elapsed_monotonic_ms"] = (current - started) // 1000000 + record["elapsed_wall_ms"] = (wall_current - wall_started) // 1000000 + record["total_monotonic_ms"] = (current - absolute_started) // 1000000 + if current < previous or wall_current < previous_wall: + record["outcome"] = "clock-invalid" + raise ValueError("fixture cooldown clock moved backwards") + previous, previous_wall = current, wall_current + if current >= deadline: + record["outcome"] = "deadline" + raise ValueError("fixture cooldown prerequisite exceeded deadline") + return (deadline - current) / 1000000000 + + # Each subprocess called by confirm receives a fresh remaining-time bound. + confirm(observe) + observe() + # Start the full cooldown only after the idle/service snapshot is confirmed. + # Confirmation itself still spends the original absolute 35-second budget. + started, wall_started = previous, previous_wall + record["before_confirmed"] = True + while True: + remaining = observe() + if min(record["elapsed_monotonic_ms"], record["elapsed_wall_ms"]) >= COOLDOWN_MS: + break + sleep(min(0.1, remaining)) + confirm(observe) + observe() + record["after_confirmed"] = True + record["outcome"] = "elapsed" + validate_cooldown(record) + + def resume(work): # The outer driver calls this BEFORE any other fixture teardown, including on failure. guest(work) @@ -94,8 +181,16 @@ def resume(work): os.close(descriptor) -def validate(value, publication, named): - require(value["schema_version"] == 1 and value["success"] is True +def validate(value, publication, named, *, historical=False): + require(type(value.get("schema_version")) is int + and value["schema_version"] == (1 if historical else 2), + "wrong current or historical cancellation schema") + if historical: + require("cooldown_prerequisite" not in value, "historical cancellation cannot claim later prerequisite") + else: + require(validate_cooldown(value.get("cooldown_prerequisite"))["outcome"] == "elapsed", + "missing confirmed fixture cooldown prerequisite") + require(value["success"] is True and value["scope"] == "named-pre-ready-cancellation-and-pending-bootstrap-shutdown" and value["fresh_cache_initially_absent"] is True and value["fresh_cache_created"] is True and value["cancelled_output_absent"] is True, "missing real pre-ready cancellation") @@ -200,17 +295,33 @@ def busy(label): "client did not start idle") descriptor = os.pidfd_open(helper_process["pid"]) children, opened, probe = [], [], None - evidence = dict(schema_version=1, success=False, + evidence = dict(schema_version=2, success=False, scope="named-pre-ready-cancellation-and-pending-bootstrap-shutdown", helper=helper, agent=dict(unit=AGENT, process=agent_process, invocation_id=agent_unit["InvocationID"]), fresh_cache_initially_absent=True, initial_status=initial_status, initial_paths=initial_paths) + def confirm_idle(remaining): + require(owned_helper(helper, timeout=min(5, remaining())) not in {"T", "t", "Z", "X"}, + "original helper is not running before cancellation fixture") + require(properties(AGENT, timeout=min(5, remaining())) == agent_unit + and identity(agent_process["pid"])[0] == agent_process, + "original agent changed before cancellation fixture") + label = "cooldown-after" if evidence["cooldown_prerequisite"]["before_confirmed"] else "cooldown-before" + status = command(f"{label}-status", ["status"], timeout=min(3, remaining())) + paths = command(f"{label}-paths", ["paths"], timeout=min(3, remaining())) + require(status["status"] == paths["status"] == 0 + and "connected: false" in status["stdout"].splitlines() + and "active contexts: 0" in status["stdout"].splitlines() and paths["stdout"] == "", + "client changed from idle during fixture cooldown") + def interrupted(_signal, _frame): raise InterruptedError("disposable cancellation fixture interrupted") previous_term = signal.signal(signal.SIGTERM, interrupted) try: + evidence["cooldown_prerequisite"] = cooldown_record() + await_fixture_cooldown(evidence["cooldown_prerequisite"], confirm_idle) # Persist exact recovery identity before SIGSTOP, even if the parent is interrupted. write(work / f"{PREFIX}-pause.json", helper) owned_helper(helper) @@ -338,7 +449,11 @@ def sample(): sha256="c" * 64, bytes=99, peer_bytes=0, providers_used=0, origin_body_bytes=0, origin_range_requests=0, provider_peer_ids=[], control_relay_peer_id="", local_delivery=True, ownership_changed=False) - value = dict(schema_version=1, success=True, scope="named-pre-ready-cancellation-and-pending-bootstrap-shutdown", + cooldown = cooldown_record() + cooldown.update(elapsed_monotonic_ms=30001, elapsed_wall_ms=30001, total_monotonic_ms=30001, + before_confirmed=True, after_confirmed=True, outcome="elapsed") + value = dict(schema_version=2, success=True, scope="named-pre-ready-cancellation-and-pending-bootstrap-shutdown", + cooldown_prerequisite=cooldown, fresh_cache_initially_absent=True, fresh_cache_created=True, cancelled_output_absent=True, initial_status=dict(status=0, stdout="connected: false\nactive contexts: 0\n"), initial_paths=dict(status=0, stdout=""), diff --git a/tests/integration/content-provider-smoke.py b/tests/integration/content-provider-smoke.py index f688ae379..ae6ea2104 100644 --- a/tests/integration/content-provider-smoke.py +++ b/tests/integration/content-provider-smoke.py @@ -164,7 +164,7 @@ def build_named_publication(work, nodes): serves={node: read(work / f"content-provider-named-{node}-serve.json") for node in nodes}) -def validate_transfer(evidence): +def validate_transfer(evidence, *, historical=False): publication, output, fetch = evidence["publication"], evidence["output"], evidence["fetch"] require(publication["bytes"] == output["bytes"] == fetch["bytes"] == BYTES and publication["object_sha256"] == output["sha256"] == SHA @@ -324,7 +324,7 @@ def validate_transfer(evidence): if "requester_cancellation" in evidence: cancellation = runpy.run_path(str(Path(__file__).with_name("content-cancellation-smoke.py"))) cancellation["validate"](evidence["requester_cancellation"], publication, - evidence["named_publication"]["fetch"]) + evidence["named_publication"]["fetch"], historical=historical) def build_site_publication(work): @@ -458,7 +458,7 @@ def validate_report(report, revision, *, historical=False): and report["host_state"]["before_sha256"] == report["host_state"]["after_sha256"] and re.fullmatch(r"[0-9a-f]{64}", report["host_state"]["before_sha256"]) and all(report[flag] is False for flag in SCOPE), "wrong source, cleanup, host state or scope") - validate_transfer(report["transfer"]) + validate_transfer(report["transfer"], historical=historical) if __name__ == "__main__": diff --git a/tests/integration/test-content-cancellation-smoke.py b/tests/integration/test-content-cancellation-smoke.py new file mode 100644 index 000000000..68be01c1d --- /dev/null +++ b/tests/integration/test-content-cancellation-smoke.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert prerequisite clock/receipt checks; not live cancellation evidence.""" + +import copy +from pathlib import Path +import re +import runpy +import unittest + +HERE = Path(__file__).parent +CHECK = runpy.run_path(str(HERE / "content-cancellation-smoke.py")) + + +class Clock: + def __init__(self, wall_rate=1): + self.monotonic = 0 + self.wall = 1000000000000 + self.wall_rate = wall_rate + + def sleep(self, seconds): + increment = round(seconds * 1000000000) + self.monotonic += increment + self.wall += round(increment * self.wall_rate) + + def wait(self, record, confirm): + CHECK["await_fixture_cooldown"]( + record, confirm, monotonic_ns=lambda: self.monotonic, + wall_ns=lambda: self.wall, sleep=self.sleep) + + +class CooldownTests(unittest.TestCase): + def test_idle_does_not_mean_preselection_ready_and_pause_follows_both_clocks(self): + clock, record, events = Clock(), CHECK["cooldown_record"](), [] + + def confirm(remaining): + self.assertGreater(remaining(), 0) + # Both snapshots may say idle while production preselection is cooling. + events.append(("same-running-services-and-idle", clock.monotonic)) + + self.assertLess(8000, CHECK["COOLDOWN_MS"]) + clock.wait(record, confirm) + events.append(("pause-helper-and-start-fetch", clock.monotonic)) + self.assertEqual(events, [("same-running-services-and-idle", 0), + ("same-running-services-and-idle", 30000000000), + ("pause-helper-and-start-fetch", 30000000000)]) + self.assertEqual(CHECK["validate_cooldown"](record)["outcome"], "elapsed") + + def test_forward_wall_jump_cannot_replace_monotonic_cooldown(self): + clock, record = Clock(wall_rate=100), CHECK["cooldown_record"]() + clock.wait(record, lambda remaining: remaining()) + self.assertEqual(record["elapsed_monotonic_ms"], 30000) + self.assertEqual(record["elapsed_wall_ms"], 3000000) + + def test_slow_wall_clock_must_also_elapse_before_admission(self): + clock, record = Clock(wall_rate=0.9), CHECK["cooldown_record"]() + clock.wait(record, lambda remaining: remaining()) + self.assertGreaterEqual(record["elapsed_monotonic_ms"], 33333) + self.assertGreaterEqual(record["elapsed_wall_ms"], 30000) + self.assertLess(record["elapsed_monotonic_ms"], 35000) + + def test_wall_clock_stalled_exhausts_absolute_deadline_without_pause(self): + clock, record, confirms = Clock(wall_rate=0), CHECK["cooldown_record"](), [] + with self.assertRaisesRegex(ValueError, "exceeded deadline"): + clock.wait(record, lambda remaining: confirms.append(remaining())) + self.assertEqual(len(confirms), 1) + self.assertEqual(record["outcome"], "deadline") + self.assertFalse(record["after_confirmed"]) + self.assertEqual(record["elapsed_monotonic_ms"], 35000) + CHECK["validate_cooldown"](record) + + def test_wall_clock_rollback_fails_without_final_confirmation(self): + clock, record = Clock(wall_rate=-1), CHECK["cooldown_record"]() + with self.assertRaisesRegex(ValueError, "moved backwards"): + clock.wait(record, lambda remaining: remaining()) + self.assertEqual(record["outcome"], "clock-invalid") + self.assertFalse(record["after_confirmed"]) + CHECK["validate_cooldown"](record) + + def test_final_identity_and_idle_checks_share_original_deadline(self): + clock, record, limits = Clock(), CHECK["cooldown_record"](), [] + + def confirm(remaining): + limits.append(remaining()) + if len(limits) == 2: + clock.sleep(5) + + with self.assertRaisesRegex(ValueError, "exceeded deadline"): + clock.wait(record, confirm) + self.assertEqual(limits, [35, 5]) + self.assertEqual(record["outcome"], "deadline") + self.assertFalse(record["after_confirmed"]) + + def test_initial_confirmation_time_does_not_shorten_cooldown(self): + clock, record, confirms = Clock(), CHECK["cooldown_record"](), [] + + def confirm(remaining): + confirms.append((clock.monotonic, remaining())) + if len(confirms) == 1: + clock.sleep(2) + + clock.wait(record, confirm) + self.assertEqual(confirms, [(0, 35), (32000000000, 3)]) + self.assertEqual(record["elapsed_monotonic_ms"], 30000) + self.assertEqual(record["elapsed_wall_ms"], 30000) + self.assertEqual(record["total_monotonic_ms"], 32000) + + def test_initial_confirmation_cannot_extend_absolute_budget(self): + clock, record, confirms = Clock(), CHECK["cooldown_record"](), [] + + def confirm(remaining): + confirms.append(remaining()) + clock.sleep(5) + + with self.assertRaisesRegex(ValueError, "exceeded deadline"): + clock.wait(record, confirm) + self.assertEqual(confirms, [35]) + self.assertEqual(record["total_monotonic_ms"], 35000) + self.assertFalse(record["after_confirmed"]) + + def test_changed_service_or_idle_condition_never_confirms_prerequisite(self): + for failed_check in (1, 2): + with self.subTest(failed_check=failed_check): + clock, record, confirms = Clock(), CHECK["cooldown_record"](), [] + + def confirm(remaining): + confirms.append(remaining()) + if len(confirms) == failed_check: + raise ValueError("service identity or idle changed") + + with self.assertRaisesRegex(ValueError, "identity or idle"): + clock.wait(record, confirm) + self.assertFalse(record["after_confirmed"]) + self.assertEqual(record["outcome"], "not-confirmed") + CHECK["validate_cooldown"](record) + + def test_closed_receipt_rejects_shortened_clocks_scope_and_unknown_fields(self): + value, publication, named = CHECK["sample"]() + mutations = [dict(required_ms=8000), dict(deadline_ms=40000), + dict(elapsed_monotonic_ms=29999), dict(elapsed_wall_ms=29999), + dict(elapsed_monotonic_ms=35000), dict(before_confirmed=False), + dict(after_confirmed=False), dict(scope="route-ready"), + dict(schema_version=True), dict(elapsed_wall_ms=30000.0), + dict(raw_log="private"), dict(outcome="deadline"), dict(outcome=[]), + dict(total_monotonic_ms=35000), dict(total_monotonic_ms=29999)] + for mutation in mutations: + with self.subTest(mutation=mutation): + broken = copy.deepcopy(value) + broken["cooldown_prerequisite"].update(mutation) + with self.assertRaises(ValueError): + CHECK["validate"](broken, publication, named) + + def test_required_cooldown_matches_production_both_clocks(self): + source = (HERE.parents[1] / "crates/volparossa-agent/src/discovery/preselection_observation.rs").read_text() + wall = re.search(r"const COOLDOWN_MS: u64 = ([0-9_]+);", source) + monotonic = re.search(r"const COOLDOWN: Duration = Duration::from_secs\(([0-9]+)\);", source) + self.assertIsNotNone(wall) + self.assertIsNotNone(monotonic) + self.assertEqual(CHECK["COOLDOWN_MS"], int(wall[1].replace("_", ""))) + self.assertEqual(CHECK["COOLDOWN_MS"], 1000 * int(monotonic[1])) + + def test_old_receipt_requires_explicit_historical_validation(self): + value, publication, named = CHECK["sample"]() + CHECK["validate"](value, publication, named) + with self.assertRaises(ValueError): + CHECK["validate"](value, publication, named, historical=True) + value["schema_version"] = 1 + with self.assertRaises(ValueError): + CHECK["validate"](value, publication, named, historical=True) + value.pop("cooldown_prerequisite") + CHECK["validate"](value, publication, named, historical=True) + with self.assertRaises(ValueError): + CHECK["validate"](value, publication, named) + value["schema_version"] = 2 + with self.assertRaises(ValueError): + CHECK["validate"](value, publication, named) + + def test_fixture_orders_prerequisite_before_pause_and_retains_existing_deadlines(self): + source = (HERE / "content-cancellation-smoke.py").read_text() + run = source.split("def run(", 1)[1].split("def sample(", 1)[0] + self.assertLess(run.index("await_fixture_cooldown("), run.index("signal.SIGSTOP")) + self.assertLess(run.index("await_fixture_cooldown("), run.index("fetching = subprocess.Popen")) + self.assertIn("time.monotonic() + 8", run) + self.assertIn("remaining = 4 - (time.monotonic() - started)", run) + self.assertIn('shutdown["control_eof_ms"] < 2000', run) + self.assertIn('timeout=min(5, remaining())', run) + self.assertIn('timeout=min(3, remaining())', run) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-content-provider-smoke.py b/tests/integration/test-content-provider-smoke.py index 3bfb6a458..cbf310944 100644 --- a/tests/integration/test-content-provider-smoke.py +++ b/tests/integration/test-content-provider-smoke.py @@ -612,6 +612,8 @@ def test_exact_scoped_report(self): lambda item: item.update(schema_version=2.0), lambda item: item.pop("schema_version"), lambda item: item.update(requester_cancellation=False), + lambda item: item["transfer"]["requester_cancellation"].update(schema_version=1), + lambda item: item["transfer"]["requester_cancellation"].pop("cooldown_prerequisite"), lambda item: (item.pop("requester_cancellation"), item["transfer"].pop("requester_cancellation")), lambda item: item["cleanup"].update(remaining_owned_objects=1), lambda item: item["host_state"].update(after_sha256="c" * 64), @@ -624,6 +626,9 @@ def test_exact_scoped_report(self): historical["schema_version"] = 1 historical.pop("late_provider_recovery") historical["transfer"].pop("late_provider_recovery") + # Actual old receipts had neither the new schema nor the prerequisite. + historical["transfer"]["requester_cancellation"]["schema_version"] = 1 + historical["transfer"]["requester_cancellation"].pop("cooldown_prerequisite") CHECK["validate_report"](historical, "a" * 40, historical=True) with self.assertRaises(ValueError): CHECK["validate_report"](historical, "a" * 40) diff --git a/tests/integration/test-kvm-alpha-topology-contract.sh b/tests/integration/test-kvm-alpha-topology-contract.sh index 98bc2f618..da16c6a6a 100755 --- a/tests/integration/test-kvm-alpha-topology-contract.sh +++ b/tests/integration/test-kvm-alpha-topology-contract.sh @@ -1267,6 +1267,7 @@ sh -n "$HERE/content-provider-https-smoke.sh" grep -F 'content_provider_run' "$GUEST" >/dev/null grep -F 'Require actual native provider discovery and protected fetch' "$WORKFLOW" >/dev/null python3 -B "$HERE/test-content-provider-smoke.py" +python3 -B "$HERE/test-content-cancellation-smoke.py" python3 -B "$HERE/test-content-provider-https-smoke.py" printf '%s\n' 'KVM alpha, reciprocity, local-link, mixed-link, sharing, wifi-link and uplink-link topology static contract passed' @@ -1304,8 +1305,34 @@ import sys host = Path(sys.argv[1]).read_text() driver = host.split("<<'GUEST_DRIVER_SCRIPT'\n", 1)[1].split("\nGUEST_DRIVER_SCRIPT\n", 1)[0] parser = driver.split('scenario=$5\n', 1)[1].split('\ncd /home/vpci\n', 1)[0] -outer = host.rsplit('case $scenario in\n agent-native-coding|agent-native-editor)\n', 1)[1] -outer = 'case $scenario in\n agent-native-coding|agent-native-editor)\n' + outer.split('\nesac\n', 1)[0] + '\nesac\n' +guest_end = '\nGUEST_DRIVER_SCRIPT\n' +dispatch_end = '\nesac\nGUEST_STATUS=$?\nset -e\n' + +def outer_dispatch(source): + # Bind the host call outside the heredoc, independently of case-arm ordering. + # Never fall back to the similar argument parser embedded in the guest script. + assert source.count(guest_end) == 1, 'guest-driver boundary must be unique' + assert source.count(dispatch_end) == 1, 'host dispatch boundary must be unique' + tail = source.split(guest_end, 1)[1] + assert dispatch_end in tail, 'host dispatch must follow the guest heredoc' + before = tail.split(dispatch_end, 1)[0] + start = '\ncase $scenario in\n' + assert before.count(start) == 1, 'host dispatch case must be unambiguous' + return 'case $scenario in\n' + before.split(start, 1)[1] + '\nesac\n' + +outer = outer_dispatch(host) +for invalid_source in ( + host.replace(guest_end, '', 1), + host + guest_end, + host + dispatch_end, + host.replace(dispatch_end, '', 1).replace(guest_end, dispatch_end + guest_end, 1), +): + try: + outer_dispatch(invalid_source) + except AssertionError: + pass + else: + raise AssertionError('ambiguous or embedded dispatch must not be selected') base = ['expected', 'source', 'mpquic', 'package'] for scenario, extra, hashes in ( ('agent-native-coding', ['native'], ['native', 'none', 'none']), From ef91e5a9fa18bd040038e53ba177c764a752881e Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:14:41 +0200 Subject: [PATCH 39/49] test: retain bounded Weather rate-limit diagnostics --- .github/workflows/quality.yml | 9 + tests/integration/run-alpha-topology-vm.sh | 2 +- .../integration/test-weather-client-runner.py | 10 +- .../test-weather-products-delivery-shell.py | 14 ++ .../test-weather-products-delivery.py | 173 ++++++++++++- .../test-weather-source-observer.mjs | 230 ++++++++++++++++++ .../integration/weather-products-delivery.py | 94 ++++++- .../integration/weather-products-delivery.sh | 2 +- tests/integration/weather-source-observer.mjs | 115 +++++++++ 9 files changed, 634 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test-weather-source-observer.mjs create mode 100644 tests/integration/weather-source-observer.mjs diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 1c724ece2..b93e03a30 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -71,6 +71,15 @@ jobs: python3 -B tests/integration/transaction-abci-contract-tests.py python3 -B tests/integration/transaction-abci-wiring-tests.py + - name: Check closed Weather source diagnostics without network access + run: | + node --version + node -e 'if (Number(process.versions.node.split(".")[0]) < 22) process.exit(1)' + node --test tests/integration/test-weather-source-observer.mjs + python3 -B tests/integration/test-weather-products-delivery.py + python3 -B tests/integration/test-weather-products-delivery-shell.py + python3 -B tests/integration/test-weather-client-runner.py + - name: Check formatting and shell sources run: | cargo fmt --all -- --check diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index e4dbdce62..cfeacfbf6 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1113,7 +1113,7 @@ WEATHER_PRODUCTS_NAMES = {name.replace("weather-client-delivery-", "weather-prod for name in WEATHER_CLIENT_NAMES} WEATHER_PRODUCTS_FAILURE_NAMES = {f"weather-products-delivery-{suffix}.json" for suffix in ( "acquisition-failure", "export-status", "tree-before", "input-cleanup")} -WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", *WEATHER_CLIENT_HELPERS) +WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", "weather-source-observer.mjs", *WEATHER_CLIENT_HELPERS) LATE_PROVIDER_NAMES = {f"content-provider-late-recovery-{suffix}.json" for suffix in ( "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure")} diff --git a/tests/integration/test-weather-client-runner.py b/tests/integration/test-weather-client-runner.py index fb6dd9be4..facfb84de 100644 --- a/tests/integration/test-weather-client-runner.py +++ b/tests/integration/test-weather-client-runner.py @@ -197,6 +197,8 @@ def setUpClass(cls): exec(compile(body, "weather-collector", "exec"), cls.api) def test_products_helper_requires_complete_root_owned_closure_without_v1_fallback(self): + self.assertIn("weather-source-observer.mjs", self.api["WEATHER_PRODUCTS_HELPERS"]) + self.assertNotIn("weather-source-observer.mjs", self.api["WEATHER_CLIENT_HELPERS"]) with tempfile.TemporaryDirectory() as temporary: work = Path(temporary).resolve() directory = work / "bin" @@ -229,6 +231,11 @@ def fixture_stat(path): with self.subTest(name=name, fault=fault): self.assertIsNone(self.api["weather_client_helper"](work)) bad_path, fault = None, {} + observer = directory / "weather-source-observer.mjs" + observer.unlink() + self.assertIsNone(self.api["weather_client_helper"](work)) + observer.write_text("// inert fixture\n") + observer.chmod(0o555) products = directory / "weather-products-delivery.py" products.unlink() self.assertEqual(self.api["weather_client_helper"](work), directory / "weather-client-delivery.py") @@ -245,7 +252,8 @@ def inert_copy(command, **kwargs): self.assertEqual(command[4:6], ["copy-artifacts", str(root)]) copied = Path(command[6]) for name in self.api["WEATHER_CLIENT_NAMES"] | self.api["WEATHER_PRODUCTS_NAMES"] | { - "weather-products-delivery-config.json", "forecast.json", "private.log"}: + "weather-products-delivery-config.json", "forecast.json", "private.log", + "weather-origin-http.json", "air-quality-origin-http.json", "weather-source-observer.mjs"}: (copied / name).write_text("{}") with patch.object(self.api["subprocess"], "run", inert_copy): diff --git a/tests/integration/test-weather-products-delivery-shell.py b/tests/integration/test-weather-products-delivery-shell.py index 5027b1f8a..f56983f47 100644 --- a/tests/integration/test-weather-products-delivery-shell.py +++ b/tests/integration/test-weather-products-delivery-shell.py @@ -31,6 +31,20 @@ def test_syntax_and_inert_sourcing(self): self.ok(self.call(":")) self.assertEqual(list(self.root.iterdir()), []) + def test_source_observer_is_staged_root_readonly_before_untrusted_acquisition(self): + source = SOURCE.read_text() + staging = " for weather_script in " + source.split(" for weather_script in ", 1)[1].split( + " install -o root -g root -m 0600", 1)[0] + result = self.call(''' +source_directory=/verified-core +install() { printf '%s\\n' "$*"; } +fail() { exit 99; } +''' + staging) + self.ok(result) + expected = f"-o root -g root -m 0555 /verified-core/tests/integration/weather-source-observer.mjs {self.root}/bin/weather-source-observer.mjs" + self.assertEqual(result.stdout.splitlines().count(expected), 1) + self.assertLess(source.index(staging), source.index("PHASE=weather-products-delivery-acquire")) + def test_closed_independent_product_mapping(self): result = self.call(''' for kind in weather air_quality; do diff --git a/tests/integration/test-weather-products-delivery.py b/tests/integration/test-weather-products-delivery.py index 4183f46e1..be01d75d9 100644 --- a/tests/integration/test-weather-products-delivery.py +++ b/tests/integration/test-weather-products-delivery.py @@ -11,6 +11,7 @@ import subprocess import sys import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -78,6 +79,11 @@ def diagnostic(stage="weather-run"): return A["AcquisitionFailure"](stage, ValueError("private coordinates, path, URL and raw stderr")).diagnostic +def source_response(key="weather", observed=CLOCK): + return dict(schema=1, kind="weather-source-rate-limit", product=key, status=429, + observed_at_ms=observed, retry_after_state="valid-seconds", retry_not_before_ms=observed+60000) + + def failure_fixture(work, raw=None): save(A["artifact"](work, "tree-before"), dict(version=1, action="verify-tree", manifest_sha256=SHA, files=193, runtime_executed=False)) @@ -173,7 +179,8 @@ def joined_fixture(work, version=2): class Products(unittest.TestCase): def setUp(self): - clock = patch.dict(A["prepare"].__globals__, observed_ms=lambda: CLOCK) + clock = patch.dict(A["prepare"].__globals__, observed_ms=lambda: CLOCK, + source_observer=lambda: "file:///root-owned/bin/weather-source-observer.mjs") clock.start(); self.addCleanup(clock.stop) def test_profiles_and_browser_exports_never_accept_other_consumers(self): @@ -229,7 +236,7 @@ def gate(stage): def execute(command, cwd, maximum, timeout): if command[-1] == "-V": gate("decoder"); return b"ecCodes Version 2.44.0\n" - key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" + key = "weather" if command[4].endswith("/forecast.mjs") else "air_quality" gate(A["PRODUCTS"][key]["suffix"] + "-run") if failed_stage == A["PRODUCTS"][key]["suffix"] + "-validation": return b'{"SECRET":' return json.dumps(forecast(key, command[command.index("--run")+1])).encode() @@ -267,6 +274,137 @@ def test_acquisition_failure_classification_does_not_leak_subprocess_arguments(s self.assertEqual(value["stage"], "preconditions"); self.assertEqual(result.stderr, b"") self.assertNotIn(temporary.encode(), result.stdout) + def test_source_response_schema_preserves_bounded_advice_without_retry_authority(self): + for key in A["PRODUCTS"]: + good = source_response(key) + A["validate_source_response"](good, key, (CLOCK, CLOCK)) + for state in ("missing", "malformed", "out-of-range"): + A["validate_source_response"](dict(good, retry_after_state=state, retry_not_before_ms=None), key) + for until in (0, -1000, -A["MAX_SOURCE_EPOCH_MS"], CLOCK+A["MAX_RETRY_ADVICE_MS"]): + A["validate_source_response"](dict(good, retry_after_state="valid-date", retry_not_before_ms=until), key) + for delay in (0, A["MAX_RETRY_ADVICE_MS"]): + A["validate_source_response"](dict(good, retry_not_before_ms=CLOCK+delay), key) + bad_values = [dict(good, schema=True), dict(good, status=True), dict(good, status=503), + dict(good, product="wrong"), dict(good, raw_headers="SECRET"), dict(good, url="SECRET"), + dict(good, observed_at_ms=True), dict(good, observed_at_ms=-1), + dict(good, observed_at_ms=A["MAX_SOURCE_EPOCH_MS"]+1), dict(good, retry_after_state=[]), + dict(good, retry_after_state="retry"), dict(good, retry_after_state="missing"), + dict(good, retry_not_before_ms=None), dict(good, retry_not_before_ms=True), + dict(good, retry_not_before_ms=CLOCK-1000), dict(good, retry_not_before_ms=CLOCK+1), + dict(good, retry_not_before_ms=CLOCK+A["MAX_RETRY_ADVICE_MS"]+1000), + dict(good, retry_after_state="valid-date", retry_not_before_ms=-1001), + dict(good, retry_after_state="valid-date", retry_not_before_ms=-A["MAX_SOURCE_EPOCH_MS"]-1000)] + for bad in bad_values: + with self.subTest(bad=bad), self.assertRaises(ValueError): A["validate_source_response"](bad, key) + for window in ((CLOCK+1, CLOCK+2), (CLOCK-2, CLOCK-1), (CLOCK+1, CLOCK), (True, CLOCK)): + with self.subTest(window=window), self.assertRaises(ValueError): + A["validate_source_response"](good, key, window) + + def test_source_sidecar_requires_private_bounded_unique_current_product_record(self): + cases = ("valid", "missing", "mode", "link", "hardlink", "directory", "oversize", "empty", + "duplicate", "wrong-product", "old", "future", "raw-field", "owner", "clock-backward") + for case in cases: + with self.subTest(case=case), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); path = root / A["SOURCE_SIDECARS"]["weather"] + value = source_response() + if case == "wrong-product": value["product"] = "air_quality" + elif case == "old": value["observed_at_ms"] -= 1 + elif case == "future": value["observed_at_ms"] += 1 + elif case == "raw-field": value["url"] = "SECRET" + if case == "link": path.symlink_to(root / "absent") + elif case == "directory": path.mkdir(mode=0o700) + elif case != "missing": + raw = b"x" * 1025 if case == "oversize" else b"" if case == "empty" else \ + b'{"schema":1,"schema":1}' if case == "duplicate" else A["B"]["encode"](value) + A["write_new"](path, raw, 0o644 if case == "mode" else 0o600) + if case == "hardlink": os.link(path, root / "second-link") + with patch.object(A["os"], "getuid", return_value=os.getuid()+1) if case == "owner" else \ + patch.dict(A["source_response"].__globals__, observed_ms=lambda: CLOCK): + state, result = A["source_response"](root, "weather", CLOCK, + CLOCK-1 if case == "clock-backward" else CLOCK) + self.assertEqual(state, "valid" if case == "valid" else "missing" if case == "missing" else "invalid") + self.assertEqual(result, value if case == "valid" else None) + self.assertEqual(A["source_response"](root, "weather", None, None), ("missing", None)) + + def test_observer_requires_fixed_root_owned_readonly_single_link_core_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary); target = root / "weather-source-observer.mjs" + target.touch() + for mode, owner, links, valid in ((0o100555, 0, 1, True), (0o100755, 0, 1, False), + (0o100555, 1000, 1, False), (0o100555, 0, 2, False), (0o040555, 0, 1, False)): + info = SimpleNamespace(st_mode=mode, st_uid=owner, st_nlink=links) + with patch.dict(A["source_observer"].__globals__, HERE=root), \ + patch.object(Path, "lstat", return_value=info), \ + patch.dict(A["B"]["B"], canonical=lambda path: self.assertEqual(path, target)), \ + patch.dict(A["source_observer"].__globals__, raw_file=lambda path, maximum, mode: + self.assertEqual((path, maximum, mode), (target, 64*1024, 0o555))): + if valid: self.assertEqual(A["source_observer"](), target.as_uri()) + else: + with self.assertRaises(ValueError): A["source_observer"]() + + def test_failure_diagnostics_keep_historical_v2_distinct_from_new_source_metadata(self): + old = diagnostic(); old["schema"] = 2 + del old["source_response_state"]; del old["source_response"] + A["validate_acquisition_diagnostic"](old) + value = dict(diagnostic(), source_response_state="valid", source_response=source_response()) + A["validate_acquisition_diagnostic"](value) + for bad in (dict(old, source_response=None), dict(old, schema=3), dict(value, schema=2), + dict(value, source_response_state="missing"), dict(value, source_response_state=[]), + dict(value, stage="decoder", product=None), dict(value, product="air_quality")): + with self.subTest(bad=bad), self.assertRaises(ValueError): A["validate_acquisition_diagnostic"](bad) + for inner in (old, value): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); failure_fixture(work, A["B"]["encode"](inner)) + output = work / "output"; output.mkdir() + status = A["copy_failure_artifacts"](work, output, os.getuid(), os.getgid()) + self.assertTrue(status["success"]) + retained = A["read"](A["artifact"](output, "acquisition-failure")) + self.assertEqual(retained["schema"], 2); self.assertEqual(retained["diagnostic"], inner) + self.assertFalse(any(path.name in A["SOURCE_SIDECARS"].values() for path in output.iterdir())) + + def test_acquisition_failure_binds_only_actual_failing_subprocess_observation(self): + for failed_key, sidecar_key, observed, expected in (("weather", "weather", CLOCK, "valid"), + ("air_quality", "air_quality", CLOCK, "valid"), ("air_quality", "weather", CLOCK, "missing"), + ("weather", "weather", CLOCK-1, "invalid"), ("weather", "weather", CLOCK+1, "invalid")): + with self.subTest(failed_key=failed_key, sidecar_key=sidecar_key, observed=observed), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) + source = base / "inputs"; source.mkdir(mode=0o700) + A["initialize"](root); A["write_new"](source / "INPUTS.json", b"fixture", 0o400) + def execute(command, cwd, maximum, timeout): + if command[-1] == "-V": return b"ecCodes Version 2.44.0\n" + key = command[3].split("?product=")[1] + if key == failed_key: + save(root / A["SOURCE_SIDECARS"][sidecar_key], source_response(sidecar_key, observed)) + raise ValueError("PRIVATE source response and URL") + return json.dumps(forecast(key, command[command.index("--run")+1])).encode() + with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None), \ + patch.dict(A["B"], verify_tree=lambda *args: None, manifest=lambda *args: {"version": 3}), \ + patch.object(A["time"], "time", return_value=CLOCK//1000): + with self.assertRaises(A["AcquisitionFailure"]) as raised: + A["acquisition"](source, SHA, root, "net:[1]", "net:[2]", 1000) + value = raised.exception.diagnostic; A["validate_acquisition_diagnostic"](value) + self.assertEqual(value["source_response_state"], expected) + self.assertEqual(value["product"], failed_key); self.assertNotIn("PRIVATE", json.dumps(value)) + self.assertEqual(value["source_response"], source_response(sidecar_key, observed) if expected == "valid" else None) + + def test_preexisting_product_sidecar_prevents_child_and_is_not_reused(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary); root = base / "weather-publisher"; root.mkdir(mode=0o700) + source = base / "inputs"; source.mkdir(mode=0o700) + A["initialize"](root); A["write_new"](source / "INPUTS.json", b"fixture", 0o400) + calls = [] + def execute(command, *args): + calls.append(command) + save(root / A["SOURCE_SIDECARS"]["weather"], source_response()) + return b"ecCodes Version 2.44.0\n" + with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None), \ + patch.dict(A["B"], verify_tree=lambda *args: None, manifest=lambda *args: {"version": 3}): + with self.assertRaises(A["AcquisitionFailure"]) as raised: + A["acquisition"](source, SHA, root, "net:[1]", "net:[2]", 1000) + self.assertEqual(len(calls), 1) + self.assertEqual(raised.exception.diagnostic["source_response_state"], "missing") + self.assertEqual(raised.exception.diagnostic["stage"], "weather-run") + def test_retained_acquisition_failures_are_closed_and_not_success_claims(self): cases = [(None, "missing"), (b"", "missing"), (b'{"secret":', "invalid"), (A["B"]["encode"](acquisition()), "invalid"), @@ -347,7 +485,7 @@ def test_acquisition_commands_include_precipitation_and_independent_noaa_cycle_w def execute(command, cwd, maximum, timeout): calls.append((command, timeout)) if command[-1] == "-V": return b"ecCodes Version 2.44.0\n" - key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" + key = "weather" if command[4].endswith("/forecast.mjs") else "air_quality" return json.dumps(forecast(key, command[command.index("--run")+1])).encode() clock = datetime.datetime(2026, 10, 11, 14, tzinfo=datetime.timezone.utc).timestamp() with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None), \ @@ -357,6 +495,10 @@ def execute(command, cwd, maximum, timeout): result = A["acquisition"](source, SHA, root, "net:[1]", "net:[2]", 1000) A["validate_acquisition"](result) weather, aq = calls[1][0], calls[2][0] + for key, command in (("weather", weather), ("air_quality", aq)): + self.assertEqual(command[1:4], ["--max-old-space-size=256", "--import", + "file:///root-owned/bin/weather-source-observer.mjs?product=" + key]) + self.assertEqual(command[4], str(source / A["B"]["WEATHER"] / "scripts" / A["PRODUCTS"][key]["script"])) self.assertIn("--include-precipitation", weather); self.assertNotIn("--include-precipitation", aq) self.assertEqual(weather[weather.index("--run")+1], "2026-10-11T00:00:00.000Z") self.assertEqual(aq[aq.index("--run")+1], "2026-10-11T00:00:00.000Z") @@ -382,7 +524,7 @@ def test_acquisition_preserves_future_native_coverage_across_midnight_and_noon(s started, received = A["stamp"](when), [] def execute(command, cwd, maximum, timeout): if command[-1] == "-V": return b"ecCodes Version 2.44.0\n" - key = "weather" if command[2].endswith("/forecast.mjs") else "air_quality" + key = "weather" if command[4].endswith("/forecast.mjs") else "air_quality" value = forecast(key, command[command.index("--run")+1], iso(started + 2000)) received.append(value); return json.dumps(value).encode() with patch.dict(A["acquisition"].__globals__, run=execute, boundary=lambda *args: None, @@ -636,6 +778,29 @@ def test_cleanup_validates_all_targets_before_deleting_two_product_sources(self) self.assertEqual(A["cleanup"](root, True), dict(directory_removed=True, files_removed=True)) self.assertFalse(root.exists()) + def test_cleanup_removes_only_exact_private_bounded_source_sidecars(self): + for bad in (None, "empty", "unexpected", "directory", "mode", "oversize", "symlink", "hardlink"): + with self.subTest(bad=bad), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "weather-publisher"; root.mkdir(mode=0o700) + A["write_new"](root / "forecast.json", b"retained-until-all-paths-checked") + weather = root / A["SOURCE_SIDECARS"]["weather"] + aq = root / A["SOURCE_SIDECARS"]["air_quality"] + # An interrupted/malformed diagnostic is removable, never exported raw. + A["write_new"](aq, b'{"incomplete":') + if bad == "directory": weather.mkdir(mode=0o700) + elif bad == "symlink": weather.symlink_to(aq) + else: + A["write_new"](weather, b"x" * (1025 if bad == "oversize" else 0 if bad == "empty" else 100), + 0o644 if bad == "mode" else 0o600) + if bad == "unexpected": A["write_new"](root / "other-origin-http.json", b"keep") + if bad == "hardlink": os.link(weather, root / "linked-diagnostic") + if bad not in (None, "empty"): + with self.assertRaises((ValueError, OSError)): A["cleanup"](root, True) + self.assertTrue((root / "forecast.json").exists()); self.assertTrue(aq.exists()) + else: + self.assertEqual(A["cleanup"](root, True), dict(directory_removed=True, files_removed=True)) + self.assertFalse(root.exists()) + def test_cleanup_accepts_larger_two_product_expected_but_not_links(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) / "weather-viewer"; root.mkdir(mode=0o700) diff --git a/tests/integration/test-weather-source-observer.mjs b/tests/integration/test-weather-source-observer.mjs new file mode 100644 index 000000000..d9f66ff21 --- /dev/null +++ b/tests/integration/test-weather-source-observer.mjs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-3.0-only +// Inert fake-fetch/filesystem tests: no origin requests or downloaded executables. +import assert from 'node:assert/strict'; +import { chmodSync, closeSync, constants, existsSync, fstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, rmSync, symlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { test } from 'node:test'; +import { MAX_RECEIPT_BYTES, MAX_RETRY_SECONDS, observeFetch, parseRetryAfter, writeReceipt } from './weather-source-observer.mjs'; + +const NOW = Date.parse('2026-10-12T12:00:00.123Z'); +const URL = 'https://data.ecmwf.int/forecasts/fixture.grib2'; +const state = (name, until = null) => ({ retry_after_state: name, retry_not_before_ms: until }); +const receipt = (product = 'weather') => ({ schema: 1, kind: 'weather-source-rate-limit', product, + status: 429, observed_at_ms: NOW, ...parseRetryAfter('60', NOW) }); +function response(header = '60', overrides = {}) { + return { url: URL, redirected: false, status: 429, + headers: { get(name) { assert.equal(name, 'retry-after'); return header; } }, + get body() { throw new Error('body must not be accessed'); }, + ...overrides }; +} +function privateDirectory(t) { + const directory = mkdtempSync(join(tmpdir(), 'weather-source-observer-')); + chmodSync(directory, 0o700); + t.after(() => rmSync(directory, { recursive: true, force: false })); + return directory; +} +function readOpened(path) { + const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { return { info: fstatSync(descriptor), raw: readFileSync(descriptor) }; } + finally { closeSync(descriptor); } +} + +test('bounded integral seconds preserve exact delay including zero and leading zeros', () => { + for (const [input, seconds] of [['0', 0], ['001', 1], ['60', 60], [String(MAX_RETRY_SECONDS), MAX_RETRY_SECONDS]]) { + assert.deepEqual(parseRetryAfter(input, NOW), state('valid-seconds', NOW + seconds * 1000)); + } +}); + +test('missing, malformed, combined and excessive guidance never becomes a usable delay', () => { + assert.deepEqual(parseRetryAfter(null, NOW), state('missing')); + for (const input of ['', ' 60 ', '-1', '+1', '0.5', '1e3', '60, 120', '60\r\nSECRET', 'é', '1'.repeat(129), 60, undefined]) { + assert.deepEqual(parseRetryAfter(input, NOW), state('malformed')); + } + for (const input of [String(MAX_RETRY_SECONDS + 1), '9007199254740993', '9'.repeat(128)]) { + assert.deepEqual(parseRetryAfter(input, NOW), state('out-of-range')); + } +}); + +test('canonical dates preserve future and past absolute guidance including signed epochs', () => { + for (const input of ['Mon, 12 Oct 2026 12:01:00 GMT', 'Mon, 12 Oct 2026 11:59:00 GMT', 'Wed, 31 Dec 1969 23:59:59 GMT']) { + assert.deepEqual(parseRetryAfter(input, NOW), state('valid-date', Date.parse(input))); + } + assert.deepEqual(parseRetryAfter('Tue, 20 Oct 2026 12:00:00 GMT', NOW), state('out-of-range')); +}); + +test('date roundtrip rejects incorrect weekday, impossible calendar and noncanonical forms', () => { + for (const input of ['Tue, 12 Oct 2026 12:01:00 GMT', 'Mon, 31 Feb 2026 12:01:00 GMT', + 'Mon, 12 Oct 2026 24:00:00 GMT', 'Mon, 12 Oct 2026 12:00:60 GMT', 'Mon, 12 Oct 2026 12:01:00 UTC', + '2026-10-12T12:01:00Z', 'Monday, 12-Oct-26 12:01:00 GMT', + 'Mon, 12 Oct 2026 12:01:00 GMT, Mon, 12 Oct 2026 12:02:00 GMT']) { + assert.deepEqual(parseRetryAfter(input, NOW), state('malformed')); + } +}); + +test('invalid observation clocks reject parsing without inventing an epoch', () => { + for (const clock of [NaN, Infinity, -1, 1.5, Number.MAX_SAFE_INTEGER, '123']) { + assert.throws(() => parseRetryAfter('60', clock), TypeError); + } + assert.deepEqual(parseRetryAfter('1', 8640000000000000), state('out-of-range')); +}); + +test('exactly one delegation keeps receiver, all arguments, signal and Response identity; no body access', async () => { + const controller = new AbortController(); + const options = { method: 'GET', headers: { Range: 'bytes=1-3' }, signal: controller.signal, redirect: 'error' }; + const receiver = {}, original = response(), emitted = []; + let calls = 0; + const wrapped = observeFetch(function (...args) { + calls++; assert.equal(this, receiver); assert.equal(args[0], URL); assert.equal(args[1], options); + assert.equal(args.length, 2); assert.equal(args[1].signal, controller.signal); return Promise.resolve(original); + }, { product: 'weather', now: () => NOW, emit: value => emitted.push(value) }); + assert.equal(await wrapped.call(receiver, URL, options), original); + assert.equal(calls, 1); assert.deepEqual(emitted, [receipt()]); assert.equal(controller.signal.aborted, false); + assert.deepEqual(options.headers, { Range: 'bytes=1-3' }); +}); + +test('native AbortSignal and original fetch rejection retain exact identity', async () => { + const controller = new AbortController(), error = new Error('private original transport failure'); + let calls = 0, emissions = 0; + const wrapped = observeFetch((_url, options) => { + calls++; assert.equal(options.signal, controller.signal); + return new Promise((_resolve, reject) => options.signal.addEventListener('abort', () => reject(error), { once: true })); + }, { product: 'weather', now() { throw new Error('broken clock'); }, emit() { emissions++; } }); + const pending = wrapped(URL, { signal: controller.signal }); controller.abort(); + await assert.rejects(pending, actual => actual === error); + assert.equal(calls, 1); assert.equal(emissions, 0); +}); + +test('observer IO, header and clock failures preserve original Response without retries', async () => { + for (const failure of ['clock', 'header', 'write', 'backward', 'invalid']) { + let calls = 0, ticks = 0, emissions = 0; + const original = response('60', failure === 'header' ? { headers: { get() { throw new Error('header failed'); } } } : {}); + const wrapped = observeFetch(() => { calls++; return original; }, { product: 'weather', + now() { if (failure === 'clock') throw new Error('clock failed'); + return failure === 'invalid' ? NaN : failure === 'backward' ? NOW - ticks++ : NOW; }, + emit() { emissions++; throw new Error('write failed'); } }); + assert.equal(await wrapped(URL), original); assert.equal(calls, 1); + assert.equal(emissions, failure === 'write' ? 1 : 0); + } +}); + +test('forward wall clock is observation time, not an automatic retry or clamped date', async () => { + let ticks = 0; const emitted = []; + const later = NOW + 3600000; + const original = response('Mon, 12 Oct 2026 12:01:00 GMT'); + const wrapped = observeFetch(() => original, { product: 'weather', now: () => ticks++ ? later : NOW, + emit: value => emitted.push(value) }); + assert.equal(await wrapped(URL), original); + assert.equal(emitted[0].observed_at_ms, later); + assert.equal(emitted[0].retry_not_before_ms, Date.parse('Mon, 12 Oct 2026 12:01:00 GMT')); +}); + +test('only exact allowed original HTTPS origin, final URL and nonredirected429 are observed', async () => { + for (const [url, overrides] of [[URL, { status: 200 }], [URL, { status: 503 }], [URL, { redirected: true }], + [URL, { url: `${URL}?changed` }], ['http://data.ecmwf.int/fixture', {}], + ['https://data.ecmwf.int:8443/fixture', {}], ['https://secret@data.ecmwf.int/fixture', {}], + ['https://data.ecmwf.int.invalid/fixture', {}], ['https://noaa-gefs-pds.s3.amazonaws.com/fixture', {}]]) { + let calls = 0, emissions = 0; + const original = response('60', { url, ...overrides }); + const wrapped = observeFetch(() => { calls++; return original; }, { product: 'weather', now: () => NOW, + emit() { emissions++; } }); + assert.equal(await wrapped(url), original); assert.equal(calls, 1); assert.equal(emissions, 0); + } + const url = 'https://noaa-gefs-pds.s3.amazonaws.com/fixture', original = response(null, { url }), emitted = []; + await observeFetch(() => original, { product: 'air_quality', now: () => NOW, emit: value => emitted.push(value) })(url); + assert.deepEqual(emitted, [{ ...receipt('air_quality'), ...state('missing') }]); +}); + +test('first matching429 alone writes metadata, including failed diagnostic IO', async () => { + for (const fail of [false, true]) { + let calls = 0, emissions = 0; + const original = response(); + const wrapped = observeFetch(() => { calls++; return original; }, { product: 'weather', now: () => NOW, + emit() { emissions++; if (fail) throw new Error('occupied'); } }); + assert.equal(await wrapped(URL), original); assert.equal(await wrapped(URL), original); + assert.equal(calls, 2); assert.equal(emissions, 1); + } +}); + +test('fixed sidecars are exclusive private bounded regular files and preserve existing evidence', t => { + const directory = privateDirectory(t); + for (const [product, name] of [['weather', 'weather-origin-http.json'], ['air_quality', 'air-quality-origin-http.json']]) { + writeReceipt(receipt(product), directory); + const file = join(directory, name), { info, raw: before } = readOpened(file); + assert.ok(info.isFile()); assert.equal(info.nlink, 1); assert.equal(info.mode & 0o777, 0o600); + assert.equal(info.uid, process.getuid()); assert.ok(info.size <= MAX_RECEIPT_BYTES); + assert.deepEqual(JSON.parse(before), receipt(product)); + assert.throws(() => writeReceipt(receipt(product), directory), { code: 'EEXIST' }); + assert.deepEqual(readOpened(file).raw, before); + for (const secret of [URL, 'Range', 'headers', 'body']) assert.equal(before.includes(secret), false); + } +}); + +test('sidecar symlinks, unsafe directory and unknown receipt fields cannot write', t => { + const directory = privateDirectory(t), target = join(directory, 'must-not-create'); + symlinkSync(target, join(directory, 'weather-origin-http.json')); + assert.throws(() => writeReceipt(receipt(), directory)); assert.equal(existsSync(target), false); + const unsafe = join(directory, 'unsafe'); mkdirSync(unsafe, { mode: 0o755 }); + assert.throws(() => writeReceipt(receipt(), unsafe)); assert.deepEqual(readdirSync(unsafe), []); + for (const changes of [{ raw_header: 'PRIVATE' }, { product: '../outside' }, { retry_after_state: 'missing', retry_not_before_ms: NOW }, + { retry_not_before_ms: NOW + 604801000 }, { retry_not_before_ms: NOW + 1 }, { observed_at_ms: NaN }]) { + assert.throws(() => writeReceipt({ ...receipt('air_quality'), ...changes }, directory), TypeError); + } + assert.equal(existsSync(join(directory, 'air-quality-origin-http.json')), false); +}); + +test('ordinary import is inert; explicit product query wraps only fetch without starting it', t => { + const directory = privateDirectory(t), module = new globalThis.URL('./weather-source-observer.mjs', import.meta.url).href; + const program = `import assert from 'node:assert/strict'; + const original = globalThis.fetch; await import(${JSON.stringify(module)}); + assert.equal(globalThis.fetch, original); + await import(${JSON.stringify(`${module}?product=weather`)}); + assert.notEqual(globalThis.fetch, original);`; + const result = spawnSync(process.execPath, ['--input-type=module', '--eval', program], + { cwd: directory, encoding: 'utf8', timeout: 5000 }); + assert.equal(result.status, 0, result.stderr); assert.deepEqual(readdirSync(directory), []); + assert.deepEqual(readdirSync(directory), []); +}); + +test('malformed bootstrap selectors fail before application with zero fetch calls or sidecars', t => { + const directory = privateDirectory(t), module = new globalThis.URL('./weather-source-observer.mjs', import.meta.url).href; + const guard = `let calls = 0; + globalThis.fetch = () => { calls++; throw Error('unexpected fake fetch'); }; + process.on('exit', () => process.stdout.write(JSON.stringify({ calls })));`; + for (const query of ['?product=other', '?product=weather&path=outside', '?product=weather&product=air_quality', + '?product=%77eather', '?product=weather#fragment', '#fragment']) { + const invalid = spawnSync(process.execPath, ['--import', `data:text/javascript,${encodeURIComponent(guard)}`, + '--import', `${module}${query}`, '--eval', 'process.exit(99)'], + { cwd: directory, encoding: 'utf8', timeout: 5000, maxBuffer: 4096, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' } }); + assert.equal(invalid.status, 1, invalid.stderr); + assert.deepEqual(JSON.parse(invalid.stdout), { calls: 0 }); + assert.deepEqual(readdirSync(directory), []); + } +}); + +test('actual explicit --import captures one fake429 without rewriting script arguments', t => { + const directory = privateDirectory(t), module = new globalThis.URL('./weather-source-observer.mjs', import.meta.url).href; + const bootstrap = `globalThis.calls = 0; globalThis.original = { + url: ${JSON.stringify(URL)}, status: 429, redirected: false, + headers: { get(name) { if (name !== 'retry-after') throw Error('unexpected header access'); return '60'; } }, + get body() { throw Error('body access'); } + }; globalThis.fetch = async (...args) => { globalThis.calls++; globalThis.forwarded = args; return globalThis.original; };`; + const program = `import assert from 'node:assert/strict'; + const options = { signal: new AbortController().signal }; + const value = await fetch(${JSON.stringify(URL)}, options); + assert.equal(value, globalThis.original); assert.equal(globalThis.calls, 1); + assert.equal(globalThis.forwarded[1], options); + assert.deepEqual(process.argv.slice(1), ['origin', '--unchanged']);`; + const result = spawnSync(process.execPath, ['--import', `data:text/javascript,${encodeURIComponent(bootstrap)}`, + '--import', `${module}?product=weather`, '--input-type=module', '--eval', program, 'origin', '--unchanged'], + { cwd: directory, encoding: 'utf8', timeout: 5000, maxBuffer: 4096, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' } }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(readdirSync(directory), ['weather-origin-http.json']); + const value = JSON.parse(readOpened(join(directory, 'weather-origin-http.json')).raw); + assert.equal(value.product, 'weather'); assert.equal(value.retry_after_state, 'valid-seconds'); + assert.equal(value.retry_not_before_ms - value.observed_at_ms, 60000); + assert.equal(value.status, 429); +}); diff --git a/tests/integration/weather-products-delivery.py b/tests/integration/weather-products-delivery.py index e2b4c91bc..49368978a 100644 --- a/tests/integration/weather-products-delivery.py +++ b/tests/integration/weather-products-delivery.py @@ -49,6 +49,60 @@ "check-or-execution-failed"} FAILURE_EXPORTS = ("acquisition-failure", "tree-before", "input-cleanup") SAFE_ERRORS = (OSError, ValueError, KeyError, TypeError, StopIteration, subprocess.SubprocessError) +SOURCE_SIDECARS = {"weather": "weather-origin-http.json", "air_quality": "air-quality-origin-http.json"} +MAX_SOURCE_RESPONSE = 1024 +MAX_SOURCE_EPOCH_MS = 8640000000000000 # JavaScript Date range, within safe integers. +MAX_RETRY_ADVICE_MS = 7 * 24 * 3600000 +SOURCE_RESPONSE_STATES = {"valid-seconds", "valid-date", "missing", "malformed", "out-of-range"} + + +def source_observer(): + # This explicit fixture helper belongs to the verified core, not the frozen + # Weather bundle. The root orchestrator stages it read-only before dropping UID. + path = HERE / "weather-source-observer.mjs" + B["B"]["canonical"](path) + info = path.lstat() + require(stat.S_ISREG(info.st_mode) and info.st_uid == 0 and info.st_nlink == 1 + and stat.S_IMODE(info.st_mode) == 0o555) + raw_file(path, 64 * 1024, 0o555) + return path.as_uri() + + +def validate_source_response(value, key, window=None): + require(type(value) is dict and set(value) == {"schema", "kind", "product", "status", + "observed_at_ms", "retry_after_state", "retry_not_before_ms"}) + same(value["schema"], 1); same(value["kind"], "weather-source-rate-limit") + same(value["product"], key); product(key); same(value["status"], 429) + observed, until, state = value["observed_at_ms"], value["retry_not_before_ms"], value["retry_after_state"] + require(type(observed) is int and 0 <= observed <= MAX_SOURCE_EPOCH_MS + and type(state) is str and state in SOURCE_RESPONSE_STATES) + if window is not None: + start, end = window + require(type(start) is int and type(end) is int and 0 <= start <= observed <= end <= MAX_SOURCE_EPOCH_MS) + if state in ("valid-seconds", "valid-date"): + require(type(until) is int and abs(until) <= MAX_SOURCE_EPOCH_MS + and until - observed <= MAX_RETRY_ADVICE_MS) + require(until >= observed and (until - observed) % 1000 == 0 + if state == "valid-seconds" else until % 1000 == 0) + else: + same(until, None) + + +def source_response(root, key, started_at, finished_at): + # No file from an earlier product/run can explain this failure. These files + # are never exported directly, and their advice never triggers a retry. + if key is None or started_at is None: + return "missing", None + path = root / SOURCE_SIDECARS[key] + try: + if not path.exists() and not path.is_symlink(): return "missing", None + private(path, directory=False) + value = json.loads(raw_file(path, MAX_SOURCE_RESPONSE, 0o600), object_pairs_hook=B["B"]["unique_pairs"]) + end = observed_ms() if finished_at is None else finished_at + validate_source_response(value, key, (started_at, end)) + return "valid", value + except SAFE_ERRORS: + return "invalid", None def ui_profile(version): @@ -65,24 +119,34 @@ def ui_profile(version): class AcquisitionFailure(ValueError): """Carry only a closed classification, never a subprocess error or argv.""" - def __init__(self, stage, error): + def __init__(self, stage, error, source_response_state="missing", source_response=None): failure = ("missing-input" if isinstance(error, FileNotFoundError) else "permission-denied" if isinstance(error, PermissionError) else "io-failed" if isinstance(error, OSError) else "timeout" if isinstance(error, subprocess.TimeoutExpired) else "process-failed" if isinstance(error, subprocess.CalledProcessError) else "check-or-execution-failed") - self.diagnostic = dict(schema=2, kind="weather-products-acquisition-diagnostic", success=False, - stage=stage, product=ACQUISITION_STAGES[stage], failure=failure) + self.diagnostic = dict(schema=3, kind="weather-products-acquisition-diagnostic", success=False, + stage=stage, product=ACQUISITION_STAGES[stage], failure=failure, + source_response_state=source_response_state, source_response=source_response) super().__init__("weather_products_acquisition_failed") def validate_acquisition_diagnostic(value): - require(type(value) is dict and set(value) == {"schema", "kind", "success", "stage", "product", "failure"}) - same(value["schema"], 2); same(value["kind"], "weather-products-acquisition-diagnostic"); same(value["success"], False) + require(type(value) is dict and type(value.get("schema")) is int and value["schema"] in (2, 3)) + fields = {"schema", "kind", "success", "stage", "product", "failure"} + # Historical v2 proves only the stage; do not invent HTTP metadata for it. + require(set(value) == fields | ({"source_response_state", "source_response"} if value["schema"] == 3 else set())) + same(value["kind"], "weather-products-acquisition-diagnostic"); same(value["success"], False) require(type(value["stage"]) is str and value["stage"] in ACQUISITION_STAGES and type(value["failure"]) is str and value["failure"] in FAILURE_CLASSES) same(value["product"], ACQUISITION_STAGES[value["stage"]]) + if value["schema"] == 3: + state = value["source_response_state"] + require(type(state) is str and state in {"missing", "valid", "invalid"}) + if value["product"] is None: same(state, "missing") + if state == "valid": validate_source_response(value["source_response"], value["product"]) + else: same(value["source_response"], None) def validate_acquisition_failure(value): @@ -174,6 +238,7 @@ def forecast_times(value, key, model_run, observed_at, acquisition_window=None): def acquisition(source, manifest_sha, root, parent, client, group): stage = "preconditions" + active_product, started_at, finished_at = None, None, None try: private(root) require(root.name == "weather-publisher" and set(p.name for p in root.iterdir()) == {"passphrase", "scratch"}) @@ -182,6 +247,7 @@ def acquisition(source, manifest_sha, root, parent, client, group): stage = "tree" B["verify_tree"](source, manifest_sha) ui_profile(B["manifest"](raw_file(source / "INPUTS.json", B["MAX_MANIFEST"], 0o400), manifest_sha)["version"]) + observer = source_observer() deadline = time.monotonic() + 720 stage = "decoder" require(run([str(source / "decoder/grib_ls"), "-V"], root, 4096, 10).strip() == b"ecCodes Version 2.44.0") @@ -194,8 +260,11 @@ def acquisition(source, manifest_sha, root, parent, client, group): products = {} for key, p in PRODUCTS.items(): stage = p["suffix"] + "-run" + active_product, started_at, finished_at = key, None, None + sidecar = root / SOURCE_SIDECARS[key] + require(not sidecar.exists() and not sidecar.is_symlink()) model_run = datetime.datetime.fromtimestamp(cycles[key], datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") - command = [str(source / "runtime/node"), "--max-old-space-size=256", + command = [str(source / "runtime/node"), "--max-old-space-size=256", "--import", observer + "?product=" + key, str(source / B["WEATHER"] / "scripts" / p["script"]), "origin", "--latitude", "52.52", "--longitude", "13.41", "--run", model_run, "--steps", ",".join(map(str, p["steps"])), "--community-area", "--execute", "--accept-data-license", "--decoder", str(source / "decoder/grib_ls"), @@ -225,13 +294,15 @@ def acquisition(source, manifest_sha, root, parent, client, group): products[key] = dict(forecast_sha256=hashlib.sha256(raw).hexdigest(), forecast_bytes=len(raw), fields=fields) write_new(root / p["filename"], raw) stage = "final-validation" + active_product, started_at, finished_at = None, None, None result = dict(schema=2, success=True, origin_https=True, acquisition_outside_overlay=True, upstream_signature_attested=False, synthetic_weather=False, synthetic_air_quality=False, independent_product_runs=True, decoder_sha256=B["DECODER_SHA"], products=products) validate_acquisition(result) return result except SAFE_ERRORS as error: - raise AcquisitionFailure(stage, error) from None + state, response = source_response(root, active_product, started_at, finished_at) + raise AcquisitionFailure(stage, error, state, response) from None def validate_publication(value, key): @@ -366,9 +437,11 @@ def cleanup(root, publisher): allowed = {"passphrase", "identity.key", "scratch"} if publisher else {"selection.json", "config.json", "expected.json", "browser-output"} allowed |= {p[field] for p in PRODUCTS.values() for field in (("filename", "manifest", "published", "source_cache") if publisher else ("filename", "receipt"))} + if publisher: allowed |= set(SOURCE_SIDECARS.values()) files, directories, total = [], [], 0 for path in root.iterdir(): require(path.name in allowed) + sidecar_info = private(path, directory=False) if path.name in SOURCE_SIDECARS.values() else None if path.is_dir(): private(path); directories.append(path) children = list(path.iterdir()); require(len(children) <= 20) @@ -378,7 +451,12 @@ def cleanup(root, publisher): (re.fullmatch(r"[a-f0-9]{64}", child.name) or child.name in (".volparossa-owner-v1", ".volparossa-index-v1"))) total += len(raw_file(child, MAX_FORECAST)); files.append(child) else: - total += len(raw_file(path, MAX_EXPECTED)); files.append(path) + if path.name in SOURCE_SIDECARS.values(): + # Exclusive create followed by interruption can leave zero bytes. + total += len(raw_file(path, MAX_SOURCE_RESPONSE, 0o600)) if sidecar_info.st_size else 0 + else: + total += len(raw_file(path, MAX_EXPECTED)) + files.append(path) require(total <= 4*1024*1024) for path in files: path.unlink() for path in directories: path.rmdir() diff --git a/tests/integration/weather-products-delivery.sh b/tests/integration/weather-products-delivery.sh index c8c1629e4..a00621ff8 100644 --- a/tests/integration/weather-products-delivery.sh +++ b/tests/integration/weather-products-delivery.sh @@ -122,7 +122,7 @@ weather_products_delivery_run() { "$WORK/state-$provider_node_a/air-quality-cache" "$WORK/state-$provider_node_b/air-quality-cache"; do if [ -e "$weather_new" ] || [ -L "$weather_new" ]; then fail WEATHER_PRODUCTS_PATH_EXISTS; fi done - for weather_script in weather-products-delivery.py weather-client-delivery.py weather-client-bundle.py \ + for weather_script in weather-products-delivery.py weather-source-observer.mjs weather-client-delivery.py weather-client-bundle.py \ browser-filter-bundle.py content-provider-https-smoke.py content-network-smoke.py; do install -o root -g root -m 0555 "$source_directory/tests/integration/$weather_script" \ "$WORK/bin/$weather_script" || fail WEATHER_PRODUCTS_DRIVER_UNAVAILABLE diff --git a/tests/integration/weather-source-observer.mjs b/tests/integration/weather-source-observer.mjs new file mode 100644 index 000000000..5b6cebaec --- /dev/null +++ b/tests/integration/weather-source-observer.mjs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-3.0-only +// Explicit disposable-fixture instrumentation; never retries or consumes a body. +import { closeSync, constants, fstatSync, openSync, writeFileSync } from 'node:fs'; + +export const MAX_RETRY_SECONDS = 7 * 24 * 60 * 60; +export const MAX_RECEIPT_BYTES = 1024; +const PRODUCTS = Object.freeze({ + weather: { origin: 'https://data.ecmwf.int', sidecar: 'weather-origin-http.json' }, + air_quality: { origin: 'https://noaa-gefs-pds.s3.amazonaws.com', sidecar: 'air-quality-origin-http.json' }, +}); +const STATES = new Set(['valid-seconds', 'valid-date', 'missing', 'malformed', 'out-of-range']); +const KEYS = ['schema', 'kind', 'product', 'status', 'observed_at_ms', 'retry_after_state', 'retry_not_before_ms'].sort(); +const epoch = value => Number.isSafeInteger(value) && Math.abs(value) <= 8640000000000000; +const observedEpoch = value => epoch(value) && value >= 0; +const productInfo = product => typeof product === 'string' && Object.hasOwn(PRODUCTS, product) ? PRODUCTS[product] : null; + +export function parseRetryAfter(value, observedAt) { + if (!observedEpoch(observedAt)) throw new TypeError('invalid observation clock'); + const result = (state, until = null) => ({ retry_after_state: state, retry_not_before_ms: until }); + if (value === null) return result('missing'); + if (typeof value !== 'string' || value.length > 128 || !/^[\x20-\x7e]+$/.test(value)) { + return result('malformed'); + } + if (/^\d+$/.test(value)) { + // BigInt avoids rounding large/overflowed advice into an earlier usable time. + const seconds = BigInt(value); + if (seconds > BigInt(MAX_RETRY_SECONDS)) return result('out-of-range'); + const until = observedAt + Number(seconds) * 1000; + return epoch(until) ? result('valid-seconds', until) : result('out-of-range'); + } + // Only canonical IMF-fixdate. Roundtrip rejects weekday and calendar rollover. + if (!/^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value)) { + return result('malformed'); + } + const until = Date.parse(value); + if (!epoch(until) || new Date(until).toUTCString() !== value) return result('malformed'); + if (until - observedAt > MAX_RETRY_SECONDS * 1000) return result('out-of-range'); + // Past absolute guidance is preserved, not converted into permission to retry. + return result('valid-date', until); +} + +function encodeReceipt(receipt) { + if (!receipt || Object.keys(receipt).sort().join(',') !== KEYS.join(',') + || receipt.schema !== 1 || receipt.kind !== 'weather-source-rate-limit' + || !productInfo(receipt.product) || receipt.status !== 429 || !observedEpoch(receipt.observed_at_ms) + || !STATES.has(receipt.retry_after_state)) throw new TypeError('invalid source diagnostic'); + const until = receipt.retry_not_before_ms; + if (receipt.retry_after_state.startsWith('valid-')) { + if (!epoch(until) || until - receipt.observed_at_ms > MAX_RETRY_SECONDS * 1000 + || (receipt.retry_after_state === 'valid-seconds' + ? until < receipt.observed_at_ms || (until - receipt.observed_at_ms) % 1000 !== 0 + : until % 1000 !== 0)) throw new TypeError('invalid retry guidance'); + } else if (until !== null) throw new TypeError('unknown retry guidance must remain unknown'); + const raw = Buffer.from(`${JSON.stringify(receipt)}\n`); + if (raw.length > MAX_RECEIPT_BYTES) throw new TypeError('oversized source diagnostic'); + return raw; +} + +export function writeReceipt(receipt, directory = process.cwd()) { + const raw = encodeReceipt(receipt); + // Anchor creation to one already-open private directory, never follow a sidecar + // symlink or overwrite prior evidence. The parent owns the fixed publisher cwd. + const folder = openSync(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + let file; + try { + const info = fstatSync(folder); + if (info.uid !== process.getuid() || (info.mode & 0o777) !== 0o700) { + throw new TypeError('private source diagnostic directory required'); + } + file = openSync(`/proc/self/fd/${folder}/${productInfo(receipt.product).sidecar}`, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + const created = fstatSync(file); + if (created.uid !== process.getuid() || created.nlink !== 1 || (created.mode & 0o777) !== 0o600) { + throw new TypeError('private source diagnostic file required'); + } + writeFileSync(file, raw); + } finally { + if (file !== undefined) closeSync(file); + closeSync(folder); + } +} + +export function observeFetch(fetcher, { product, now = Date.now, emit = writeReceipt } = {}) { + const info = productInfo(product); + if (!info || typeof fetcher !== 'function') throw new TypeError('invalid source observer'); + let attempted = false; + return async function (...args) { + let started; + try { started = now(); } catch { /* Diagnostics must not replace fetch errors. */ } + const response = await Reflect.apply(fetcher, this, args); + try { + if (attempted || typeof args[0] !== 'string') return response; + const input = new URL(args[0]); + if (input.origin !== info.origin || input.username || input.password + || response.url !== args[0] || response.redirected !== false || response.status !== 429) return response; + attempted = true; + const observedAt = now(); + if (!observedEpoch(started) || !observedEpoch(observedAt) || observedAt < started) return response; + emit({ schema: 1, kind: 'weather-source-rate-limit', product, status: 429, + observed_at_ms: observedAt, ...parseRetryAfter(response.headers.get('retry-after'), observedAt) }); + } catch { /* Missing/invalid metadata is not a new source failure. */ } + return response; + }; +} + +// No query means a side-effect-free library import. The core supplies only this +// fixed product selector via explicit --import, not environment or CLI rewriting. +const self = new URL(import.meta.url); +if (self.search || self.hash) { + const product = self.searchParams.get('product'); + if (self.hash || !productInfo(product) || self.search !== `?product=${product}`) { + throw new TypeError('invalid source observer bootstrap'); + } + globalThis.fetch = observeFetch(globalThis.fetch, { product }); +} From c63ddf36312a0ce1c3b59bb527886dae217a9521 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:16:41 +0200 Subject: [PATCH 40/49] test: share late discovery's existing gate deadline --- docs/IMPLEMENTATION_STATUS.md | 58 +++++++++-- .../content-provider-late-recovery.py | 5 +- .../test-content-provider-late-recovery.py | 99 +++++++++++++++++++ 3 files changed, 153 insertions(+), 9 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index a85513ba6..7d8c51c8b 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,22 +29,64 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete requester cancellation, then join Weather delivery to the verified cache path. | Hosted run `38190664934` on `ebf0d077` passes native and HTTPS three-provider reconstruction and actual late-provider recovery. Cancellation stops before admission because its eight-second prerequisite overlaps the previous bootstrap's thirty-second cooldown. The fixture correction passes focused tests and independent review; repeat the protected trial without relaxing cancellation deadlines. Pressure-retirement continuation passes deterministic tests but was not triggered in this healthy live run; a protected pressure-specific result remains open. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost. Closed failure retention now passes focused checks; a separate exact-source probe encounters ECMWF HTTP 429. Weather PR10 fixes cleanup of rejected HTTP responses without retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38193112015` on `24463595` again passes native and HTTPS three-provider reconstruction, but the late-provider fixture stops waiting before its first discovery completes. The fixture correction shares the existing six-second budget instead of stopping at four seconds; thirteen focused checks pass. Cancellation remains unreached. Actual late recovery passed in the preceding run; pressure-retirement continuation still needs its own protected result. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost; a separate exact-source probe encounters ECMWF HTTP 429. Closed retention now includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests, retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows. Browser08 passes actual synthetic UI checks with the corrected ECMWF module, and a new immutable archive verifies against that proof. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Finish the standalone -provider proof, which needs no weather downloads, at its remaining cancellation -boundary. The latest run proves concurrent reconstruction and late discovery, -but does not exercise the pressure-retirement continuation itself. +provider proof, which needs no weather downloads, through late discovery and +cancellation. The latest run proves concurrent reconstruction, while the +preceding run proves late recovery. Neither exercises the pressure-retirement +continuation itself. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. -The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38190664934) +The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38193112015) +on `24463595` is **failed overall**, at the late-provider fixture's +`first-snapshot` stage. A's offer is verified after 25 milliseconds, but the +first discovery response reaches the client after 5.09 seconds. The fixture +stops waiting at four seconds despite its six-second total gate budget. +B is never activated; requester cancellation is not reached. The fetch is +interrupted by cleanup, not reported as a completed product failure. + +The correction lets discovery, activation and release share the same absolute +six-second deadline. Subprocesses receive only the remaining budget, capped at +two seconds each; the 300-second fetch limit and production discovery limits +are unchanged. Thirteen late-provider tests pass, including a reproduction of +the 5.09-second response and expired discovery, activation and release cases. +Nineteen aggregate checks and ten inert cancellation checks also pass. A new +protected run must still prove the complete sequence. + +Independent reconstruction again validates native and cooperative HTTPS: +each delivers the exact 15,728,640-byte object in sixty chunks from three +providers, with triple bulk overlap of 4.614 and 4.437 seconds. HTTPS uses one +authenticated HEAD and no origin body or range requests. Both worker budgets +remain at twelve with no pressure flags. All twelve role/control captures +drain without drops; cleanup leaves zero owned objects. Evidence is retained +at `.git/ci-evidence/content-provider-hosted-244.hTq3Mu/`. +The rebuilt adaptive report has SHA-256 +`a08848a7371493e45e7e7b4d3599c3e9af2896dfda34216eabba60a41f3c88ce`. +Guest-root snapshots are byte-identical, SHA-256 +`d99467378218274580a2abd6927f76651566e885d18c95dea576c5b5680f6f76`. +For this source, both [Quality](https://github.com/VOLPAROSSA/volparossa/actions/runs/38193076296) +and [CodeQL](https://github.com/VOLPAROSSA/volparossa/actions/runs/38193073338) +pass. No Weather acquisition is included in this hosted trial. + +Weather source diagnostics now observe an existing, non-redirected HTTP 429 +response without another request or reading its body. Only the product, +observation time and bounded Retry-After interpretation enter a closed failure +receipt; raw headers, URLs and sidecar files are not exported. Missing, malformed +or excessive advice remains explicitly unknown, and never enables automatic +retry. The observer is a separately verified core fixture helper; the frozen +Weather input bundle is unchanged. Sixteen Node, thirty-three product, +thirteen shell and eighteen runner checks pass, including interrupted-file +cleanup. This is diagnostic verification, not a successful joined Weather run. + +The preceding [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38190664934) remains **failed overall**, at `content-provider-requester-cancellation`. Independent reconstruction from its raw evidence validates the earlier native and cooperative-HTTPS transfers: each reconstructs 15,728,640 bytes in sixty @@ -87,8 +129,8 @@ That test extracted the embedded guest argument parser instead of the host dispatch after the case-arm order changed. A test-only correction now selects the unique host boundary, rejects four ambiguous/embedded variants, and passes the affected static contract script, syntax and ShellCheck. The product runner -and argument/hash assertions are unchanged; hosted verification of the new -fixture revision is still required. +and argument/hash assertions are unchanged; the subsequent `24463595` Quality +run passes this correction. Evidence is retained at `.git/ci-evidence/content-provider-hosted-ebf.dCSta4/`. The independently rebuilt adaptive and late reports have SHA-256 diff --git a/tests/integration/content-provider-late-recovery.py b/tests/integration/content-provider-late-recovery.py index cccb3e34b..fe1cf3273 100644 --- a/tests/integration/content-provider-late-recovery.py +++ b/tests/integration/content-provider-late-recovery.py @@ -255,7 +255,10 @@ def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") initial = snapshot(baseline, deadline) require(count(initial["client"], EVENTS[0]) <= 1, "first snapshot already passed") if first_snapshot(initial): break - require(time.monotonic_ns() < deadline - 2_000_000_000, "first snapshot not available within gate budget") + # Discovery, activation and release share the same six-second + # deadline; reserving two seconds here rejected valid DHT replies + # before that deadline. Slow later commands still fail closed. + require(time.monotonic_ns() < deadline, "first snapshot not available within gate budget") time.sleep(0.02) observed = time.monotonic_ns() write(path(work, "first-snapshot"), initial) diff --git a/tests/integration/test-content-provider-late-recovery.py b/tests/integration/test-content-provider-late-recovery.py index ae5f190e5..46e6c6363 100644 --- a/tests/integration/test-content-provider-late-recovery.py +++ b/tests/integration/test-content-provider-late-recovery.py @@ -325,6 +325,105 @@ def release(_work, **_kwargs): if failure in ("snapshot", "activation"): self.assertLess(calls.index("cleanup-gate"), calls.index("fetch-terminate")) + def test_one_absolute_gate_deadline_accepts_late_discovery_but_not_slow_activation_or_release(self): + for case in ("discovery-5.09s", "discovery-expired", "activation-expired", "release-expired"): + with self.subTest(case=case), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); native = work / "state-client/content-late-recovery"; native.mkdir(parents=True) + binary = work / "volparossa"; binary.touch() + value, calls, clock = fixture(), [], [1_000_000_000] + gate_started, fetch_started, deadlines = [], [], [] + for name, contents in (("layout", value["layout"]), ("peers", value["expected_peers"]), + ("publication", value["publication"])): + CHECK["write"](CHECK["path"](work, name), contents) + class Process: + pid = 77 + returncode = None + def poll(self): return self.returncode + def wait(child, timeout): + calls.append("fetch-wait") + if child.returncode is not None: + self.assertEqual(timeout, 2) + return child.returncode + self.assertAlmostEqual(timeout, (fetch_started[0]+300_000_000_000-clock[0])/1e9) + clock[0] += 100_000_000 + (native / "object.bin").write_bytes( + b"".join(bytes([16+i])*262144 for i in range(8)) + bytes([25])*123) + child.returncode = 0 + return 0 + def terminate(child): calls.append("fetch-terminate"); child.returncode = -15 + def kill(child): calls.append("fetch-kill"); child.returncode = -9 + process = Process() + def spawn(*args, **kwargs): + fetch_started.append(clock[0]); calls.append("fetch-start") + return process + def advance(seconds): clock[0] += round(seconds*1e9) + def execute(args, **kwargs): + # Exercise real command(): every subprocess receives only its + # remaining absolute budget, capped at two seconds. + duration, output = 0.005, b"" + if args[-2:] == ["list", "tables"]: output = b'{"nftables":[]}' + elif args[-2:] == ["-f", "-"]: + gate_started.append(clock[0]); calls.append("gate-install") + elif "list" in args and "table" in args: output = json.dumps(gate_fixture()).encode() + elif "logs" in args: + label = "client" if "runtime-client" in args[2] else "broker" + if process.returncode == 0: snapshot = value["final_snapshot"] + elif case != "discovery-expired" and clock[0] >= fetch_started[0]+5_090_000_000: + snapshot = value["first_snapshot"] + else: snapshot = dict(client=[], broker=value["first_snapshot"]["broker"]) + output = "".join(f"{row['unix_ms']} INFO event={row['event']}\n" for row in snapshot[label]).encode() + elif "serve" in args: + calls.append("activate-b") + duration = 1.0 if case == "activation-expired" else 0.1 + output = json.dumps(value["providers"]["relay5"]["serve"]).encode() + elif "delete" in args: + calls.append("gate-release") + duration = 1.0 if case == "release-expired" else 0.01 + else: self.fail(f"unexpected command {args}") + if gate_started and process.returncode is None: + deadline = gate_started[0]+CHECK["GATE_NS"] + self.assertGreater(kwargs["timeout"], 0) + self.assertLessEqual(kwargs["timeout"], min(2, (deadline-clock[0])/1e9)) + deadlines.append(deadline) + if duration > kwargs["timeout"]: + advance(kwargs["timeout"]) + raise subprocess.TimeoutExpired(args, kwargs["timeout"]) + advance(duration) + return SimpleNamespace(returncode=0, stdout=output) + def release(_work, **_kwargs): + calls.append("cleanup-gate") + self.assertLessEqual(clock[0], gate_started[0]+CHECK["GATE_NS"]+20_000_000) + if case != "discovery-5.09s": self.assertTrue(CHECK["path"](work, "failure").exists()) + with patch.dict(CHECK["run"].__globals__, guest=lambda work: None, + namespace=lambda work: dict(name="va-00000000-r4", identity="1:2"), + process_identity=lambda pid: dict(pid=pid, start_ticks=100), remove_gate=release), \ + patch.object(CHECK["subprocess"], "run", side_effect=execute), \ + patch.object(CHECK["subprocess"], "Popen", side_effect=spawn), \ + patch.object(CHECK["time"], "monotonic_ns", side_effect=lambda: clock[0]), \ + patch.object(CHECK["time"], "time_ns", return_value=1_000_000_000), \ + patch.object(CHECK["time"], "sleep", side_effect=advance): + if case == "discovery-5.09s": CHECK["run"](work, binary, "va-00000000-r4") + else: + with self.assertRaises((ValueError, subprocess.TimeoutExpired)): + CHECK["run"](work, binary, "va-00000000-r4") + self.assertEqual(set(deadlines), {gate_started[0]+6_000_000_000}) + cleanup = CHECK["read"](CHECK["path"](work, "run-cleanup")) + self.assertEqual(cleanup, dict(fetch_reaped=True, gate_removed=True, + operation_completed=case == "discovery-5.09s")) + if case == "discovery-5.09s": + output = CHECK["read"](CHECK["path"](work, "object")) + self.assertGreaterEqual(output["first_snapshot_observed_ns"]-output["fetch_started_ns"], 5_090_000_000) + self.assertLessEqual(output["gate_released_ns"]-output["gate_started_ns"], 6_000_000_000) + value["output"] = output; CHECK["validate"](value) + else: + failure = CHECK["read"](CHECK["path"](work, "failure")); CHECK["validate_failure"](failure) + self.assertEqual(failure["stage"], {"discovery-expired": "first-snapshot", + "activation-expired": "activate-b", "release-expired": "gate-release"}[case]) + self.assertFalse(CHECK["path"](work, "object").exists()) + self.assertLess(calls.index("cleanup-gate"), calls.index("fetch-terminate")) + if case == "discovery-expired": self.assertNotIn("activate-b", calls) + if case == "activation-expired": self.assertNotIn("gate-release", calls) + def test_builder_requires_exact_original_snapshots_and_disconnected_route(self): with tempfile.TemporaryDirectory() as temporary: work = Path(temporary); expected = fixture() From 1fb13275c779eab571899a54d6797a3961a9a565 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:54:19 +0200 Subject: [PATCH 41/49] content: observe bounded pressure-retirement continuations --- .github/workflows/alpha-topology.yml | 1 + .github/workflows/quality.yml | 3 +- .../volparossa-agent/src/content/parallel.rs | 8 + .../parallel/adaptive_tests/continuation.rs | 130 +++++++-- .../src/content/parallel/continuation.rs | 28 +- .../parallel/continuation/diagnostics.rs | 258 ++++++++++++++++++ docs/IMPLEMENTATION_STATUS.md | 58 +++- .../content-provider-worker-budget.py | 58 +++- tests/integration/kvm-alpha-topology.sh | 2 +- tests/integration/run-alpha-topology-vm.sh | 1 + .../test-content-provider-worker-budget.py | 95 ++++++- 11 files changed, 603 insertions(+), 39 deletions(-) create mode 100644 crates/volparossa-agent/src/content/parallel/continuation/diagnostics.rs diff --git a/.github/workflows/alpha-topology.yml b/.github/workflows/alpha-topology.yml index eb75f6564..114e3c09b 100644 --- a/.github/workflows/alpha-topology.yml +++ b/.github/workflows/alpha-topology.yml @@ -180,6 +180,7 @@ jobs: sh -n tests/integration/content-provider-filter-smoke.sh python3 -B tests/integration/test-content-provider-filter-smoke.py python3 -B tests/integration/test-content-provider-smoke.py + python3 -B tests/integration/test-content-provider-worker-budget.py python3 -B tests/integration/test-content-cancellation-smoke.py sh -n tests/integration/content-provider-late-recovery.sh python3 -B tests/integration/test-content-provider-late-recovery.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index b93e03a30..21820d8fe 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -71,8 +71,9 @@ jobs: python3 -B tests/integration/transaction-abci-contract-tests.py python3 -B tests/integration/transaction-abci-wiring-tests.py - - name: Check closed Weather source diagnostics without network access + - name: Check closed provider and Weather diagnostics without network access run: | + python3 -B tests/integration/test-content-provider-worker-budget.py node --version node -e 'if (Number(process.versions.node.split(".")[0]) < 22) process.exit(1)' node --test tests/integration/test-weather-source-observer.mjs diff --git a/crates/volparossa-agent/src/content/parallel.rs b/crates/volparossa-agent/src/content/parallel.rs index 82fc8d94f..cc7ff355c 100644 --- a/crates/volparossa-agent/src/content/parallel.rs +++ b/crates/volparossa-agent/src/content/parallel.rs @@ -295,6 +295,10 @@ impl continuation::Sources for ProtectedContinuations<'_> { self.budget.available_workers() > 0 } + fn active_workers(&self) -> &AtomicUsize { + self.active + } + fn transport(&self, index: usize) -> &VerifiedManifest { self.providers[index] .transport @@ -308,6 +312,7 @@ impl continuation::Sources for ProtectedContinuations<'_> { download: ParallelDownload, worker: ChunkWorker, store: &mut ChunkStore, + diagnostic: &continuation::diagnostics::Diagnostics<'_>, ) -> Result<(TransferProgress, bool), ContentError> { if self.scope.is_none() || RecentProviderScope::for_route(self.context, self.policy).await != self.scope @@ -320,6 +325,9 @@ impl continuation::Sources for ProtectedContinuations<'_> { self.active, || self.budget.try_acquire(), |mut worker| async move { + // This callback runs only after actual resource-lease admission. + // Admission is not a completed socket or authenticated transfer. + diagnostic.admitted(); // The clean original session consumed the first transport attempt. // This single continuation spends the last one; no nested reconnect. Ok(matches!( diff --git a/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs b/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs index 6cdf9a0b5..2ea27cdd6 100644 --- a/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs +++ b/crates/volparossa-agent/src/content/parallel/adaptive_tests/continuation.rs @@ -21,7 +21,10 @@ use super::{Fixture, fixture_with_disjoint}; use crate::content::{ ContentError, now, parallel::{ - continuation::{RetiredBatch, Sources}, + continuation::{ + RetiredBatch, Sources, + diagnostics::{Diagnostics, tests::Capture}, + }, drivers, receive_workers_retired, }, worker_budget::{BatchBudget, WorkerBudget}, @@ -121,6 +124,7 @@ enum Mode { Complete, Failed, Cancel, + PolicyAfterProgress, } struct ActualSources<'a> { @@ -138,6 +142,9 @@ impl Sources for ActualSources<'_> { fn available(&self) -> bool { self.budget.available_workers() > 0 } + fn active_workers(&self) -> &AtomicUsize { + &self.active + } fn transport(&self, _: usize) -> &VerifiedManifest { self.fixture_manifest } @@ -147,6 +154,7 @@ impl Sources for ActualSources<'_> { download: ParallelDownload, worker: ChunkWorker, store: &mut ChunkStore, + diagnostic: &Diagnostics<'_>, ) -> Result<(TransferProgress, bool), ContentError> { assert_eq!(self.active.load(Ordering::SeqCst), 0, "serial continuation"); let driver = drivers::assigned( @@ -155,6 +163,7 @@ impl Sources for ActualSources<'_> { &self.active, || self.budget.try_acquire(), |mut worker| async { + diagnostic.admitted(); assert_eq!( self.opened[index].fetch_add(1, Ordering::SeqCst), 0, @@ -201,6 +210,10 @@ impl Sources for ActualSources<'_> { "a continuation cannot authorize another one" ); assert_eq!(self.active.load(Ordering::SeqCst), 0); + if matches!(self.mode, Mode::PolicyAfterProgress) { + assert!(progress[0].bytes > 0); + return Err(ContentError::Policy); + } Ok((progress[0], expired)) } } @@ -213,6 +226,7 @@ enum Case { Expired, Unclean, Cancel, + PolicyAfterProgress, } async fn exercise(case: Case) { @@ -224,6 +238,7 @@ async fn exercise(case: Case) { deadline, } = prepare().await; let initial_bytes = progress.iter().map(|part| part.bytes).sum::(); + let initial_chunks = progress.iter().map(|part| part.chunks).sum::(); let retired_count = retired.len(); let capacity = usize::from(!matches!(case, Case::NoResources)) as u64; let budget = WorkerBudget::for_test(capacity * 32 * 8 * 1024 * 1024, capacity * 4 * 8, true); @@ -243,6 +258,7 @@ async fn exercise(case: Case) { mode: match case { Case::Failed => Mode::Failed, Case::Cancel => Mode::Cancel, + Case::PolicyAfterProgress => Mode::PolicyAfterProgress, _ => Mode::Complete, }, started: Notify::new(), @@ -258,23 +274,7 @@ async fn exercise(case: Case) { deadline }, }; - let mut work = Box::pin(retired.resume(&mut fixture.cache, &sources)); - if matches!(case, Case::Cancel) { - tokio::select! { - () = sources.started.notified() => {}, - _ = &mut work => panic!("continuation must remain owned until cancellation"), - } - assert_eq!(sources.active.load(Ordering::SeqCst), 1); - drop(work); - } else { - let outcome = work.as_mut().await.unwrap(); - assert_eq!( - outcome.attempted, - matches!(case, Case::Complete | Case::Failed) - ); - assert_eq!(outcome.timed_out, matches!(case, Case::Expired)); - drop(work); - } + let capture = observe_resume(case, retired, &mut fixture.cache, &sources).await; assert_eq!(sources.active.load(Ordering::SeqCst), 0); let opened = sources .opened @@ -282,6 +282,22 @@ async fn exercise(case: Case) { .map(|value| value.load(Ordering::SeqCst)) .sum::(); assert!(opened <= retired_count); + check_diagnostic( + case, + &capture, + retired_count, + opened, + ( + fixture.bytes.len() as u64 - initial_bytes, + 12 - initial_chunks, + ), + ); + if matches!(case, Case::PolicyAfterProgress) { + assert!( + fixture.cache.usage().bytes > initial_bytes, + "failed receive can still have verified writes" + ); + } let diagnosis = batch.snapshot_for_test().unwrap(); assert_eq!(diagnosis["active_leases"], 0); assert!(diagnosis["peak_leases"].as_u64().unwrap() <= 1); @@ -310,6 +326,77 @@ async fn exercise(case: Case) { } } +async fn observe_resume( + case: Case, + retired: RetiredBatch<'_>, + cache: &mut ChunkStore, + sources: &ActualSources<'_>, +) -> Capture { + let capture = Capture::default(); + // These Tokio tests use the current-thread runtime; capture also covers Drop. + let _subscriber = tracing::subscriber::set_default(capture.subscriber(tracing::Level::DEBUG)); + let mut work = Box::pin(retired.resume(cache, sources)); + if matches!(case, Case::Cancel) { + tokio::select! { + () = sources.started.notified() => {}, + _ = &mut work => panic!("continuation must remain owned until cancellation"), + } + assert_eq!(sources.active.load(Ordering::SeqCst), 1); + } else if matches!(case, Case::PolicyAfterProgress) { + assert!(matches!(work.as_mut().await, Err(ContentError::Policy))); + } else { + let outcome = work.as_mut().await.unwrap(); + assert_eq!( + outcome.attempted, + matches!(case, Case::Complete | Case::Failed) + ); + assert_eq!(outcome.timed_out, matches!(case, Case::Expired)); + } + drop(work); + capture +} + +fn check_diagnostic( + case: Case, + capture: &Capture, + retired: usize, + opened: usize, + remaining: (u64, usize), +) { + let summaries = capture.summaries(); + assert_eq!(summaries.len(), 1, "one Drop event per retired batch"); + let summary = &summaries[0]; + assert_eq!(summary.as_object().unwrap().len(), 11); + assert_eq!(summary["schema_version"], 1); + assert_eq!(summary["retired_candidates"], retired); + assert_eq!( + summary["outcome"], + match case { + Case::Cancel => "interrupted", + Case::PolicyAfterProgress => "failed", + _ => "returned", + } + ); + assert_eq!(summary["prepared_continuations"], opened); + assert_eq!(summary["admitted_continuations"], opened); + assert_eq!(summary["initial_active_workers"], 0); + assert_eq!(summary["peak_active_workers"], usize::from(opened > 0)); + assert_eq!( + summary["final_active_workers"], 0, + "emission follows owned driver drain, including cancellation" + ); + if matches!(case, Case::Complete) { + assert_eq!(summary["returned_useful_continuations"], retired); + assert_eq!(summary["returned_verified_bytes"], remaining.0); + assert_eq!(summary["returned_verified_chunks"], remaining.1); + } else { + // Includes post-write policy error: only returned progress is measured. + assert_eq!(summary["returned_useful_continuations"], 0); + assert_eq!(summary["returned_verified_chunks"], 0); + assert_eq!(summary["returned_verified_bytes"], 0); + } +} + #[tokio::test] async fn production_continuation_completes_disjoint_coverage_after_dynamic_pressure() { tokio::time::timeout(Duration::from_secs(15), exercise(Case::Complete)) @@ -331,3 +418,10 @@ async fn production_continuation_never_renews_failed_unclean_or_unavailable_work .unwrap(); } } + +#[tokio::test] +async fn production_continuation_diagnostic_excludes_unreturned_policy_error_progress() { + tokio::time::timeout(Duration::from_secs(15), exercise(Case::PolicyAfterProgress)) + .await + .unwrap(); +} diff --git a/crates/volparossa-agent/src/content/parallel/continuation.rs b/crates/volparossa-agent/src/content/parallel/continuation.rs index 189575d8f..bd0f54136 100644 --- a/crates/volparossa-agent/src/content/parallel/continuation.rs +++ b/crates/volparossa-agent/src/content/parallel/continuation.rs @@ -1,6 +1,10 @@ //! Single-use serial coverage after locally pressure-retired streams have drained. use std::future::Future; +use std::sync::atomic::AtomicUsize; + +pub(super) mod diagnostics; +use diagnostics::Diagnostics; use tokio::time::Instant; use volparossa_content::{ @@ -15,6 +19,7 @@ use super::{ContentError, complete, drivers}; pub(super) trait Sources { fn available(&self) -> bool; + fn active_workers(&self) -> &AtomicUsize; fn transport(&self, index: usize) -> &VerifiedManifest; fn receive( &self, @@ -22,6 +27,7 @@ pub(super) trait Sources { download: ParallelDownload, worker: ChunkWorker, store: &mut ChunkStore, + diagnostic: &Diagnostics<'_>, ) -> impl Future> + Send; } @@ -47,6 +53,20 @@ impl RetiredBatch<'_> { self, store: &mut ChunkStore, sources: &impl Sources, + ) -> Result { + let diagnostic = Diagnostics::new(self.retired.len(), sources.active_workers()); + // The awaited future owns every driver and borrows this diagnostic. On + // cancellation it drops first, before the final active-worker observation. + let result = self.resume_observed(store, sources, &diagnostic).await; + diagnostic.finish(result.is_ok()); + result + } + + async fn resume_observed( + self, + store: &mut ChunkStore, + sources: &impl Sources, + diagnostic: &Diagnostics<'_>, ) -> Result { let mut outcome = Outcome::default(); for retirement in self.retired { @@ -76,7 +96,13 @@ impl RetiredBatch<'_> { Err(_) => return Err(ContentError::Invalid), }; outcome.attempted = true; - let (part, expired) = sources.receive(index, download, worker, store).await?; + diagnostic.prepared(); + let (part, expired) = sources + .receive(index, download, worker, store, diagnostic) + .await?; + // Only returned writer progress is observed here. A cancelled or + // failed receive may have cached bytes that this summary cannot see. + diagnostic.returned_progress(part); let progress = &mut self.progress[index]; progress.bytes = progress .bytes diff --git a/crates/volparossa-agent/src/content/parallel/continuation/diagnostics.rs b/crates/volparossa-agent/src/content/parallel/continuation/diagnostics.rs new file mode 100644 index 000000000..dabd4dee1 --- /dev/null +++ b/crates/volparossa-agent/src/content/parallel/continuation/diagnostics.rs @@ -0,0 +1,258 @@ +//! Opt-in evidence of token preparation, real admission and returned writer progress. +//! +//! Returned progress excludes writes from a receive that fails or is cancelled before +//! returning. Zero here is not a zero-traffic/cache-write claim. No resource is sampled. + +use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +use serde::Serialize; +use volparossa_content::transfer::TransferProgress; + +#[derive(Clone, Copy, Default, Serialize)] +#[serde(rename_all = "snake_case")] +enum Outcome { + Returned, + Failed, + #[default] + Interrupted, +} + +#[derive(Clone, Default, Serialize)] +struct Summary { + schema_version: u8, + outcome: Outcome, + retired_candidates: u32, + prepared_continuations: u32, + admitted_continuations: u32, + returned_useful_continuations: u32, + returned_verified_chunks: u32, + returned_verified_bytes: u64, + initial_active_workers: u32, + peak_active_workers: u32, + final_active_workers: u32, +} + +fn bounded(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + +pub(in crate::content::parallel) struct Diagnostics<'a> { + summary: Option>, + active: &'a AtomicUsize, +} + +impl<'a> Diagnostics<'a> { + pub(super) fn new(retired: usize, active: &'a AtomicUsize) -> Self { + let enabled = retired > 0 + && tracing::enabled!(target: "volparossa_agent::content::parallel::continuation", tracing::Level::DEBUG); + Self { + summary: enabled.then(|| { + let initial = bounded(active.load(Ordering::Acquire)); + Mutex::new(Summary { + schema_version: 1, + retired_candidates: bounded(retired), + initial_active_workers: initial, + peak_active_workers: initial, + ..Summary::default() + }) + }), + active, + } + } + + fn update(&self, update: impl FnOnce(&mut Summary)) { + if let Some(summary) = &self.summary { + update( + &mut summary + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + } + } + + pub(super) fn prepared(&self) { + self.update(|summary| { + summary.prepared_continuations = summary.prepared_continuations.saturating_add(1); + }); + } + + /// Invoked inside the real assigned-driver callback, after its lease is acquired. + pub(in crate::content::parallel) fn admitted(&self) { + self.update(|summary| { + summary.admitted_continuations = summary.admitted_continuations.saturating_add(1); + summary.peak_active_workers = summary + .peak_active_workers + .max(bounded(self.active.load(Ordering::Acquire))); + }); + } + + pub(super) fn returned_progress(&self, progress: TransferProgress) { + self.update(|summary| { + summary.returned_verified_bytes = summary + .returned_verified_bytes + .saturating_add(progress.bytes); + summary.returned_verified_chunks = summary + .returned_verified_chunks + .saturating_add(bounded(progress.chunks)); + if progress.bytes > 0 { + summary.returned_useful_continuations = + summary.returned_useful_continuations.saturating_add(1); + } + }); + } + + pub(super) fn finish(&self, returned: bool) { + self.update(|summary| { + summary.outcome = if returned { + Outcome::Returned + } else { + Outcome::Failed + }; + }); + } +} + +impl Drop for Diagnostics<'_> { + fn drop(&mut self) { + let Some(summary) = &self.summary else { return }; + let mut summary = summary + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + summary.final_active_workers = bounded(self.active.load(Ordering::Acquire)); + summary.peak_active_workers = summary + .peak_active_workers + .max(summary.final_active_workers); + // No log sink runs while the diagnostic mutex is held. + if let Ok(continuation) = serde_json::to_string(&summary) { + tracing::debug!(target: "volparossa_agent::content::parallel::continuation", + code = "CONTENT_PROVIDER_CONTINUATION_SUMMARY", continuation = %continuation); + } + } +} + +#[cfg(test)] +pub(in crate::content::parallel) mod tests { + use std::{io::Write, sync::Arc}; + + use super::*; + + #[derive(Clone, Default)] + pub(in crate::content::parallel) struct Capture(Arc>>); + + impl Write for Capture { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl Capture { + pub(in crate::content::parallel) fn subscriber( + &self, + level: tracing::Level, + ) -> impl tracing::Subscriber + Send + Sync { + let capture = self.clone(); + tracing_subscriber::fmt() + .json() + .with_max_level(level) + .with_writer(move || capture.clone()) + .finish() + } + + pub(in crate::content::parallel) fn summaries(&self) -> Vec { + self.0 + .lock() + .unwrap() + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .filter_map(|line| { + let value: serde_json::Value = serde_json::from_slice(line).unwrap(); + if value["target"] != "volparossa_agent::content::parallel::continuation" { + return None; + } + assert_eq!(value["level"], "DEBUG"); + let fields = value["fields"].as_object().unwrap(); + assert_eq!(fields.len(), 2); + assert_eq!(fields["code"], "CONTENT_PROVIDER_CONTINUATION_SUMMARY"); + Some(serde_json::from_str(fields["continuation"].as_str().unwrap()).unwrap()) + }) + .collect() + } + } + + #[test] + fn continuation_diagnostics_are_opt_in_closed_and_saturating() { + let active = AtomicUsize::new(0); + let capture = Capture::default(); + tracing::subscriber::with_default(capture.subscriber(tracing::Level::INFO), || { + let disabled = Diagnostics::new(2, &active); + assert!(disabled.summary.is_none()); + disabled.prepared(); + disabled.admitted(); + }); + assert!(capture.summaries().is_empty()); + tracing::subscriber::with_default(capture.subscriber(tracing::Level::DEBUG), || { + let absent = Diagnostics::new(0, &active); + assert!(absent.summary.is_none()); + let diagnostic = Diagnostics::new(usize::MAX, &active); + diagnostic.update(|value| { + value.prepared_continuations = u32::MAX; + value.admitted_continuations = u32::MAX; + value.returned_useful_continuations = u32::MAX; + value.returned_verified_chunks = u32::MAX; + value.returned_verified_bytes = u64::MAX; + }); + diagnostic.prepared(); + diagnostic.admitted(); + diagnostic.returned_progress(TransferProgress { + bytes: 1, + chunks: 1, + missing: 0, + }); + }); + let summaries = capture.summaries(); + assert_eq!(summaries.len(), 1); + let value = &summaries[0]; + assert_eq!(value.as_object().unwrap().len(), 11); + for key in [ + "retired_candidates", + "prepared_continuations", + "admitted_continuations", + "returned_useful_continuations", + "returned_verified_chunks", + ] { + assert_eq!(value[key], u32::MAX); + } + assert_eq!(value["returned_verified_bytes"], u64::MAX); + assert_eq!(value["schema_version"], 1); + assert_eq!(value["outcome"], "interrupted"); + assert_eq!(value["initial_active_workers"], 0); + assert_eq!(value["peak_active_workers"], 0); + assert_eq!(value["final_active_workers"], 0); + } + + #[test] + fn continuation_diagnostics_retain_unexpected_active_workers_without_admission() { + let active = AtomicUsize::new(2); + let capture = Capture::default(); + tracing::subscriber::with_default(capture.subscriber(tracing::Level::DEBUG), || { + let diagnostic = Diagnostics::new(1, &active); + diagnostic.finish(false); + active.store(3, Ordering::Release); + }); + let summaries = capture.summaries(); + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0]["outcome"], "failed"); + assert_eq!(summaries[0]["admitted_continuations"], 0); + assert_eq!(summaries[0]["initial_active_workers"], 2); + assert_eq!(summaries[0]["peak_active_workers"], 3); + assert_eq!(summaries[0]["final_active_workers"], 3); + } +} diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 7d8c51c8b..0439eb448 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,15 +29,15 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38193112015` on `24463595` again passes native and HTTPS three-provider reconstruction, but the late-provider fixture stops waiting before its first discovery completes. The fixture correction shares the existing six-second budget instead of stopping at four seconds; thirteen focused checks pass. Cancellation remains unreached. Actual late recovery passed in the preceding run; pressure-retirement continuation still needs its own protected result. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38195177153` on `c63ddf36` passes native and HTTPS three-provider reconstruction, but its first late-provider query returns empty through a newly selected broker. The six-second fixture correction cannot repair that unmet first-snapshot condition. Establish genuine signed-service readiness before the measured late-publication phase; do not inject provider hints or widen product deadlines. Cancellation and protected pressure-retirement recovery remain unproved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost; a separate exact-source probe encounters ECMWF HTTP 429. Closed retention now includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests, retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. The closed bundle profile 3 reuses the two-product acquisition/transfer path and checks exact ordered Flutter rows. Browser08 passes actual synthetic UI checks with the corrected ECMWF module, and a new immutable archive verifies against that proof. Actual joined core delivery, runtime table extraction and native Android/iOS execution remain unverified. | +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. Browser08 and its immutable input bundle retain their synthetic UI proof. A separate actual-browser test of the exact ordered Flutter tables fails at the initial table assertion; its browser, fixture and private profile are cleaned up. Diagnose the mismatch without weakening row checks or changing the frozen bundle. Joined core delivery and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Finish the standalone provider proof, which needs no weather downloads, through late discovery and -cancellation. The latest run proves concurrent reconstruction, while the -preceding run proves late recovery. Neither exercises the pressure-retirement +cancellation. The latest run proves concurrent reconstruction, while run +`38190664934` proves late recovery. Neither exercises the pressure-retirement continuation itself. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface @@ -45,7 +45,50 @@ changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. -The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38193112015) +The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38195177153) +on `c63ddf36` is **failed overall**, at the late-provider `first-snapshot` +stage. This time the broker changes from relay1 to relay2. Its first query +returns empty after 1.151 seconds, with two dial failures and no verified offer. +The second query returns an offer after another 5.063 seconds, or 6.214 seconds +from the first request. The required first completed query with exactly one +offer cannot then be established. No B activation or requester-cancellation +proof follows. The logs establish the failed connections, not their exact cause +or which bounded polling command timed out. + +A separate, bounded signed-service readiness phase is the next fixture +candidate. It must retain its own evidence, preserve the cold content cache, +and explicitly distinguish prepared control connectivity from cold discovery. +The measured first-one-offer and later-B-publication assertions stay intact. +No production address fix or longer deadline is justified by these logs alone. + +Independent reconstruction of the same run validates native and cooperative +HTTPS delivery of 15,728,640 bytes in sixty chunks from three providers. Their +triple bulk overlaps are 4.023 and 4.489 seconds. HTTPS uses one authenticated +HEAD and no origin body or range requests. Both worker allowances stay at eleven +with no pressure flags; this does not exercise pressure-retirement recovery. +Late-fetch cleanup reaps the requester and removes the gate. Guest-root +snapshots are identical, SHA-256 +`31e9d7daf3831ccf018a040bafdf7973fd0e4de41920c81d729e2ec7caefc7b5`. +Evidence is retained at `.git/ci-evidence/content-provider-hosted-c63.rDjHnL/`; +the independently rebuilt adaptive report has SHA-256 +`5a38b175f257dedecd46e30628bd046b3f8a2aab270febf075badc6a4850ba69`. +[Quality](https://github.com/VOLPAROSSA/volparossa/actions/runs/38195158877) and +[CodeQL](https://github.com/VOLPAROSSA/volparossa/actions/runs/38195155580) pass +for this exact source. No Weather acquisition is included. + +The pressure-retirement candidate now has separate, opt-in continuation +diagnostics. They distinguish retired candidates, prepared tokens, real worker +admissions and verified progress returned by successful receives, with worker +counts observed before admission and after owned work drains. Failed or +cancelled receives can already have written cache chunks: zero **returned** +bytes is not a zero-traffic or zero-write claim. Normal logging stays unchanged; +the diagnostic contains no content, paths, peer identities or raw pressure +samples. Native and HTTPS phases export independent closed records alongside +the unchanged worker-budget reports. Twelve focused Rust tests and sixteen +extractor/fixture tests pass, including cancellation and a policy error after +verified writes. Protected recovery under actual pressure remains unproved. + +The earlier [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38193112015) on `24463595` is **failed overall**, at the late-provider fixture's `first-snapshot` stage. A's offer is verified after 25 milliseconds, but the first discovery response reaches the client after 5.09 seconds. The fixture @@ -58,8 +101,9 @@ six-second deadline. Subprocesses receive only the remaining budget, capped at two seconds each; the 300-second fetch limit and production discovery limits are unchanged. Thirteen late-provider tests pass, including a reproduction of the 5.09-second response and expired discovery, activation and release cases. -Nineteen aggregate checks and ten inert cancellation checks also pass. A new -protected run must still prove the complete sequence. +Nineteen aggregate checks and ten inert cancellation checks also pass. The +subsequent `c63ddf36` run above exercises this correction but exposes the +different first-empty-query condition; the complete sequence remains unproved. Independent reconstruction again validates native and cooperative HTTPS: each delivers the exact 15,728,640-byte object in sixty chunks from three diff --git a/tests/integration/content-provider-worker-budget.py b/tests/integration/content-provider-worker-budget.py index 42e087663..43ede7945 100644 --- a/tests/integration/content-provider-worker-budget.py +++ b/tests/integration/content-provider-worker-budget.py @@ -5,6 +5,9 @@ The private markers bind byte ranges of one owned agent log. Exported reports contain only bounded budget summaries, never log envelopes, identifiers or paths. These observations do not establish transfer completion, concurrency or cleanup. +Continuation records are exported separately from the unchanged budget schema. +Returned progress excludes partial writes from failed or cancelled receives; +zero returned bytes never means that no bytes travelled or entered the cache. """ import io import json @@ -16,6 +19,8 @@ TARGET = "volparossa_agent::content::worker_budget" CODE = "CONTENT_WORKER_BUDGET_SUMMARY" +CONTINUATION_TARGET = "volparossa_agent::content::parallel::continuation" +CONTINUATION_CODE = "CONTENT_PROVIDER_CONTINUATION_SUMMARY" PHASES = ("native", "https") MAX_INPUT = 8 * 1024 * 1024 MAX_LINE = 16384 @@ -29,6 +34,9 @@ COUNTERS = {"peak_shared_held", "admitted_leases", "denied_leases", "peak_leases", "active_leases"} CAUSES = {"cpu_pressure", "memory_pressure", "io_pressure", "cpu_unavailable", "memory_unavailable", "io_unavailable", "resource_unavailable", "lock_unavailable"} +CONTINUATION_COUNTS = {"retired_candidates", "prepared_continuations", "admitted_continuations", + "returned_useful_continuations", "returned_verified_chunks", + "initial_active_workers", "peak_active_workers", "final_active_workers"} def require(condition): @@ -66,8 +74,25 @@ def summary(value): return value -def extract(raw): +def continuation_summary(value): + require(type(value) is dict and set(value) == + {"schema_version", "outcome", "returned_verified_bytes"} | CONTINUATION_COUNTS) + require(type(value["schema_version"]) is int and value["schema_version"] == 1) + require(type(value["outcome"]) is str and value["outcome"] in ("returned", "failed", "interrupted")) + for key in CONTINUATION_COUNTS | {"returned_verified_bytes"}: + maximum = 2**64 - 1 if key == "returned_verified_bytes" else U32 + require(type(value[key]) is int and 0 <= value[key] <= maximum) + require(value["returned_useful_continuations"] <= value["admitted_continuations"] + <= value["prepared_continuations"] <= value["retired_candidates"]) + require(max(value["initial_active_workers"], value["final_active_workers"]) + <= value["peak_active_workers"]) + return value + + +def extract(raw, *, continuation=False): """Accept complete JSON log lines only; never return an unvalidated raw line.""" + target, code, field, validate = (CONTINUATION_TARGET, CONTINUATION_CODE, "continuation", continuation_summary) \ + if continuation else (TARGET, CODE, "budget", summary) require(type(raw) is bytes and len(raw) <= MAX_INPUT) records = [] incomplete = False @@ -83,16 +108,16 @@ def extract(raw): continue event = decode(line) require(type(event) is dict) - if event.get("target") != TARGET: + if event.get("target") != target: continue fields = event.get("fields") require(type(fields) is dict) - if fields.get("code") != CODE: + if fields.get("code") != code: continue - require(event.get("level") == "DEBUG" and set(fields) == {"code", "budget"}) - budget = fields["budget"] - require(type(budget) is str and len(budget.encode("utf-8")) <= MAX_BUDGET) - records.append(summary(decode(budget))) + require(event.get("level") == "DEBUG" and set(fields) == {"code", field}) + payload = fields[field] + require(type(payload) is str and len(payload.encode("utf-8")) <= MAX_BUDGET) + records.append(validate(decode(payload))) require(len(records) <= MAX_RECORDS) return records, incomplete @@ -151,6 +176,12 @@ def paths(work, phase): work / f"content-provider-adaptive{suffix}-worker-budget.json") +def continuation_path(work, phase): + require(phase in PHASES) + suffix = "" if phase == "native" else "-https" + return work / f"content-provider-adaptive{suffix}-continuation.json" + + def mark(work, phase): marker, _ = paths(work, phase) fd, info = checked_open(work / "agent-client.log") @@ -168,6 +199,7 @@ def finish(work, phase, *, pending=False): marker, output = paths(work, phase) report = dict(schema_version=1, kind="content-worker-budget-diagnostic", phase=phase, phase_closed=not pending, complete=False, status="invalid", records=[]) + continuation_report = dict(report, kind="content-provider-continuation-diagnostic", records=[]) try: value = decode(read_private(marker, MAX_BUDGET)) require(type(value) is dict and set(value) == @@ -184,15 +216,21 @@ def finish(work, phase, *, pending=False): require(len(raw) == end - value["start"] and os.fstat(fd).st_size >= end) finally: os.close(fd) - records, incomplete = extract(raw) if not pending and value["end"] is None: value["end"] = end write_private(marker, value, MAX_BUDGET) - report.update(records=records, status="incomplete" if incomplete or pending else - "observed" if records else "missing", complete=bool(records) and not incomplete and not pending) + for selected, is_continuation in ((report, False), (continuation_report, True)): + try: + records, incomplete = extract(raw, continuation=is_continuation) + selected.update(records=records, status="incomplete" if incomplete or pending else + "observed" if records else "missing", + complete=bool(records) and not incomplete and not pending) + except (ValueError, KeyError, TypeError, UnicodeError, RecursionError): + pass # Invalid payloads do not erase the other closed diagnostic. except (OSError, ValueError, KeyError, TypeError, UnicodeError, RecursionError): pass # The closed invalid report replaces neither the failure nor absent evidence with success. write_private(output, report, MAX_OUTPUT) + write_private(continuation_path(work, phase), continuation_report, MAX_OUTPUT) return report diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 558b48768..49617d20c 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -3972,7 +3972,7 @@ launch_agent() { agent_rust_log=volparossa_agent=info if [ "$scenario" = content-provider ] && [ "$node" = client ]; then # Disposable fixture opt-in: closed per-batch resources only, no object/route/peer identities. - agent_rust_log=$agent_rust_log,volparossa_agent::content::worker_budget=debug + agent_rust_log=$agent_rust_log,volparossa_agent::content::worker_budget=debug,volparossa_agent::content::parallel::continuation=debug fi if [ "$scenario" = browser-network ] && [ "$node" = client ]; then agent_rust_log=$agent_rust_log,volparossa_agent::browser_gateway::connect=debug diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index cfeacfbf6..7fbf59657 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -915,6 +915,7 @@ SAFE_NAMES = {"runner.stdout", "runner.stderr", "guest-exit-status", "current-ph "private-unbound-package.json", "private-unbound-package-lifecycle.json", "worker-network-diagnostics.txt", "host-state-before.json", "host-state-after.json", "content-provider-adaptive-worker-budget.json", "content-provider-adaptive-https-worker-budget.json", + "content-provider-adaptive-continuation.json", "content-provider-adaptive-https-continuation.json", "content-provider-adaptive-resource-ready.json", "content-provider-adaptive-https-resource-ready.json", "report.json", "local-link-smoke.json", "wifi-link-smoke.json", "reciprocity-smoke.json", "mixed-link-smoke.json", "mpquic-growth-smoke.json", "mptcp-growth-smoke.json", "mptcp-refill-smoke.json", "sharing-smoke.json", "download-sharing-smoke.json", diff --git a/tests/integration/test-content-provider-worker-budget.py b/tests/integration/test-content-provider-worker-budget.py index 55204aded..e6abf26a9 100644 --- a/tests/integration/test-content-provider-worker-budget.py +++ b/tests/integration/test-content-provider-worker-budget.py @@ -32,6 +32,20 @@ def event(value=None, **changes): return json.dumps(envelope).encode() + b"\n" +def continuation(**changes): + value = dict(schema_version=1, outcome="returned", retired_candidates=2, + prepared_continuations=2, admitted_continuations=2, returned_useful_continuations=2, + returned_verified_chunks=6, returned_verified_bytes=1572864, + initial_active_workers=0, peak_active_workers=1, final_active_workers=0) + value.update(changes) + return value + + +def continuation_event(value=None): + return event(target=CHECK["CONTINUATION_TARGET"], fields=dict(code=CHECK["CONTINUATION_CODE"], + continuation=json.dumps(value or continuation()))) + + class WorkerBudgetDiagnostics(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory(prefix="provider-budget-") @@ -196,11 +210,89 @@ def test_actual_cli_marker_finish_and_failed_acceptance_retains_reports(self): (self.work / "content-provider-smoke.json").write_text('{"success":false}') self.assertEqual(self.report("native").read_bytes(), report) + def test_continuations_use_same_phase_boundary_but_separate_closed_report(self): + self.append(continuation_event(continuation(retired_candidates=99))) + self.marker("native") + self.append(event() + continuation_event()) + legacy = self.finish("native") + target = CHECK["continuation_path"](self.work, "native") + retained = target.read_bytes() + parsed = json.loads(retained) + self.assertEqual(legacy["records"], [budget()]) + self.assertEqual(parsed, dict(schema_version=1, kind="content-provider-continuation-diagnostic", + phase="native", phase_closed=True, complete=True, status="observed", records=[continuation()])) + self.assertEqual(target.stat().st_mode & 0o777, 0o600) + self.marker("https") + self.append(continuation_event(continuation(outcome="failed"))) + self.finish("https") + self.append(continuation_event(continuation(outcome="interrupted"))) + CHECK["finalize"](self.work) + self.assertEqual(target.read_bytes(), retained) + https = json.loads(CHECK["continuation_path"](self.work, "https").read_bytes()) + self.assertEqual([row["outcome"] for row in https["records"]], ["failed"]) + for secret in ("timestamp", "not-for-export", "target", "device", "inode", "success"): + self.assertNotIn(secret, retained.decode()) + + def test_continuation_pending_and_missing_are_not_success_or_zero_traffic(self): + self.marker("native") + self.finish("native", pending=True) + target = CHECK["continuation_path"](self.work, "native") + self.assertFalse(json.loads(target.read_bytes())["complete"]) + interrupted = continuation(outcome="interrupted", returned_useful_continuations=0, + returned_verified_chunks=0, returned_verified_bytes=0) + self.append(continuation_event(interrupted)) + CHECK["finalize"](self.work) + self.assertEqual(json.loads(target.read_bytes())["records"], [interrupted]) + self.assertFalse(CHECK["continuation_path"](self.work, "https").exists()) + self.marker("https"); self.finish("https") + missing = json.loads(CHECK["continuation_path"](self.work, "https").read_bytes()) + self.assertEqual((missing["status"], missing["complete"], missing["records"]), ("missing", False, [])) + + def test_continuation_rejects_unknown_fields_types_ranges_and_envelope(self): + mutations = ({"schema_version": True}, {"outcome": "complete"}, {"peer_id": "SECRET"}, + {"retired_candidates": -1}, {"prepared_continuations": 3}, {"admitted_continuations": 3}, + {"returned_useful_continuations": 3}, {"returned_verified_chunks": 2**32}, + {"returned_verified_bytes": 2**64}, {"returned_verified_bytes": True}, + {"initial_active_workers": 2}, {"final_active_workers": 2}, {"peak_active_workers": None}) + for mutation in mutations: + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + CHECK["extract"](continuation_event(continuation(**mutation)), continuation=True) + for fields in ({"code": CHECK["CONTINUATION_CODE"], "continuation": "NaN"}, + {"code": CHECK["CONTINUATION_CODE"], "continuation": '{}', "secret": 'PRIVATE'}, + {"code": CHECK["CONTINUATION_CODE"], "continuation": json.dumps(continuation()).replace( + '"schema_version": 1', '"schema_version":1,"schema_version":1')}, + {"code": CHECK["CONTINUATION_CODE"], "continuation": "x" * (CHECK["MAX_BUDGET"]+1)}): + with self.assertRaises(ValueError): + CHECK["extract"](event(target=CHECK["CONTINUATION_TARGET"], fields=fields), continuation=True) + with self.assertRaises(ValueError): + CHECK["extract"](continuation_event() * (CHECK["MAX_RECORDS"]+1), continuation=True) + maximal = continuation(retired_candidates=2**32-1, returned_verified_chunks=2**32-1, + returned_verified_bytes=2**64-1) + self.assertEqual(CHECK["extract"](continuation_event(maximal), continuation=True)[0], [maximal]) + + def test_invalid_payload_does_not_erase_other_report_or_reopen_closed_phase(self): + self.marker("native") + self.append(event() + continuation_event(continuation(peer_id="PRIVATE"))) + legacy = self.finish("native") + target = CHECK["continuation_path"](self.work, "native") + retained = target.read_bytes() + self.assertEqual(legacy["records"], [budget()]) + self.assertEqual(json.loads(retained)["status"], "invalid") + self.append(continuation_event()) + CHECK["finalize"](self.work) + self.assertEqual(target.read_bytes(), retained) + self.marker("https") + self.append(event(budget(peer_id="PRIVATE")) + continuation_event()) + self.assertEqual(self.finish("https")["status"], "invalid") + self.assertEqual(json.loads(CHECK["continuation_path"](self.work, "https").read_bytes())["records"], + [continuation()]) + def test_glue_opt_in_cleanup_order_and_closed_emergency_allowlist(self): topology = (HERE / "kvm-alpha-topology.sh").read_text() optin = ('if [ "$scenario" = content-provider ] && [ "$node" = client ]; then\n' ' # Disposable fixture opt-in: closed per-batch resources only, no object/route/peer identities.\n' - ' agent_rust_log=$agent_rust_log,volparossa_agent::content::worker_budget=debug') + ' agent_rust_log=$agent_rust_log,volparossa_agent::content::worker_budget=debug,' + 'volparossa_agent::content::parallel::continuation=debug') self.assertIn(optin, topology) retire = topology.index('for cleanup_unit in $AGENT_UNITS; do retire_unit "$cleanup_unit"') finalize = topology.index(" content_provider_worker_budget_finalize", retire) @@ -215,6 +307,7 @@ def test_glue_opt_in_cleanup_order_and_closed_emergency_allowlist(self): emergency = (HERE / "run-alpha-topology-vm.sh").read_text() for phase in CHECK["PHASES"]: self.assertIn('"' + self.report(phase).name + '"', emergency) + self.assertIn('"' + CHECK["continuation_path"](self.work, phase).name + '"', emergency) for name in ("kvm-alpha-topology.sh", "run-alpha-topology-vm.sh", "content-provider-adaptive-smoke.sh"): subprocess.run(["/bin/sh", "-n", str(HERE / name)], check=True, timeout=5) From 0e23140b903c58b5abf80502923d40f705b773c0 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 05:21:47 +0200 Subject: [PATCH 42/49] test: separate signed-service readiness from late recovery --- Cargo.lock | 2 + crates/volparossa-content/Cargo.toml | 2 + .../examples/content-acceptance-fixture.rs | 6 + .../content-acceptance-fixture/readiness.rs | 189 +++++++++++++++ docs/IMPLEMENTATION_STATUS.md | 42 +++- .../content-provider-late-recovery.py | 224 +++++++++++++++++- .../content-provider-late-recovery.sh | 3 + tests/integration/content-provider-smoke.py | 15 +- tests/integration/content-provider-smoke.sh | 2 +- tests/integration/kvm-alpha-topology.sh | 12 +- tests/integration/run-alpha-topology-vm.sh | 2 +- .../test-content-provider-late-recovery.py | 152 +++++++++++- .../test-content-provider-smoke.py | 13 +- 13 files changed, 630 insertions(+), 34 deletions(-) create mode 100644 crates/volparossa-content/examples/content-acceptance-fixture/readiness.rs diff --git a/Cargo.lock b/Cargo.lock index 8da22703b..f4716e24c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4678,6 +4678,7 @@ dependencies = [ "hex", "hpke", "http", + "libp2p-identity", "prost", "rand_chacha 0.10.0", "rand_core 0.6.4", @@ -4694,6 +4695,7 @@ dependencies = [ "tokio", "tokio-rustls", "url", + "volparossa-local-control", "zeroize", ] diff --git a/crates/volparossa-content/Cargo.toml b/crates/volparossa-content/Cargo.toml index 204c7b1c7..dc470f2ce 100644 --- a/crates/volparossa-content/Cargo.toml +++ b/crates/volparossa-content/Cargo.toml @@ -33,8 +33,10 @@ url.workspace = true zeroize.workspace = true [dev-dependencies] +libp2p-identity.workspace = true rand_core.workspace = true rcgen.workspace = true +volparossa-local-control = { path = "../volparossa-local-control" } [lints] workspace = true diff --git a/crates/volparossa-content/examples/content-acceptance-fixture.rs b/crates/volparossa-content/examples/content-acceptance-fixture.rs index 4f4d59e67..b8e84aada 100644 --- a/crates/volparossa-content/examples/content-acceptance-fixture.rs +++ b/crates/volparossa-content/examples/content-acceptance-fixture.rs @@ -10,6 +10,9 @@ mod adaptive; #[path = "content-acceptance-fixture/repair.rs"] mod repair; +#[path = "content-acceptance-fixture/readiness.rs"] +mod readiness; + use std::fs::{self, File}; use std::io::{Read, Write}; use std::net::SocketAddr; @@ -37,6 +40,9 @@ type Result = std::result::Result>; async fn main() -> Result<()> { let args: Vec = std::env::args().skip(1).collect(); match args.as_slice() { + [mode, socket, manifest, publisher] if mode == "discover-providers" => { + readiness::discover(Path::new(socket), Path::new(manifest), publisher).await + } [mode, root] if mode == "seed" => seed(Path::new(root), None), [mode, root, first, second] if mode == "seed-providers" => { seed_replicas(Path::new(root), None, Path::new(first), Path::new(second), false) diff --git a/crates/volparossa-content/examples/content-acceptance-fixture/readiness.rs b/crates/volparossa-content/examples/content-acceptance-fixture/readiness.rs new file mode 100644 index 000000000..2d79af5a1 --- /dev/null +++ b/crates/volparossa-content/examples/content-acceptance-fixture/readiness.rs @@ -0,0 +1,189 @@ +//! One normal guarded discovery request. The fixture never supplies desired providers. +use std::{fs::OpenOptions, io::Read as _, os::unix::fs::OpenOptionsExt as _, path::Path}; + +use rand_core::RngCore as _; +use serde_json::json; +use tokio::net::UnixStream; +use volparossa_local_control::{ + CONTROL_PROTOCOL_VERSION, ContentCustodyDiscoverRequest, ControlRequest, ControlResult, + control_request::Operation, control_response::Payload, read_response, write_request, +}; + +pub(super) async fn discover(socket: &Path, manifest: &Path, publisher: &str) -> super::Result<()> { + let publisher_key = hex::decode(publisher)?; + let mut file = OpenOptions::new() + .read(true) + .custom_flags(i32::try_from( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::NONBLOCK).bits(), + )?) + .open(manifest)?; + let info = file.metadata()?; + if !info.is_file() || info.len() == 0 || info.len() > 65536 { + return Err("invalid readiness manifest".into()); + } + let mut manifest = Vec::new(); + file.by_ref().take(65537).read_to_end(&mut manifest)?; + let mut request_id = [0; 16]; + rand_core::OsRng.fill_bytes(&mut request_id); + let request = ControlRequest { + protocol_version: CONTROL_PROTOCOL_VERSION, + request_id: request_id.to_vec(), + operation: Some(Operation::ContentCustodyDiscover( + ContentCustodyDiscoverRequest { + manifest, + publisher_key, + max_providers: 16, + }, + )), + }; + println!("{}", query(socket, &request).await?); + Ok(()) +} + +async fn query(socket: &Path, request: &ControlRequest) -> super::Result { + // The owner supervises one absolute preparation deadline and closes this socket + // on termination. No retry, data fetch, cache path or provider selector exists here. + let mut stream = UnixStream::connect(socket).await?; + write_request(&mut stream, request).await?; + let response = read_response(&mut stream).await?; + if response.request_id != request.request_id || response.result != ControlResult::Ok as i32 { + return Err("readiness request rejected".into()); + } + let Some(Payload::ContentCustodyDiscovered(found)) = response.payload else { + return Err("unexpected readiness response".into()); + }; + let mut providers = Vec::new(); + for offer in found.providers { + let key = libp2p_identity::ed25519::PublicKey::try_from_bytes(&offer.provider_key)?; + let key = libp2p_identity::PublicKey::from(key); + providers.push(json!({"peer_id": key.to_peer_id().to_string(), + "offer_expires_unix_seconds": offer.offer_expires_unix_seconds})); + } + Ok( + json!({"schema_version": 1, "kind": "content-provider-discovery-readiness", + "control_relay_peer_id": found.control_relay_peer_id, "providers": providers}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::{ + io::AsyncReadExt as _, + net::UnixListener, + sync::oneshot, + time::{Duration, timeout}, + }; + use volparossa_local_control::{ + ContentCustodyDiscovered, ContentCustodyProvider, ControlResponse, read_request, + write_response, + }; + + fn request() -> ControlRequest { + ControlRequest { + protocol_version: CONTROL_PROTOCOL_VERSION, + request_id: vec![7; 16], + operation: Some(Operation::ContentCustodyDiscover( + ContentCustodyDiscoverRequest { + manifest: vec![1, 2, 3], + publisher_key: vec![8; 32], + max_providers: 16, + }, + )), + } + } + + #[tokio::test] + async fn one_typed_request_correlates_and_converts_only_authenticated_reply_keys() { + for mode in ["ok", "wrong-id", "denied"] { + let directory = tempfile::tempdir().unwrap(); + let socket = directory.path().join("agent.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let key = ed25519_dalek::SigningKey::from_bytes(&[9; 32]) + .verifying_key() + .to_bytes(); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_request(&mut stream).await.unwrap(); + let Some(Operation::ContentCustodyDiscover(discover)) = request.operation else { + panic!("unexpected operation") + }; + assert_eq!(discover.manifest, [1, 2, 3]); + assert_eq!(discover.publisher_key, [8; 32]); + assert_eq!(discover.max_providers, 16); + let response = ControlResponse { + protocol_version: CONTROL_PROTOCOL_VERSION, + request_id: if mode == "wrong-id" { + vec![0; 16] + } else { + request.request_id + }, + result: if mode == "denied" { + ControlResult::Policy + } else { + ControlResult::Ok + } as i32, + diagnostic_code: "CONTENT_CUSTODY_DISCOVERED".into(), + payload: Some(Payload::ContentCustodyDiscovered( + ContentCustodyDiscovered { + control_relay_peer_id: "broker".into(), + providers: vec![ContentCustodyProvider { + provider_key: key.to_vec(), + offer_expires_unix_seconds: 12345, + }], + }, + )), + }; + write_response(&mut stream, &response).await.unwrap(); + let mut byte = [0]; + assert_eq!( + stream.read(&mut byte).await.unwrap(), + 0, + "no second request or payload stream" + ); + }); + let result = timeout(Duration::from_secs(2), query(&socket, &request())) + .await + .unwrap(); + if mode == "ok" { + let key = libp2p_identity::PublicKey::from( + libp2p_identity::ed25519::PublicKey::try_from_bytes(&key).unwrap(), + ); + assert_eq!( + result.unwrap(), + json!({"schema_version":1,"kind":"content-provider-discovery-readiness", + "control_relay_peer_id":"broker","providers":[{"peer_id":key.to_peer_id().to_string(),"offer_expires_unix_seconds":12345}]}) + ); + } else { + assert!(result.is_err()); + } + timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(); + } + } + + #[tokio::test] + async fn dropping_pending_query_closes_exact_socket_without_another_request() { + let directory = tempfile::tempdir().unwrap(); + let socket = directory.path().join("agent.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let (ready, received) = oneshot::channel(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + read_request(&mut stream).await.unwrap(); + ready.send(()).unwrap(); + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.unwrap(), 0); + }); + let request = request(); + let mut query = Box::pin(query(&socket, &request)); + tokio::select! { _ = &mut query => panic!("unexpected reply"), _ = received => {} } + drop(query); + timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + } +} diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 0439eb448..f4d2f4047 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,9 +29,9 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38195177153` on `c63ddf36` passes native and HTTPS three-provider reconstruction, but its first late-provider query returns empty through a newly selected broker. The six-second fixture correction cannot repair that unmet first-snapshot condition. Establish genuine signed-service readiness before the measured late-publication phase; do not inject provider hints or widen product deadlines. Cancellation and protected pressure-retirement recovery remain unproved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38195177153` on `c63ddf36` passes native and HTTPS three-provider reconstruction, but its first late-provider query returns empty through a newly selected broker. A separate signed-service readiness prerequisite is now implemented and component-tested, without provider hints or longer product deadlines. Its live execution, cancellation and protected pressure-retirement recovery remain unproved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost; a separate exact-source probe encounters ECMWF HTTP 429. Closed retention now includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests, retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. Browser08 and its immutable input bundle retain their synthetic UI proof. A separate actual-browser test of the exact ordered Flutter tables fails at the initial table assertion; its browser, fixture and private profile are cleaned up. Diagnose the mismatch without weakening row checks or changing the frozen bundle. Joined core delivery and native Android/iOS execution remain unverified. | +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. Browser08 and its immutable input bundle retain their synthetic UI proof. The exact table verifier is corrected and passes a separate actual-browser trial with synthetic products, before and after pointer refresh. Rebuild a new immutable bundle with that verifier; do not change the old bundle. Joined core delivery and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Finish the standalone @@ -55,11 +55,39 @@ offer cannot then be established. No B activation or requester-cancellation proof follows. The logs establish the failed connections, not their exact cause or which bounded polling command timed out. -A separate, bounded signed-service readiness phase is the next fixture -candidate. It must retain its own evidence, preserve the cold content cache, -and explicitly distinguish prepared control connectivity from cold discovery. -The measured first-one-offer and later-B-publication assertions stay intact. -No production address fix or longer deadline is justified by these logs alone. +A separate signed-service readiness prerequisite is now implemented in the +fixture. It uses the existing typed `ContentCustodyDiscover` request, at most +twice within one thirty-second deadline. No desired provider or address is sent; +the fixture checks the returned signed A offer only afterward. It retains all +attempts and separate real broker/client events, verifies unchanged routes and +broker, keeps B off and leaves the content cache cold. Real owner-first interface +accounting remains enabled; the role-isolated client adds no contribution queues. + +The measured first-one-offer and later-B-publication assertions start after +preparation and retain the original six-second gate and 300-second fetch limits. +The receipt explicitly says **warm control plane, cold content cache**, not cold +discovery. Owned helper termination closes its typed socket and is reaped; +preparation/setup time consumes the original deadline. Two Rust socket tests, +seventeen late-provider tests, nineteen aggregate checks and ten existing inert +cancellation checks pass, with an independent review. Current aggregate schema 3 +requires late-component schema 2 and its readiness evidence; historical schemas +remain explicitly separate. Live readiness and the complete sequence are still +unproved. No production address fix is claimed. + +The separate Overmorrow table trial now passes in the actual isolated Firefox +CanvasKit UI using synthetic products. Earlier extraction assigned both tables +to the last preceding title; Flutter can combine the metadata labels before the +table children. The corrected verifier identifies tables by exact static ordered +headers, then checks all original row/cell values and title-grouped metadata. +It does not assert separate product DOM containers. Four weather and five +particulate rows match before and after a real pointer refresh, with exactly two +requests per product. Browser, fixture, profile and namespace cleanup pass. +The retained report at Weather's `build/overmorrow-table-browser-04/report.json` +has SHA-256 `3ed8ceeaf89aa22bb2c9fd656c9b4228dd5f75b12e87424820bd430dc9dc214d`. +Five pure DOM regressions accompany the existing exact-value checks. This proves +the corrected UI verifier, not core, origin, cold or native-mobile delivery. +Old reports and the frozen bundle are unchanged; a future joined trial needs a +new bundle with the corrected driver. Independent reconstruction of the same run validates native and cooperative HTTPS delivery of 15,728,640 bytes in sixty chunks from three providers. Their diff --git a/tests/integration/content-provider-late-recovery.py b/tests/integration/content-provider-late-recovery.py index fe1cf3273..b836d1547 100644 --- a/tests/integration/content-provider-late-recovery.py +++ b/tests/integration/content-provider-late-recovery.py @@ -22,11 +22,12 @@ TABLE = "vpa_content_late_gate" LIMIT = 262144 GATE_NS = 6_000_000_000 +PREPARE_NS = 30_000_000_000 BYTES = 2097275 SHA = hashlib.sha256(b"".join(bytes([16+i])*262144 for i in range(8)) + bytes([25])*123).hexdigest() EVENTS = ("CONTENT_DISCOVERY_COMPLETED", "CONTENT_DISCOVERY_RECEIVED_OFFERS", "CONTENT_DISCOVERY_RECEIVED_EMPTY", "CONTENT_PROVIDER_QUERY_STARTED", "CONTENT_PROVIDER_OFFER_VERIFIED") -FAILURE_STAGES = ("gate-install", "gate-inspect", "fetch-start", "first-snapshot", "activate-b", +FAILURE_STAGES = ("readiness", "gate-install", "gate-inspect", "fetch-start", "first-snapshot", "activate-b", "gate-release", "fetch-wait", "final-snapshot", "output") FAILURE_CATEGORIES = ("command-failed", "timeout", "interrupted", "os-error", "invalid-evidence") @@ -37,6 +38,10 @@ def __init__(self, returncode): self.returncode = returncode +class ReadinessCleanupError(OSError): + pass + + def failure_evidence(stage, error): category, code = "invalid-evidence", None if isinstance(error, FixtureCommandError): category, code = "command-failed", error.returncode @@ -160,8 +165,8 @@ def process_identity(pid): return dict(pid=pid, start_ticks=int(fields[19])) -def stop_fetch(work): - record = path(work, "fetch", "state") +def stop_fetch(work, suffix="fetch"): + record = path(work, suffix, "state") if not record.exists() and not record.is_symlink(): return guest(work); identity = private_read(record) require(set(identity) == {"pid", "start_ticks"} and all(type(v) is int and v > 1 for v in identity.values()), @@ -182,7 +187,9 @@ def stop_fetch(work): def cleanup(work): # Gate release must not wait for child termination near the connect deadline. try: remove_gate(work) - finally: stop_fetch(work) + finally: + try: stop_fetch(work) + finally: stop_fetch(work, "readiness-process") def event_rows(raw, baseline): @@ -205,6 +212,193 @@ def event_rows(raw, baseline): def count(rows, code): return sum(row["event"] == code for row in rows) +def discovery_reply(value): + require(type(value) is dict and set(value) == {"schema_version", "kind", "control_relay_peer_id", "providers"} + and type(value["schema_version"]) is int and value["schema_version"] == 1 + and value["kind"] == "content-provider-discovery-readiness" + and type(value["control_relay_peer_id"]) is str and 0 < len(value["control_relay_peer_id"]) <= 128 + and type(value["providers"]) is list and len(value["providers"]) <= 16, "invalid readiness reply") + peers = set() + for offer in value["providers"]: + require(type(offer) is dict and set(offer) == {"peer_id", "offer_expires_unix_seconds"} + and type(offer["peer_id"]) is str and 0 < len(offer["peer_id"]) <= 128 + and offer["peer_id"] not in peers and type(offer["offer_expires_unix_seconds"]) is int + and 0 < offer["offer_expires_unix_seconds"] < 2**64, "invalid readiness offer") + peers.add(offer["peer_id"]) + return value + + +def readiness_paths(raw, selected): + rows = raw.decode("ascii").splitlines() + require(0 < len(rows) <= 8, "wrong readiness paths") + observed = [] + for row in rows: + match = re.fullmatch(r"context=([0-9a-f]{32}) path=([1-8]) relay=(\S+) exit=(\S+) " + r"state=[0-9]+ rtt_us=[0-9]+ bytes=[0-9]+ acked_transport_bytes=[0-9]+", row) + require(match is not None, "invalid readiness path row") + context, ident, relay, exit_peer = match.groups() + require(context == selected["route_context_id"] and exit_peer == selected["exact_selected_exit"], "readiness route changed") + observed.append((int(ident), relay)) + expected = [(row["path_id"], row["relay_peer_id"]) for row in selected["paths"]] + require(sorted(observed) == sorted(expected), "readiness path selection changed") + + +def readiness_request(work, executable, args, deadline): + """One owned typed helper; termination closes its actual agent socket (EOF).""" + remaining = (deadline-time.monotonic_ns()) / 1e9 + require(remaining > 0, "readiness deadline exceeded") + process = subprocess.Popen([str(executable), "discover-providers", *args], stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + try: + # Replaced only after the previous owned child has been reaped. + record = path(work, "readiness-process", "state") + if record.exists(): + private_read(record); record.unlink() + write(record, process_identity(process.pid)) + remaining = (deadline-time.monotonic_ns()) / 1e9 + if remaining <= 0: raise subprocess.TimeoutExpired("readiness", 0) + stdout, _ = process.communicate(timeout=remaining) + if process.returncode: raise FixtureCommandError(process.returncode) + require(len(stdout) <= 8192, "oversized readiness reply") + return discovery_reply(json.loads(stdout)) + finally: + try: + if process.poll() is None: + try: process.kill(); process.wait(timeout=2) + except (OSError, subprocess.SubprocessError) as error: + raise ReadinessCleanupError("readiness child cleanup unconfirmed") from error + finally: + if process.stdout is not None: process.stdout.close() + + +def validate_readiness(value, publication, layout, selected, peers, *, successful=True): + require(type(value) is dict and set(value) == {"schema_version", "kind", "success", "scope", "started_ns", + "completed_ns", "event_baseline_unix_ms", "completed_unix_ms", "route_context_id", "manifest_id", + "control_relay_peer_id", "attempts", "checks", "failure", "events"} + and type(value["schema_version"]) is int and value["schema_version"] == 1 + and value["kind"] == "content-provider-late-readiness" + and value["scope"] == "warm-control-plane-cold-content-cache" and type(value["success"]) is bool, + "invalid readiness record") + require(all(type(value[k]) is int and value[k] > 0 for k in + ("started_ns", "completed_ns", "event_baseline_unix_ms", "completed_unix_ms")) + and value["completed_ns"] >= value["started_ns"] + and value["completed_unix_ms"] >= value["event_baseline_unix_ms"] + and value["route_context_id"] == selected["route_context_id"] + and value["manifest_id"] == publication["manifest_id"] + and value["control_relay_peer_id"] == layout["control_relay_peer_id"], "readiness binding changed") + attempts = value["attempts"] + require(type(attempts) is list and len(attempts) <= 2, "readiness request count exceeded") + previous = value["started_ns"] + for attempt in attempts: + require(set(attempt) == {"started_ns", "completed_ns", "response", "failure"} + and all(type(attempt[k]) is int for k in ("started_ns", "completed_ns")) + and previous <= attempt["started_ns"] <= attempt["completed_ns"] <= value["completed_ns"], "readiness attempt ordering") + previous = attempt["completed_ns"] + require((attempt["response"] is None) != (attempt["failure"] is None), "ambiguous readiness attempt") + if attempt["response"] is not None: discovery_reply(attempt["response"]) + else: validate_failure(attempt["failure"]) + require(set(value["checks"]) == {"route_unchanged", "b_not_serving", "cold_cache", "children_reaped"} + and all(type(item) is bool for item in value["checks"].values()), "invalid readiness checks") + if value["failure"] is not None: validate_failure(value["failure"]) + events = value["events"] + require(events is None or type(events) is dict and set(events) == {"client", "broker"}, "invalid readiness events") + if events is not None: + for rows in events.values(): + require(type(rows) is list and len(rows) <= 400 and all(set(row) == {"unix_ms", "event"} + and type(row["unix_ms"]) is int and row["unix_ms"] > value["event_baseline_unix_ms"] + and row["unix_ms"] <= value["completed_unix_ms"] + and row["event"] in EVENTS for row in rows), "invalid preparation event") + if successful or value["success"]: + require(value["success"] and value["failure"] is None and 1 <= len(attempts) <= 2 + and all(value["checks"].values()) and value["completed_ns"]-value["started_ns"] <= PREPARE_NS + and events is not None, "readiness prerequisite failed") + found = discovery_reply(attempts[-1]["response"]) + require(found["control_relay_peer_id"] == layout["control_relay_peer_id"] + and len(found["providers"]) == 1 and found["providers"][0]["peer_id"] == peers["relay4"] + and found["providers"][0]["offer_expires_unix_seconds"]*1000 > value["completed_unix_ms"], + "A signed offer not ready through selected broker") + require(all(item["failure"] is None for item in attempts) + and all(item["response"]["control_relay_peer_id"] == layout["control_relay_peer_id"] for item in attempts) + and all(not item["response"]["providers"] for item in attempts[:-1]) + and count(events["client"], EVENTS[0]) == count(events["broker"], EVENTS[3]) == len(attempts) + and count(events["client"], EVENTS[1]) == 1 + and count(events["client"], EVENTS[2]) == len(attempts)-1 + and 1 <= count(events["broker"], EVENTS[4]) <= 16*len(attempts), + "preparation did not retain real completed discovery") + + +def prepare(work, binary, executable): + guest(work) + require(executable.resolve() == executable and executable.is_file() and binary.resolve() == binary + and binary.is_file(), "unsafe readiness executable") + publication, layout, selected, peers = (read(path(work, name)) for name in ("publication", "layout", "selection", "peers")) + broker = next(node for node in ("relay0", "relay1", "relay2") if peers[node] == layout["control_relay_peer_id"]) + native = work / "state-client/content-late-recovery" + started = time.monotonic_ns(); deadline = started + PREPARE_NS + value = dict(schema_version=1, kind="content-provider-late-readiness", success=False, + scope="warm-control-plane-cold-content-cache", started_ns=started, completed_ns=started, + event_baseline_unix_ms=time.time_ns()//1000000, completed_unix_ms=time.time_ns()//1000000, + route_context_id=selected["route_context_id"], manifest_id=publication["manifest_id"], + control_relay_peer_id=layout["control_relay_peer_id"], attempts=[], events=None, failure=None, + checks=dict(route_unchanged=False, b_not_serving=False, cold_cache=False, children_reaped=True)) + def cli(node, *args): return [str(binary), "--control-socket", str(work / f"runtime-{node}/control/agent.sock"), *args] + def check_scope(): + readiness_paths(command(cli("client", "paths"), deadline), selected) + status = json.loads(command(cli("client", "content", "status"), deadline)) + require(status["control_relay_peer_id"] == layout["control_relay_peer_id"], "readiness broker changed") + status = json.loads(command(cli("relay5", "content", "status"), deadline)) + require(status["serving"] is False and status["publications"] == 0, "B served before late activation") + require(all(not (native / name).exists() and not (native / name).is_symlink() for name in ("cache", "object.bin")), "readiness warmed cache") + def snapshot(): + return {label: event_rows(command(cli(node, "logs", "--limit", "400"), deadline), value["event_baseline_unix_ms"]) + for label, node in (("client", "client"), ("broker", broker))} + def interrupted(_signal, _frame): raise InterruptedError("preparation interrupted") + handlers = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} + try: + time.sleep(0.002) # Cross only the event clock tick, not a readiness delay. + check_scope() + for _ in range(2): + attempt = dict(started_ns=time.monotonic_ns(), completed_ns=0, response=None, failure=None) + value["attempts"].append(attempt) + try: + attempt["response"] = readiness_request(work, executable, [str(work / "runtime-client/control/agent.sock"), + str(native / "manifest.bin"), publication["publisher_hex"]], deadline) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: + attempt["failure"] = failure_evidence("readiness", error) + raise + finally: attempt["completed_ns"] = time.monotonic_ns() + found = attempt["response"] + require(found["control_relay_peer_id"] == layout["control_relay_peer_id"], "readiness broker changed") + value["events"] = snapshot() + if found["providers"]: break + check_scope() + value["checks"].update(route_unchanged=True, b_not_serving=True, cold_cache=True) + value["events"] = snapshot() + value.update(success=True, completed_ns=time.monotonic_ns(), completed_unix_ms=time.time_ns()//1000000) + validate_readiness(value, publication, layout, selected, peers) + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: + value.update(success=False, failure=failure_evidence("readiness", error)) + if isinstance(error, ReadinessCleanupError): value["checks"]["children_reaped"] = False + if time.monotonic_ns() < deadline: + try: value["events"] = snapshot() + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError): pass + raise + finally: + # Preserve the exact successful decision time already checked against + # the deadline. Disk retention is not another discovery attempt. + if not value["success"]: + value.update(completed_ns=time.monotonic_ns(), completed_unix_ms=time.time_ns()//1000000) + try: write(path(work, "readiness"), value) + finally: + for sig, handler in handlers.items(): signal.signal(sig, handler) + + +def checked_readiness(work, publication, layout, peers): + value = private_read(path(work, "readiness")) + validate_readiness(value, publication, layout, read(path(work, "selection")), peers) + return value + + def first_snapshot(snapshot): return (count(snapshot["client"], EVENTS[0]) == count(snapshot["client"], EVENTS[1]) == 1 and count(snapshot["client"], EVENTS[2]) == 0 @@ -215,6 +409,7 @@ def run(work, binary, provider_namespace): guest(work); ns = namespace(work) require(ns["name"] == provider_namespace and binary.resolve() == binary and binary.is_file(), "wrong executable/namespace") layout, peers, publication = (read(path(work, name)) for name in ("layout", "peers", "publication")) + preparation = checked_readiness(work, publication, layout, peers) control = [node for node in ("relay0", "relay1", "relay2") if peers[node] == layout["control_relay_peer_id"]] require(len(control) == 1 and layout["provider_nodes"] == list(NODES), "wrong control/provider layout") native = work / "state-client/content-late-recovery" @@ -225,6 +420,9 @@ def snapshot(baseline, deadline): return {label: event_rows(command(cli(node, "logs", "--limit", "400"), deadline), baseline) for label, node in (("client", "client"), ("broker", control[0]))} baseline = time.time_ns() // 1000000 + if baseline <= preparation["completed_unix_ms"]: + time.sleep(0.002); baseline = time.time_ns() // 1000000 + require(baseline > preparation["completed_unix_ms"], "preparation overlaps measured interval") time.sleep(0.002) # Only cross the event clock tick, never wait for provider readiness. tables = json.loads(command(["ip", "netns", "exec", ns["name"], "nft", "-j", "list", "tables"], time.monotonic_ns()+2_000_000_000)) @@ -312,8 +510,9 @@ def interrupted(_signal, _frame): raise InterruptedError("fixture interrupted") for sig, handler in handlers.items(): signal.signal(sig, handler) -def validate(value): - require(value["success"] is True and value["schema_version"] == 1 +def validate(value, *, historical=False): + require(value["success"] is True and type(value["schema_version"]) is int + and value["schema_version"] == (1 if historical else 2) and value["concurrent_provider_overlap_claimed"] is False, "wrong late-recovery scope") publication, output, fetch = (value[k] for k in ("publication", "output", "fetch")) require(publication["report_kind"] == "volparossa-content-late-provider-seed" @@ -336,6 +535,13 @@ def validate(value): before, live = value["selected_before"], value["selected_route"] require(before["route_context_id"] == live["route_context_id"] != value["previous_context"] and before["benchmark_slots"] == live["benchmark_slots"] and before["transport"] == "mptcp", "route changed or not fresh") + if historical: + require("readiness" not in value, "historical recovery cannot claim readiness") + else: + validate_readiness(value["readiness"], publication, layout, before, peers) + require(value["readiness"]["completed_ns"] <= output["gate_started_ns"] + and value["readiness"]["completed_unix_ms"] < output["event_baseline_unix_ms"], + "readiness overlaps measured recovery") BASE["validate_path"](value, peers, list(NODES), False) for node, size in zip(NODES, (1048699, 1048576)): require(size <= value["privacy"]["exit"]["provider_application"][node]["response_payload_bytes"] <= size+131072, @@ -395,8 +601,9 @@ def build_evidence(work): value = {key: read(path(work, suffix)) for key, suffix in (("publication", "publication"), ("layout", "layout"), ("status_before", "status-before"), ("status_after", "status-after"), ("fetch", "fetch"), ("output", "object"), ("selected_before", "selection"), ("selected_route", "live-selection"), ("cleanup", "cleanup"), - ("first_snapshot", "first-snapshot"), ("final_snapshot", "final-snapshot"), ("gate", "gate-installed"), ("run_cleanup", "run-cleanup"))} - value.update(schema_version=1, success=True, concurrent_provider_overlap_claimed=False, + ("first_snapshot", "first-snapshot"), ("final_snapshot", "final-snapshot"), ("gate", "gate-installed"), ("run_cleanup", "run-cleanup"), + ("readiness", "readiness"))} + value.update(schema_version=2, success=True, concurrent_provider_overlap_claimed=False, expected_peers=read(work / "a01-expected-peers.json"), previous_context=read(work / "content-provider-adaptive-selection.json")["route_context_id"], providers={node: {op: read(path(work, f"{node}-{op}")) for op in ("before", "serve", "stop")} for node in NODES}, @@ -412,6 +619,7 @@ def build_evidence(work): def main(): action, *args = sys.argv[1:] if action == "run" and len(args) == 3: run(Path(args[0]), Path(args[1]), args[2]) + elif action == "prepare" and len(args) == 3: prepare(*(Path(arg) for arg in args)) elif action == "cleanup" and len(args) == 1: cleanup(Path(args[0])) elif action == "evidence" and len(args) == 2: write(Path(args[1]), build_evidence(Path(args[0]))) else: raise ValueError("invalid late recovery action") diff --git a/tests/integration/content-provider-late-recovery.sh b/tests/integration/content-provider-late-recovery.sh index 85405a27b..f7a9d0484 100644 --- a/tests/integration/content-provider-late-recovery.sh +++ b/tests/integration/content-provider-late-recovery.sh @@ -94,6 +94,9 @@ content_provider_late_recovery_run() { PROVIDER_CONTROL_PID=$! wait_observer "$PROVIDER_CONTROL_PID" "$WORK/$late_prefix-control-privacy.ready" \ || fail CONTENT_PROVIDER_LATE_CONTROL_CAPTURE_FAILED + PHASE=content-provider-late-recovery-readiness + python3 -B "$WORK/bin/content-provider-late-recovery.py" prepare "$WORK" "$binary_directory/volparossa" \ + "$binary_directory/examples/content-acceptance-fixture" || fail CONTENT_PROVIDER_LATE_READINESS_FAILED PHASE=content-provider-late-recovery-fetch python3 -B "$WORK/bin/content-provider-late-recovery.py" run "$WORK" "$binary_directory/volparossa" "$R4" \ || fail CONTENT_PROVIDER_LATE_FETCH_FAILED diff --git a/tests/integration/content-provider-smoke.py b/tests/integration/content-provider-smoke.py index ae6ea2104..a261d301c 100644 --- a/tests/integration/content-provider-smoke.py +++ b/tests/integration/content-provider-smoke.py @@ -164,7 +164,7 @@ def build_named_publication(work, nodes): serves={node: read(work / f"content-provider-named-{node}-serve.json") for node in nodes}) -def validate_transfer(evidence, *, historical=False): +def validate_transfer(evidence, *, historical=False, historical_late=False): publication, output, fetch = evidence["publication"], evidence["output"], evidence["fetch"] require(publication["bytes"] == output["bytes"] == fetch["bytes"] == BYTES and publication["object_sha256"] == output["sha256"] == SHA @@ -305,12 +305,12 @@ def validate_transfer(evidence, *, historical=False): and adaptive["publication"]["publisher_hex"] != publication["publisher_hex"] and adaptive["publication"]["manifest_id"] != publication["manifest_id"], "adaptive proof substituted the topology or reused the earlier route/publication") - # Historical component evidence predates this phase; only new aggregate v2 + # Historical component evidence predates this phase; only current aggregates # acceptance requires it. If supplied, it must validate and bind to this run. if "late_provider_recovery" in evidence: late_check = runpy.run_path(str(Path(__file__).with_name("content-provider-late-recovery.py"))) late = evidence["late_provider_recovery"] - late_check["validate"](late) + late_check["validate"](late, historical=historical_late) require(late["expected_peers"] == peers and late["previous_context"] == adaptive["selected_route"]["route_context_id"] and late["selected_route"]["route_context_id"] not in { @@ -430,11 +430,11 @@ def validate_control_underlay(evidence, control_node, provider_nodes): def validate_report(report, revision, *, historical=False): - expected_version = 1 if historical else 2 require(type(report.get("schema_version")) is int - and report["schema_version"] == expected_version, + and report["schema_version"] in ((1, 2) if historical else (3,)), "wrong aggregate report version for current or historical validation") - if historical: + legacy_without_late = historical and report["schema_version"] == 1 + if legacy_without_late: require("late_provider_recovery" not in report and "late_provider_recovery" not in report["transfer"], "historical v1 report must not claim the later recovery phase") @@ -458,7 +458,8 @@ def validate_report(report, revision, *, historical=False): and report["host_state"]["before_sha256"] == report["host_state"]["after_sha256"] and re.fullmatch(r"[0-9a-f]{64}", report["host_state"]["before_sha256"]) and all(report[flag] is False for flag in SCOPE), "wrong source, cleanup, host state or scope") - validate_transfer(report["transfer"], historical=historical) + validate_transfer(report["transfer"], historical=legacy_without_late, + historical_late=historical and report["schema_version"] == 2) if __name__ == "__main__": diff --git a/tests/integration/content-provider-smoke.sh b/tests/integration/content-provider-smoke.sh index e4f3608fa..0c7be8225 100644 --- a/tests/integration/content-provider-smoke.sh +++ b/tests/integration/content-provider-smoke.sh @@ -356,7 +356,7 @@ content_provider_finalize_report() { --argjson complete "$CLEANUP_COMPLETE" \ --argjson remaining "$REMAINING_OWNED_OBJECTS" ' $evidence_input[0] as $evidence | $host_input[0] as $host | - {schema_version:2,report_kind:"volparossa-native-content-providers", + {schema_version:3,report_kind:"volparossa-native-content-providers", source_revision:$revision,run_id:$run_id,phase:$phase, success:($status == 0 and $evidence.success == true and $evidence.ordinary_publication.success == true and $evidence.site_publication.success == true and diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index 49617d20c..c80b5eb5e 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -565,8 +565,9 @@ print_plan() { ' grant only control-group traversal and remove the exact temporary publisher identity/passphrase;' \ ' pack/publish a normal signed static site, import two replicas, remove original publisher files;' \ ' open by publisher/name in the Client, verify HTTP assets/ranges and SIGTERM spool cleanup (no browser engine);' \ - ' after existing phases, create three separate control-only broker links to R3/R4/R5 and fetch 15 disjoint chunks on a fresh route;' \ + ' after existing phases, create three separate control-only broker links to R3/R4/R5 and fetch 60 disjoint chunks on a fresh route;' \ ' require real triple provider payload overlap, exact reconstruction and normal service/route teardown;' \ + ' prepare the selected late broker using at most two guarded signed-offer queries within 30s; keep content cold;' \ ' then admit a complementary provider only after one real one-offer discovery snapshot on another fresh route;' \ ' bound the disposable provider gate below the unchanged Exit connect timeout; require real rediscovery or fail;' \ ' pause only the exact guest Client helper via pidfd, cancel a pending named download and require independent cached delivery;' \ @@ -3582,6 +3583,15 @@ write_config() { printf 'download_sharing:\n enabled: true\n interface: ar2\n' printf ' total_download_mbps: 100\n contribution_download_ceiling_mbps: 1\n' fi + if [ "$scenario" = content-provider ] && [ "$node" = client ]; then + # Real owner-first interface accounting for the bounded discovery prerequisite. + # This disposable Client has neither Relay nor Exit enabled, so these + # flags do not install contribution queues or receive-accounting rules. + printf 'sharing:\n enabled: true\n interface: cr0\n' + printf ' total_upload_mbps: 100\n contribution_upload_ceiling_mbps: 1\n' + printf 'download_sharing:\n enabled: true\n interface: cr2\n' + printf ' total_download_mbps: 100\n contribution_download_ceiling_mbps: 1\n' + fi [ "$wifi_link" != yes ] || wifi_link_config if [ "$scenario" = content-custody ] || [ "$scenario" = agent-artifact ] || [ "$scenario" = agent-jobs ]; then content_custody_config; fi [ "$scenario" != content-repair ] || content_repair_config diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index 7fbf59657..f832954bb 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1116,7 +1116,7 @@ WEATHER_PRODUCTS_FAILURE_NAMES = {f"weather-products-delivery-{suffix}.json" for "acquisition-failure", "export-status", "tree-before", "input-cleanup")} WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", "weather-source-observer.mjs", *WEATHER_CLIENT_HELPERS) LATE_PROVIDER_NAMES = {f"content-provider-late-recovery-{suffix}.json" for suffix in ( - "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure")} + "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure", "readiness")} def joined_client_helper(work, names, entrypoint): diff --git a/tests/integration/test-content-provider-late-recovery.py b/tests/integration/test-content-provider-late-recovery.py index 46e6c6363..75113f5c5 100644 --- a/tests/integration/test-content-provider-late-recovery.py +++ b/tests/integration/test-content-provider-late-recovery.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-only """Synthetic late-recovery gates, not a live DHT/datapath proof.""" import copy +import io import json from pathlib import Path import runpy @@ -32,7 +33,7 @@ def gate_fixture(): def fixture(control_node="relay2"): value = ADAPTIVE["fixture"](control_node) for key in ("https", "control_filter", "provider_payload_overlap_ns"): value.pop(key) - value.update(schema_version=1, concurrent_provider_overlap_claimed=False, previous_context="b"*32, gate=gate_fixture()) + value.update(schema_version=2, concurrent_provider_overlap_claimed=False, previous_context="b"*32, gate=gate_fixture()) value["publication"] = dict(report_kind="volparossa-content-late-provider-seed", bytes=CHECK["BYTES"], chunks=9, object_sha256=CHECK["SHA"], name="disposable-late-native-publication", publisher_hex="3"*64, manifest_id="f"*64, publisher_removed=True, publisher_private_key_persisted=False, recipient_encrypted=False, @@ -77,6 +78,16 @@ def row(event, when): return dict(event=event, unix_ms=when) value["run_cleanup"] = dict(fetch_reaped=True, gate_removed=True, operation_completed=True) value["cleanup"] = dict(route_disconnected=True, active_contexts=0, paths_empty=True, client_output_removed=True, client_manifest_removed=True, client_mount_isolated=True) + value["readiness"] = dict(schema_version=1, kind="content-provider-late-readiness", success=True, + scope="warm-control-plane-cold-content-cache", started_ns=100_000_000, completed_ns=500_000_000, + event_baseline_unix_ms=800, completed_unix_ms=999, route_context_id="c"*32, + manifest_id=value["publication"]["manifest_id"], control_relay_peer_id=value["layout"]["control_relay_peer_id"], + attempts=[dict(started_ns=200_000_000, completed_ns=400_000_000, failure=None, + response=dict(schema_version=1, kind="content-provider-discovery-readiness", + control_relay_peer_id=value["layout"]["control_relay_peer_id"], + providers=[dict(peer_id=peers["relay4"], offer_expires_unix_seconds=4600)]))], + checks=dict(route_unchanged=True, b_not_serving=True, cold_cache=True, children_reaped=True), failure=None, + events=dict(client=[row(events[1], 900), row(events[0], 900)], broker=[row(events[3], 850), row(events[4], 899)])) return value @@ -86,7 +97,7 @@ def raw_files(value): ("status_before", "status-before"), ("status_after", "status-after"), ("fetch", "fetch"), ("output", "object"), ("selected_before", "selection"), ("selected_route", "live-selection"), ("cleanup", "cleanup"), ("first_snapshot", "first-snapshot"), ("final_snapshot", "final-snapshot"), ("gate", "gate-installed"), - ("run_cleanup", "run-cleanup"))} + ("run_cleanup", "run-cleanup"), ("readiness", "readiness"))} files["a01-expected-peers.json"] = value["expected_peers"] broker = next(node for node in ("relay0", "relay1", "relay2") if value["expected_peers"][node] == value["layout"]["control_relay_peer_id"]) @@ -112,6 +123,133 @@ def disjoint_snapshot_fixture(): class LateRecovery(unittest.TestCase): + def test_configuration_is_client_only_and_preparation_precedes_unchanged_gate(self): + root = HERE.joinpath('kvm-alpha-topology.sh').read_text() + block = root.split('if [ "$scenario" = content-provider ] && [ "$node" = client ]; then', 1)[1].split('\n fi', 1)[0] + self.assertIn('interface: cr0', block); self.assertIn('interface: cr2', block) + self.assertNotIn('content_contribution:', block) + self.assertIn('write_config client null false false', root) + shell = HERE.joinpath('content-provider-late-recovery.sh').read_text() + self.assertLess(shell.index('wait_observer "$PROVIDER_CONTROL_PID"'), shell.index('.py" prepare')) + self.assertLess(shell.index('.py" prepare'), shell.index('.py" run')) + self.assertEqual(CHECK['GATE_NS'], 6_000_000_000) + self.assertEqual(CHECK['PREPARE_NS'], 30_000_000_000) + + def test_readiness_is_current_only_and_cannot_be_claimed_from_old_evidence(self): + current = fixture(); CHECK["validate"](current) + historical = copy.deepcopy(current); historical["schema_version"] = 1; historical.pop("readiness") + CHECK["validate"](historical, historical=True) + with self.assertRaises(ValueError): CHECK["validate"](historical) + with self.assertRaises(ValueError): CHECK["validate"](current, historical=True) + mutations = [(('readiness', 'success'), False), (('readiness', 'events'), dict(client=[], broker=[])), + (('readiness', 'attempts'), current['readiness']['attempts']*3), + (('readiness', 'completed_unix_ms'), current['output']['event_baseline_unix_ms']), + (('readiness', 'checks', 'cold_cache'), False), (('readiness', 'checks', 'children_reaped'), False), + (('readiness', 'route_context_id'), '0'*32), (('readiness', 'manifest_id'), '0'*64), + (('readiness', 'attempts', 0, 'response', 'providers', 0, 'peer_id'), 'wrong-provider'), + (('readiness', 'attempts', 0, 'response', 'providers', 0, 'offer_expires_unix_seconds'), 0)] + for keys, changed in mutations: + with self.subTest(keys=keys): + value = copy.deepcopy(current); target = value + for key in keys[:-1]: target = target[key] + target[keys[-1]] = changed + with self.assertRaises(ValueError): CHECK['validate'](value) + + def test_actual_preparation_is_bounded_separate_cold_and_retains_failed_rounds(self): + for case in ('ready', 'empty-then-ready', 'both-empty', 'timeout', 'second-error', 'wrong-broker', + 'wrong-provider', 'early-b', 'changed-route', 'warm-cache'): + with self.subTest(case=case), tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); value = fixture(); clock = [1_000_000_000] + value['selected_before']['exact_selected_exit'] = value['expected_peers']['exit'] + binary = work/'volparossa'; binary.touch() + executable = work/'content-acceptance-fixture'; executable.touch() + native = work/'state-client/content-late-recovery'; native.mkdir(parents=True) + for name, item in (("publication", value["publication"]), ("layout", value["layout"]), + ("selection", value["selected_before"]), ("peers", value["expected_peers"])): + CHECK['write'](CHECK['path'](work, name), item) + calls, responses, deadlines = [], [], [] + def request(_work, tool, args, deadline): + self.assertEqual(tool, executable); self.assertEqual(len(args), 3) + self.assertEqual(args[-1], value['publication']['publisher_hex']) + self.assertNotIn(value['expected_peers']['relay4'], args) + calls.append('discover'); deadlines.append(deadline); clock[0] += 1_000_000_000 + if case == 'timeout': + clock[0] = deadline; raise subprocess.TimeoutExpired(['private-socket'], 30) + if case == 'second-error' and len(calls) == 2: raise CHECK['FixtureCommandError'](1) + reply = copy.deepcopy(value['readiness']['attempts'][0]['response']) + if case in ('both-empty', 'empty-then-ready', 'second-error') and (len(calls) == 1 or case == 'both-empty'): + reply['providers'] = [] + if case == 'wrong-broker': reply['control_relay_peer_id'] = 'wrong' + if case == 'wrong-provider': reply['providers'][0]['peer_id'] = 'wrong' + responses.append(reply) + return reply + def command(args, deadline, _input=None): + deadlines.append(deadline); clock[0] += 1_000_000 + self.assertLess(clock[0], deadline) + if args[-1] == 'paths': + context = '0'*32 if case == 'changed-route' else value['selected_before']['route_context_id'] + return ''.join(f"context={context} path={p['path_id']} relay={p['relay_peer_id']} " + f"exit={value['selected_before']['exact_selected_exit']} state=1 rtt_us=1 bytes=0 acked_transport_bytes=0\n" + for p in value['selected_before']['paths']).encode() + if args[-2:] == ['content', 'status']: + if 'runtime-relay5' in args[2]: return json.dumps(dict(serving=case == 'early-b', publications=0)).encode() + return json.dumps(dict(control_relay_peer_id=value['layout']['control_relay_peer_id'])).encode() + self.assertEqual(args[-3:], ['logs', '--limit', '400']) + codes = [] + for reply in responses: + if 'runtime-client' in args[2]: + codes.extend([CHECK['EVENTS'][1] if reply['providers'] else CHECK['EVENTS'][2], CHECK['EVENTS'][0]]) + else: + codes.append(CHECK['EVENTS'][3]) + if reply['providers']: codes.append(CHECK['EVENTS'][4]) + return ''.join(f"{2001+i} INFO event={code}\n" for i, code in enumerate(codes)).encode() + if case == 'warm-cache': (native/'cache').mkdir() + with patch.dict(CHECK['prepare'].__globals__, guest=lambda work: None, readiness_request=request, command=command), \ + patch.object(CHECK['time'], 'monotonic_ns', side_effect=lambda: clock[0]), \ + patch.object(CHECK['time'], 'time_ns', side_effect=lambda: 1_000_000_000+clock[0]), \ + patch.object(CHECK['time'], 'sleep', side_effect=lambda seconds: clock.__setitem__(0, clock[0]+int(seconds*1e9))): + if case in ('ready', 'empty-then-ready'): CHECK['prepare'](work, binary, executable) + else: + with self.assertRaises((ValueError, subprocess.TimeoutExpired)): CHECK['prepare'](work, binary, executable) + record = CHECK['read'](CHECK['path'](work, 'readiness')) + CHECK['validate_readiness'](record, value['publication'], value['layout'], value['selected_before'], + value['expected_peers'], successful=False) + self.assertLessEqual(len(calls), 2); self.assertEqual(set(deadlines), {31_000_000_000}) + self.assertEqual(record['success'], case in ('ready', 'empty-then-ready')) + self.assertNotIn('private-socket', json.dumps(record)) + self.assertFalse(CHECK['path'](work, 'gate', 'state').exists()) + if case == 'empty-then-ready': + self.assertEqual(record['attempts'][0]['response']['providers'], []) + self.assertEqual(CHECK['count'](record['events']['client'], CHECK['EVENTS'][2]), 1) + if case == 'second-error': + self.assertEqual(len(record['attempts']), 2) + self.assertEqual(record['attempts'][1]['failure']['category'], 'command-failed') + self.assertEqual(CHECK['count'](record['events']['client'], CHECK['EVENTS'][2]), 1) + + def test_readiness_timeout_kills_reaps_and_closes_exact_child_output(self): + with tempfile.TemporaryDirectory() as temporary: + work = Path(temporary); calls = []; clock = [1_000_000_000] + class Process: + pid = 1234 + returncode = None + stdout = io.BytesIO() + def communicate(self, timeout): + self.timeout = timeout; raise subprocess.TimeoutExpired(['private'], timeout) + def poll(self): return self.returncode + def kill(self): calls.append('kill') + def wait(self, timeout): calls.append('reap'); self.returncode = -9 + process = Process() + def identity(pid): + clock[0] += 10_000_000_000 + return dict(pid=pid, start_ticks=2) + with patch.object(CHECK['subprocess'], 'Popen', return_value=process), \ + patch.object(CHECK['time'], 'monotonic_ns', side_effect=lambda: clock[0]), \ + patch.dict(CHECK['readiness_request'].__globals__, process_identity=identity): + with self.assertRaises(subprocess.TimeoutExpired): CHECK['readiness_request'](work, Path('/inert'), [], + CHECK['time'].monotonic_ns()+30_000_000_000) + self.assertEqual(calls, ['kill', 'reap']); self.assertTrue(process.stdout.closed) + self.assertEqual(process.timeout, 20, 'setup time cannot extend the original deadline') + def test_exact_late_phase_is_independent_of_payload_overlap(self): for broker in ("relay0", "relay1", "relay2"): value = fixture(broker) @@ -228,15 +366,15 @@ def test_log_snapshots_are_bounded_and_ignore_preexisting_or_noncontent_events(s def test_cleanup_releases_gate_before_process_ownership_check_and_attempts_both(self): events = [] - def process(_work): events.append("process"); raise ValueError("replaced") + def process(_work, suffix="fetch"): events.append(suffix); raise ValueError("replaced") with patch.dict(CHECK["cleanup"].__globals__, stop_fetch=process, remove_gate=lambda work: events.append("gate")): with self.assertRaises(ValueError): CHECK["cleanup"](Path("/not-an-actual-guest")) - self.assertEqual(events, ["gate", "process"]) + self.assertEqual(events, ["gate", "fetch", "readiness-process"]) events.clear() def gate(_work): events.append("gate"); raise ValueError("gate failed") - with patch.dict(CHECK["cleanup"].__globals__, stop_fetch=lambda work: events.append("process"), remove_gate=gate): + with patch.dict(CHECK["cleanup"].__globals__, stop_fetch=lambda work, suffix="fetch": events.append(suffix), remove_gate=gate): with self.assertRaises(ValueError): CHECK["cleanup"](Path("/not-an-actual-guest")) - self.assertEqual(events, ["gate", "process"]) + self.assertEqual(events, ["gate", "fetch", "readiness-process"]) with tempfile.TemporaryDirectory() as temporary: CHECK["cleanup"](Path(temporary)) def test_coordinator_activates_b_only_after_snapshot_and_always_releases_owned_gate(self): @@ -302,6 +440,7 @@ def release(_work, **_kwargs): self.assertEqual(record["command_exit_code"], 2 if failure == "gate-install" else None) self.assertNotIn("secret", diagnostic.read_text()) with patch.dict(CHECK["run"].__globals__, guest=lambda work: None, + checked_readiness=lambda *args: value["readiness"], namespace=lambda work: dict(name="va-00000000-r4", identity="1:2"), command=execute, process_identity=lambda pid: dict(pid=pid, start_ticks=100), remove_gate=release), \ @@ -395,6 +534,7 @@ def release(_work, **_kwargs): self.assertLessEqual(clock[0], gate_started[0]+CHECK["GATE_NS"]+20_000_000) if case != "discovery-5.09s": self.assertTrue(CHECK["path"](work, "failure").exists()) with patch.dict(CHECK["run"].__globals__, guest=lambda work: None, + checked_readiness=lambda *args: value["readiness"], namespace=lambda work: dict(name="va-00000000-r4", identity="1:2"), process_identity=lambda pid: dict(pid=pid, start_ticks=100), remove_gate=release), \ patch.object(CHECK["subprocess"], "run", side_effect=execute), \ diff --git a/tests/integration/test-content-provider-smoke.py b/tests/integration/test-content-provider-smoke.py index cbf310944..11892df37 100644 --- a/tests/integration/test-content-provider-smoke.py +++ b/tests/integration/test-content-provider-smoke.py @@ -213,7 +213,7 @@ def test_late_phase_hooks_preflight_and_cleanup_cannot_be_skipped(self): self.assertIn("content_provider_adaptive_run\n content_provider_late_recovery_run\n" " content_provider_cancellation_run", runtime) self.assertIn("$evidence.late_provider_recovery.success == true", runtime) - self.assertIn("{schema_version:2,report_kind:", runtime) + self.assertIn("{schema_version:3,report_kind:", runtime) preflight = source.split('for provider_fixture in ', 1)[1].split('; do', 1)[0] for name in ("content-provider-late-recovery.sh", "content-provider-late-recovery.py"): self.assertIn(name, preflight) @@ -257,7 +257,7 @@ def test_late_emergency_metadata_survives_bounds_without_claiming_success(self): api = {"__name__": "inert_late_collector"} exec(compile(embedded, "late-collector", "exec"), api) self.assertEqual(api["LATE_PROVIDER_NAMES"], {f"content-provider-late-recovery-{suffix}.json" - for suffix in ("first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure")}) + for suffix in ("first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure", "readiness")}) for scenario, oversized_failure in (("content-provider", False), ("content-provider", True), ("content", False)): with tempfile.TemporaryDirectory() as directory: base = Path(directory) @@ -590,7 +590,7 @@ def test_current_control_relay_is_excluded_without_changing_roles(self): self.assertEqual(fixture("relay4")["layout"]["provider_nodes"], ["relay5", "relay3"]) def test_exact_scoped_report(self): - report = dict(schema_version=2, report_kind="volparossa-native-content-providers", source_revision="a" * 40, + report = dict(schema_version=3, report_kind="volparossa-native-content-providers", source_revision="a" * 40, explicit_origin_authenticated_https=True, normal_user_publication=True, native_name_retrieval=True, native_static_site=True, public_filter_snapshot=True, adaptive_provider_workers=True, late_provider_recovery=True, requester_cancellation=True, @@ -634,6 +634,13 @@ def test_exact_scoped_report(self): CHECK["validate_report"](historical, "a" * 40) with self.assertRaises(ValueError): CHECK["validate_report"](report, "a" * 40, historical=True) + older_late = copy.deepcopy(report) + older_late["schema_version"] = 2 + older_late["transfer"]["late_provider_recovery"]["schema_version"] = 1 + older_late["transfer"]["late_provider_recovery"].pop("readiness") + CHECK["validate_report"](older_late, "a" * 40, historical=True) + with self.assertRaises(ValueError): CHECK["validate_report"](older_late, "a" * 40) + with self.assertRaises(ValueError): CHECK["validate_report"]({**report, "schema_version": 2}, "a" * 40, historical=True) with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "historical.json" path.write_text(json.dumps(historical), encoding="ascii") From 9b132f6ca0dab6c9c7b61403d97a4723cd8e74cb Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 05:38:52 +0200 Subject: [PATCH 43/49] test: pin readiness dependency and retain current UI evidence --- crates/volparossa-content/Cargo.toml | 2 +- docs/IMPLEMENTATION_STATUS.md | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/volparossa-content/Cargo.toml b/crates/volparossa-content/Cargo.toml index dc470f2ce..b2166cd1b 100644 --- a/crates/volparossa-content/Cargo.toml +++ b/crates/volparossa-content/Cargo.toml @@ -36,7 +36,7 @@ zeroize.workspace = true libp2p-identity.workspace = true rand_core.workspace = true rcgen.workspace = true -volparossa-local-control = { path = "../volparossa-local-control" } +volparossa-local-control = { path = "../volparossa-local-control", version = "=0.1.0" } [lints] workspace = true diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index f4d2f4047..967beca1c 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -31,7 +31,7 @@ The immediate milestones build on the retained candidates and evidence below: | --- | --- | --- | | Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38195177153` on `c63ddf36` passes native and HTTPS three-provider reconstruction, but its first late-provider query returns empty through a newly selected broker. A separate signed-service readiness prerequisite is now implemented and component-tested, without provider hints or longer product deadlines. Its live execution, cancellation and protected pressure-retirement recovery remain unproved. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost; a separate exact-source probe encounters ECMWF HTTP 429. Closed retention now includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests, retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR11 are merged with passing CI. Browser08 and its immutable input bundle retain their synthetic UI proof. The exact table verifier is corrected and passes a separate actual-browser trial with synthetic products, before and after pointer refresh. Rebuild a new immutable bundle with that verifier; do not change the old bundle. Joined core delivery and native Android/iOS execution remain unverified. | +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR12 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | Respect the observed source rate limit instead of repeatedly relaunching the joined trial or routing requests through other peers. Finish the standalone @@ -89,6 +89,25 @@ the corrected UI verifier, not core, origin, cold or native-mobile delivery. Old reports and the frozen bundle are unchanged; a future joined trial needs a new bundle with the corrected driver. +The table correction is merged in Weather +[PR12](https://github.com/VOLPAROSSA/volparossa-weather/pull/12), with passing +component and CodeQL checks. A fresh full synthetic browser trial also passes +on the current shared harness: ten requests per product, nine pointer refreshes, +independent failures/expiry, detail navigation and complete cleanup. Its report +SHA-256 is `48be438ea5fc932314648a6fd81b6b104b4b7b84734e8efaebccf6c20383f10b`. +This full trial and the exact-table trial remain separate evidence. + +A new immutable version-three bundle is retained at +`.git/ci-evidence/weather-overmorrow-inputs-v3-tables.JRK0OK/inputs.tar`: +176 files, 455,025,966 payload bytes, archive SHA-256 +`2b896fde7cc6575dc809438c6fa0bf8f0f97d6362337f31d761156fedfbc3039`, manifest +`0a9f81a54eb594af9884aebcdf79c03d8efb767447973f7ad40acb92844bf645`. +Capture, runtime verification, extraction and tree verification pass without +executing the bundled runtime or contacting weather sources. Only the full +browser proof, build record, shared harness and corrected table driver differ +from the previous bundle. The old archive hash is unchanged before and after +capture. This prepares the joined trial; it does not prove its delivery path. + Independent reconstruction of the same run validates native and cooperative HTTPS delivery of 15,728,640 bytes in sixty chunks from three providers. Their triple bulk overlaps are 4.023 and 4.489 seconds. HTTPS uses one authenticated From 726fe694c08c7c593bbef27f544ca962f00e01e8 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 05:53:17 +0200 Subject: [PATCH 44/49] docs: record successful protected cache sequence --- docs/IMPLEMENTATION_STATUS.md | 79 +++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 967beca1c..4cec8f900 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,23 +29,64 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Complete late-provider recovery and requester cancellation in one protected run, then join Weather delivery to the verified cache path. | Run `38195177153` on `c63ddf36` passes native and HTTPS three-provider reconstruction, but its first late-provider query returns empty through a newly selected broker. A separate signed-service readiness prerequisite is now implemented and component-tested, without provider hints or longer product deadlines. Its live execution, cancellation and protected pressure-retirement recovery remain unproved. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Join Weather delivery to the verified cache path, then exercise recovery under actual device pressure. | Run `38199501353` on `0e23140b` passes three-provider native/HTTPS reconstruction, signed-service readiness, late-provider recovery, requester cancellation and cleanup together. Its worker budget remains unchanged throughout; pressure retirement and continuation still require a separate protected trial. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost; a separate exact-source probe encounters ECMWF HTTP 429. Closed retention now includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests, retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | -| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR12 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | - -Respect the observed source rate limit instead of repeatedly relaunching the -joined trial or routing requests through other peers. Finish the standalone -provider proof, which needs no weather downloads, through late discovery and -cancellation. The latest run proves concurrent reconstruction, while run -`38190664934` proves late recovery. Neither exercises the pressure-retirement -continuation itself. +| Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR13 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | + +The standalone provider proof now passes without weather downloads. The next +joined trial uses the corrected Overmorrow bundle and bounded source acquisition; +respect the observed source rate limit instead of automatically retrying or +routing requests through other peers. Pressure retirement and continuation are +still separate evidence gaps, not implied by successful parallel delivery. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. -The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38195177153) +### Verified cache delivery and remaining integration + +The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38199501353) +on `0e23140b` **passes** the complete standalone provider sequence. Independent +validation of the retained schema-three report and reconstructed component +evidence confirms: + +- The signed-service readiness prerequisite completes in 3.015 seconds, + returning A's verified offer. No provider hints are sent; B stays off and the content cache + stays cold. The measured late-provider phase starts after this preparation. +- The first A-only snapshot arrives 5.163 seconds after gate start; B activation + and gate removal finish by 5.246 seconds, within the original six-second gate. The fetch reconstructs + all nine chunks, 2,097,275 bytes, from two useful providers in 12.976 seconds, + without an origin fallback. +- Native and cooperative HTTPS delivery each reconstruct sixty disjoint chunks, + 15,728,640 bytes, from R3, R4 and R5. Actual three-provider payload overlaps are + 4.913 and 4.479 seconds. HTTPS uses one authenticated HEAD and no origin body + or range requests. +- Requester cancellation passes after the original thirty-second cooldown. + Independent cache-only delivery remains available, control EOF is observed, + and the original agent and owned helper terminate cleanly. +- Privacy captures and cleanup pass: two distinct relay paths, no direct + client-to-exit traffic and zero remaining owned objects. The disposable + guest-root network state is identical before and after; this is guest + evidence, not a separate development-host firewall measurement. + +Both healthy parallel phases retain a worker allowance of twelve and peak at +three leases. No pressure flags or continuation events occur. This run does +**not** prove pressure retirement, recovery under load, Weather acquisition, +browser delivery or general performance improvements. + +Evidence is retained at +`.git/ci-evidence/content-provider-hosted-readiness.5siQDi/`. The aggregate report +has SHA-256 `86466078298fcab6b6ae951fbd369b8ce0b28a2bda566cf238aa69a0deebd4a6`; +the independently rebuilt adaptive report has SHA-256 +`fc691fe02e4f7b3bde36797a23cab01431b26fa9d22a41aabacf80eba0772841`. +Commit `9b132f6c` subsequently pins the local test dependency to `=0.1.0` and +updates documentation; runtime sources, validators and `Cargo.lock` are +unchanged. The protected result remains attributed to its exact tested commit. + +### Readiness correction and client evidence + +The earlier [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38195177153) on `c63ddf36` is **failed overall**, at the late-provider `first-snapshot` stage. This time the broker changes from relay1 to relay2. Its first query returns empty after 1.151 seconds, with two dial failures and no verified offer. @@ -71,8 +112,8 @@ preparation/setup time consumes the original deadline. Two Rust socket tests, seventeen late-provider tests, nineteen aggregate checks and ten existing inert cancellation checks pass, with an independent review. Current aggregate schema 3 requires late-component schema 2 and its readiness evidence; historical schemas -remain explicitly separate. Live readiness and the complete sequence are still -unproved. No production address fix is claimed. +remain explicitly separate. The successful live trial above now verifies this +sequence. No production address fix is claimed. The separate Overmorrow table trial now passes in the actual isolated Firefox CanvasKit UI using synthetic products. Earlier extraction assigned both tables @@ -86,8 +127,8 @@ The retained report at Weather's `build/overmorrow-table-browser-04/report.json` has SHA-256 `3ed8ceeaf89aa22bb2c9fd656c9b4228dd5f75b12e87424820bd430dc9dc214d`. Five pure DOM regressions accompany the existing exact-value checks. This proves the corrected UI verifier, not core, origin, cold or native-mobile delivery. -Old reports and the frozen bundle are unchanged; a future joined trial needs a -new bundle with the corrected driver. +Old reports and the previous frozen bundle are unchanged; the new bundle below +contains the corrected driver. The table correction is merged in Weather [PR12](https://github.com/VOLPAROSSA/volparossa-weather/pull/12), with passing @@ -95,7 +136,10 @@ component and CodeQL checks. A fresh full synthetic browser trial also passes on the current shared harness: ten requests per product, nine pointer refreshes, independent failures/expiry, detail navigation and complete cleanup. Its report SHA-256 is `48be438ea5fc932314648a6fd81b6b104b4b7b84734e8efaebccf6c20383f10b`. -This full trial and the exact-table trial remain separate evidence. +This full trial and the exact-table trial remain separate evidence. Its updated +build record is merged in Weather +[PR13](https://github.com/VOLPAROSSA/volparossa-weather/pull/13), with passing +component and CodeQL checks. A new immutable version-three bundle is retained at `.git/ci-evidence/weather-overmorrow-inputs-v3-tables.JRK0OK/inputs.tar`: @@ -108,7 +152,7 @@ browser proof, build record, shared harness and corrected table driver differ from the previous bundle. The old archive hash is unchanged before and after capture. This prepares the joined trial; it does not prove its delivery path. -Independent reconstruction of the same run validates native and cooperative +Independent reconstruction of the earlier `c63ddf36` run validates native and cooperative HTTPS delivery of 15,728,640 bytes in sixty chunks from three providers. Their triple bulk overlaps are 4.023 and 4.489 seconds. HTTPS uses one authenticated HEAD and no origin body or range requests. Both worker allowances stay at eleven @@ -150,7 +194,8 @@ are unchanged. Thirteen late-provider tests pass, including a reproduction of the 5.09-second response and expired discovery, activation and release cases. Nineteen aggregate checks and ten inert cancellation checks also pass. The subsequent `c63ddf36` run above exercises this correction but exposes the -different first-empty-query condition; the complete sequence remains unproved. +different first-empty-query condition. The successful `0e23140b` run above +supersedes this earlier failure without changing the measured gate or deadline. Independent reconstruction again validates native and cooperative HTTPS: each delivers the exact 15,728,640-byte object in sixty chunks from three From 9441b4514ae0c66c6353158fdeb45579773c32fa Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 06:29:42 +0200 Subject: [PATCH 45/49] fix: preserve MPTCP routes after definite socket rejection --- crates/volparossa-agent/src/helper_v3.rs | 100 ++++++++++++++ .../volparossa-agent/src/mptcp_transport.rs | 2 +- .../src/mptcp_transport/retirement.rs | 130 +++++++++++++++++- docs/IMPLEMENTATION_STATUS.md | 60 +++++++- 4 files changed, 283 insertions(+), 9 deletions(-) diff --git a/crates/volparossa-agent/src/helper_v3.rs b/crates/volparossa-agent/src/helper_v3.rs index bb315f948..d2fc87c47 100644 --- a/crates/volparossa-agent/src/helper_v3.rs +++ b/crates/volparossa-agent/src/helper_v3.rs @@ -3753,6 +3753,106 @@ mod tests { )); } + async fn acquire_transport_rejection( + result: HelperResult, + alter: fn(&mut HelperResponse), + ) -> HelperClientError { + let directory = tempfile::tempdir().expect("tempdir"); + let socket = directory.path().join("helper.sock"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + let request = read_request(&mut stream).await.expect("request"); + assert!(matches!( + request.operation, + Some(helper_request::Operation::AcquireTransportSocket(_)) + )); + let mut response = HelperResponse { + protocol_version: HELPER_PROTOCOL_VERSION, + request_id: request.request_id.clone(), + result: result as i32, + diagnostic_code: "TRANSPORT_SOCKET_UNAVAILABLE".to_owned(), + operation_digest: operation_digest(&request).expect("digest").to_vec(), + outcome: None, + }; + alter(&mut response); + let frame = if response.outcome.is_some() { + // Deliberately bypass only the test server's encoder: production must reject + // a non-Ok response carrying success metadata before treating it as definitive. + assert!(encode_response(&response).is_err()); + let body = volparossa_protocol::encode_canonical( + &response, + volparossa_routing::MAX_HELPER_FRAME, + ) + .expect("bounded protobuf body"); + let length = u32::try_from(body.len()).expect("bounded test frame"); + let mut frame = length.to_be_bytes().to_vec(); + frame.extend_from_slice(&body); + frame + } else { + encode_response(&response).expect("response") + }; + stream.write_all(&frame).await.expect("write response"); + stream.flush().await.expect("flush response"); + }); + let response = + HelperClient::new_for_test(socket, directory.path().join("unused"), geteuid().as_raw()) + .acquire_transport_socket(transport_request(TransportSocketKind::MptcpConnected)) + .await; + server.await.expect("server"); + match response { + Err(error) => error, + Ok(_) => panic!("rejected Acquire must not return a descriptor"), + } + } + + #[tokio::test] + async fn acquire_transport_exact_rejection_preserves_safe_helper_result() { + for result in [ + HelperResult::Kernel, + HelperResult::Capacity, + HelperResult::Unavailable, + ] { + assert!(matches!( + acquire_transport_rejection(result, |_| {}).await, + HelperClientError::Rejected(actual) if actual == result + )); + } + } + + #[tokio::test] + async fn acquire_transport_uncorrelated_rejection_is_not_definitive() { + for alter in [ + (|response: &mut HelperResponse| response.request_id[0] ^= 1) + as fn(&mut HelperResponse), + |response: &mut HelperResponse| response.operation_digest[0] ^= 1, + ] { + assert!(matches!( + acquire_transport_rejection(HelperResult::Kernel, alter).await, + HelperClientError::Correlation + )); + } + } + + #[tokio::test] + async fn acquire_transport_rejection_with_success_outcome_is_not_definitive() { + let error = acquire_transport_rejection(HelperResult::Kernel, |response| { + let request = transport_request(TransportSocketKind::MptcpConnected); + response.outcome = Some(helper_response::Outcome::TransportSocketReady( + TransportSocketReady { + path_id: request.path_id, + role: request.role, + descriptor_kind: request.descriptor_kind, + local: request.expected_local, + remote: request.expected_remote, + mptcp_flow_handle: vec![9; 32], + }, + )); + }) + .await; + assert!(matches!(error, HelperClientError::Protocol(_))); + } + #[test] fn token_reader_rejects_links_and_wrong_modes() { let directory = tempfile::tempdir().expect("tempdir"); diff --git a/crates/volparossa-agent/src/mptcp_transport.rs b/crates/volparossa-agent/src/mptcp_transport.rs index 7518c5761..075cd5df0 100644 --- a/crates/volparossa-agent/src/mptcp_transport.rs +++ b/crates/volparossa-agent/src/mptcp_transport.rs @@ -398,7 +398,7 @@ impl ClientMptcpTransport { let acquired = helper .acquire_transport_socket(request) .await - .map_err(|_| MptcpTransportError::AcquireUnconfirmed)?; + .map_err(|error| slot.acquire_failed(error))?; let flow = acquired.metadata().mptcp_flow_handle.clone(); let retirement = slot.bind( volparossa_routing::RetireMptcpFlow { diff --git a/crates/volparossa-agent/src/mptcp_transport/retirement.rs b/crates/volparossa-agent/src/mptcp_transport/retirement.rs index 455dcfae0..260ab7c79 100644 --- a/crates/volparossa-agent/src/mptcp_transport/retirement.rs +++ b/crates/volparossa-agent/src/mptcp_transport/retirement.rs @@ -139,6 +139,25 @@ impl Reservation { self.acquire_started = true; } + /// Preserve the shared route only when the authenticated, correlated helper response proves + /// that this request issued no descriptor or flow capability. The helper's definite kernel + /// rejection and pre-issuance capacity/unavailable results leave existing flows untouched. + /// All lost, malformed, cancelled or cleanup-ambiguous handoffs still poison this reservation; + /// clearing its flag must never clear another reservation's unconfirmed acquisition. + pub(super) fn acquire_failed(&mut self, error: HelperClientError) -> MptcpTransportError { + if matches!( + error, + HelperClientError::Rejected( + HelperResult::Kernel | HelperResult::Capacity | HelperResult::Unavailable + ) + ) { + self.acquire_started = false; + MptcpTransportError::Helper(error) + } else { + MptcpTransportError::AcquireUnconfirmed + } + } + pub(super) fn bind(mut self, request: RetireMptcpFlow, descriptor: OwnedFd) -> Guard { let mut request_id = [0; 16]; OsRng.fill_bytes(&mut request_id); @@ -192,7 +211,11 @@ impl Drop for Guard { #[cfg(test)] mod tests { use super::*; - use std::{io::Read as _, os::unix::net::UnixStream, time::Duration}; + use std::{ + io::{Read as _, Write as _}, + os::unix::net::UnixStream, + time::Duration, + }; fn pair(owner: &Retirements) -> (Guard, UnixStream) { let (socket, peer) = UnixStream::pair().unwrap(); @@ -223,6 +246,111 @@ mod tests { )); } + #[tokio::test] + async fn definite_acquire_rejection_preserves_live_sibling_and_retry_capacity() { + for result in [ + HelperResult::Kernel, + HelperResult::Capacity, + HelperResult::Unavailable, + ] { + let owner = Retirements::new(); + let (guard, mut peer) = pair(&owner); + let mut sibling = std::fs::File::from( + guard + .pending + .as_ref() + .unwrap() + .descriptor + .try_clone() + .unwrap(), + ); + let mut slot = owner.reserve().unwrap(); + slot.begin_acquire(); + assert!(matches!( + slot.acquire_failed(HelperClientError::Rejected(result)), + MptcpTransportError::Helper(HelperClientError::Rejected(actual)) if actual == result + )); + drop(slot); + assert_eq!(owner.0.slots.available_permits(), MAX_FLOWS - 1); + owner + .drain_with(|_, _, _| async { panic!("live sibling must not be retired") }) + .await + .unwrap(); + // The definite rejection permits another attempt without closing a sibling. These + // real Unix-stream bytes test ownership only, not the protected MPTCP datapath. + drop(owner.reserve().unwrap()); + sibling.write_all(b"still live").unwrap(); + let mut received = [0; 10]; + peer.read_exact(&mut received).unwrap(); + assert_eq!(&received, b"still live"); + drop(sibling); + drop(guard); + assert_eq!(peer.read(&mut [0]).unwrap(), 0); + assert_eq!(owner.0.pending.lock().unwrap().len(), 1); + assert_eq!(owner.0.slots.available_permits(), MAX_FLOWS - 1); + owner.drain_with(|_, _, _| async { Ok(()) }).await.unwrap(); + assert_eq!(owner.0.slots.available_permits(), MAX_FLOWS); + } + } + + #[tokio::test] + async fn uncertain_or_nonallowlisted_acquire_failure_still_requires_route_retirement() { + let errors = [ + HelperClientError::Timeout, + HelperClientError::Io(io::Error::other("lost response")), + HelperClientError::Correlation, + HelperClientError::DescriptorHandoff(io::Error::other("lost descriptor")), + HelperClientError::DescriptorValidation(io::Error::other("invalid descriptor")), + HelperClientError::UntrustedServer, + HelperClientError::RuntimeChanged, + HelperClientError::Rejected(HelperResult::NotFound), + HelperClientError::Rejected(HelperResult::CleanupIncomplete), + HelperClientError::Rejected(HelperResult::InvalidRequest), + HelperClientError::Rejected(HelperResult::UnauthorisedPeer), + HelperClientError::Rejected(HelperResult::AlreadyExists), + HelperClientError::Rejected(HelperResult::Expired), + ]; + for error in errors { + let owner = Retirements::new(); + let mut slot = owner.reserve().unwrap(); + slot.begin_acquire(); + assert!(matches!( + slot.acquire_failed(error), + MptcpTransportError::AcquireUnconfirmed + )); + drop(slot); + assert!(owner.reserve().is_err()); + assert!(matches!( + owner + .drain_with(|_, _, _| async { panic!("unknown capability cannot be retired") }) + .await, + Err(MptcpTransportError::AcquireUnconfirmed) + )); + } + } + + #[tokio::test] + async fn definite_rejection_never_clears_another_unconfirmed_acquisition() { + let owner = Retirements::new(); + let mut unknown = owner.reserve().unwrap(); + let mut rejected = owner.reserve().unwrap(); + unknown.begin_acquire(); + rejected.begin_acquire(); + drop(unknown); + assert!(matches!( + rejected.acquire_failed(HelperClientError::Rejected(HelperResult::Kernel)), + MptcpTransportError::Helper(HelperClientError::Rejected(HelperResult::Kernel)) + )); + drop(rejected); + assert!(owner.reserve().is_err()); + assert!(matches!( + owner + .drain_with(|_, _, _| async { panic!("unknown acquisition remains poisoned") }) + .await, + Err(MptcpTransportError::AcquireUnconfirmed) + )); + } + #[tokio::test] async fn completed_flows_reuse_bounded_slots_but_live_flows_do_not() { let owner = Retirements::new(); diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 4cec8f900..6e287e6cd 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,21 +29,63 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Join Weather delivery to the verified cache path, then exercise recovery under actual device pressure. | Run `38199501353` on `0e23140b` passes three-provider native/HTTPS reconstruction, signed-service readiness, late-provider recovery, requester cancellation and cleanup together. Its worker budget remains unchanged throughout; pressure retirement and continuation still require a separate protected trial. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run passes corrected archive-owner dispatch but fails during acquisition. Its specific failure diagnostic was lost; a separate exact-source probe encounters ECMWF HTTP 429. Closed retention now includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests, retries or relaxed validation. Respect the source limit before another joined trial. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover this delivery. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Preserve existing flows when a new socket is definitively rejected, then resume joined Weather delivery. | Run `38199501353` on `0e23140b` passes the standalone sequence. The later joined run on `9b132f6c` stops at named-cache fetch: one definite kernel rejection incorrectly destroys the shared route. The original socket errno remains unknown. Pressure retirement and continuation still require a separate protected trial. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run stops before Weather acquisition, so it makes no new weather-source requests. An earlier exact-source probe encountered ECMWF HTTP 429. Closed retention includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests or automatic retries. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover joined delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR13 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | -The standalone provider proof now passes without weather downloads. The next -joined trial uses the corrected Overmorrow bundle and bounded source acquisition; -respect the observed source rate limit instead of automatically retrying or -routing requests through other peers. Pressure retirement and continuation are -still separate evidence gaps, not implied by successful parallel delivery. +The standalone provider proof passes without weather downloads. The latest joined +trial uses the corrected Overmorrow bundle but exposes an earlier shared-route +failure. Correct that failure before another joined trial; do not automatically +retry source requests or route them through other peers to evade a source limit. +Pressure retirement and continuation remain separate evidence gaps, not implied +by successful parallel delivery. Their test helper is being prepared separately +and is not part of the failed run. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM trial still requires the unchanged 4 GiB guest plus 2 GiB host-memory reserve; do not run competing heavy builds or lower that reserve. +### Named fetch failure and route preservation + +The joined trial on `9b132f6c` fails before Weather acquisition at +`content-provider-named-fetch`, with `CONTENT_NAME_FETCH_FAILED` and +`CONTENT_UNAVAILABLE`. Both providers register successfully and named metadata +passes verification. During the subsequent body fetch, the first MPTCP socket is +ready at 04:02:53.433 UTC; the second receives a definitive helper `Kernel` +rejection at 04:02:53.499. The client starts destroying the shared route two +milliseconds later, followed by route-flow and TLS failures. No weather-source +request or client rendering is reached. + +Source inspection identifies the propagation defect: the client converts every +acquisition error to `AcquireUnconfirmed`, even a correlated rejection that +issued no descriptor or flow capability. Its reservation then also marks route +ownership uncertain on drop, forcing full context destruction and disrupting +the sibling flow. The underlying kernel syscall and errno were not retained; +route age, source-port collision or resource pressure must not be asserted as +its cause. + +The correction preserves only authenticated, correlated `Kernel`, `Capacity` +and `Unavailable` rejections. It releases the failed request's reserved slot +without clearing another request's uncertainty. The existing bounded acquisition +loop, selected route, deadlines and genuine MPTCP requirement remain unchanged. +Cancellation, malformed replies, lost descriptor handoffs and ambiguous cleanup +still require route retirement. Eleven targeted tests pass: seven retirement +tests, three helper-protocol tests and the route-loss classification test. +The tests exercise the production error conversion, real local sibling socket +ownership and malformed replies; they do not substitute for MPTCP dataplane +evidence. Formatting and independent review pass. The correction has not yet +passed a new protected network trial. + +Failure evidence is retained at +`.git/ci-evidence/weather-overmorrow-joined.f58CgX/`. The source archive hash is +`d877f84d62816886e301f29d460d58eb1ebd0a626e09993c8769153a3bad597c`; +the aggregate report hash is +`a2c9379b12f96623c95e527b57ffb4e6873337a0f35420be0e60ef2c1ca8c5e7`. +Cleanup completes with zero owned objects; the VM, private scratch directory and +forwarded test port are gone. Readable development-host network measurements are +unchanged; firewall access was unavailable before and after, not verified equal. + ### Verified cache delivery and remaining integration The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38199501353) @@ -83,6 +125,10 @@ the independently rebuilt adaptive report has SHA-256 Commit `9b132f6c` subsequently pins the local test dependency to `=0.1.0` and updates documentation; runtime sources, validators and `Cargo.lock` are unchanged. The protected result remains attributed to its exact tested commit. +Hosted [Quality checks for `9b132f6c`](https://github.com/VOLPAROSSA/volparossa/actions/runs/38200580494) +and the [documentation-only `726fe694`](https://github.com/VOLPAROSSA/volparossa/actions/runs/38202461085) +pass, as do both revisions' CodeQL checks. These results do not establish +verification of the later route-preservation correction. ### Readiness correction and client evidence From 4ceee267934d96be4d0246ab0a0c1752bc257c63 Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 06:25:10 +0200 Subject: [PATCH 46/49] test: prepare bounded guest pressure observer --- .../integration/content-provider-pressure.py | 584 ++++++++++++++++++ .../test-content-provider-pressure.py | 465 ++++++++++++++ 2 files changed, 1049 insertions(+) create mode 100644 tests/integration/content-provider-pressure.py create mode 100644 tests/integration/test-content-provider-pressure.py diff --git a/tests/integration/content-provider-pressure.py b/tests/integration/content-provider-pressure.py new file mode 100644 index 000000000..487aec670 --- /dev/null +++ b/tests/integration/content-provider-pressure.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Explicit disposable-guest load fixture; never a product budget or transfer proof. + +observe WORK starts before the ordinary native fetch. The ready receipt binds a +cold cache. Only three complete, content-address-verified public fixture chunks +can trigger two capless CPU workers. No cache is opened as a ChunkStore or written. +All production admission, transfer deadlines and continuation decisions stay real. +""" +import ctypes +import errno +import hashlib +import json +import os +from pathlib import Path +import pwd +import re +import resource +import runpy +import select +import signal +import socket +import stat +import subprocess +import sys +import time + +PSI = runpy.run_path(str(Path(__file__).with_name("content-provider-resource-ready.py"))) +PREFIX = "content-provider-pressure" +CHUNK_BYTES = 262144 +OBJECT_SHA = "8fd67e1fc14d95b27d5d9be573c6609baf59054a129a669a15a6cd4562b41ecb" +# adaptive.rs::seed publishes A..|, with chunk i stored exclusively by i % 3. +CHUNKS = tuple(hashlib.sha256(bytes([65+i]) * CHUNK_BYTES).hexdigest() for i in range(3)) +WAIT_NS = 30_000_000_000 +LOAD_NS = 10_000_000_000 +ACTIVE_NS = 9_000_000_000 # Final second is reserved for exact owned-child drain. +POLL_NS = 200_000_000 +MEMORY_FLOOR = 512 * 1024 * 1024 +ADDRESS_SPACE = 128 * 1024 * 1024 +MAX_REPORT = 4096 +SCOPE = dict(disposable_guest=True, cache_read_only=True, synthetic_budget=False, + agent_affinity_changed=False, continuation_claimed=False, + transfer_completion_claimed=False, owner_goodput_claimed=False) +STATUSES = {"observed", "trigger-timeout", "pressure-missing", "resource-unavailable", + "interrupted", "invalid", "load-failed", "cleanup-failed"} + + +def require(value): + if not value: + raise ValueError("invalid bounded pressure fixture") + + +def decode(raw): + def pairs(items): + result = {} + for key, value in items: + require(key not in result) + result[key] = value + return result + return json.loads(raw, object_pairs_hook=pairs) + + +def directory(name, owner, *, parent=None): + fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, + dir_fd=parent) + try: + info = os.fstat(fd) + require(stat.S_ISDIR(info.st_mode) and info.st_uid == owner and not info.st_mode & 0o022) + return fd + except BaseException: + os.close(fd) + raise + + +def file_bytes(parent, name, owner, maximum, *, exact=None): + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, + dir_fd=parent) + try: + before = os.fstat(fd) + require(stat.S_ISREG(before.st_mode) and before.st_uid == owner + and stat.S_IMODE(before.st_mode) == 0o600 and before.st_nlink == 1 + and 0 < before.st_size <= maximum + and (exact is None or before.st_size == exact)) + parts, count = [], 0 + while count <= maximum: + part = os.read(fd, maximum + 1 - count) + if not part: + break + parts.append(part) + count += len(part) + after = os.fstat(fd) + require(count == before.st_size and + (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) + == (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns)) + return b"".join(parts) + finally: + os.close(fd) + + +def absent(parent, name): + try: + os.stat(name, dir_fd=parent, follow_symlinks=False) + except FileNotFoundError: + return True + return False + + +def write_record(parent, name, value): + require(name in (PREFIX + "-ready.json", PREFIX + "-controller.json")) + raw = (json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode("ascii") + require(len(raw) <= MAX_REPORT) + temporary = "." + name + ".part" + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, dir_fd=parent) + try: + with os.fdopen(fd, "wb") as output: + output.write(raw) + publish(parent, temporary, name) + finally: + try: + os.unlink(temporary, dir_fd=parent) + except FileNotFoundError: + pass + + +def publish(parent, temporary, name): + # Linux guest fixture: atomic visibility, no overwrite and no temporary + # second hardlink that would violate readers' single-link safety check. + rename = ctypes.CDLL(None, use_errno=True).renameat2 + rename.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + rename.restype = ctypes.c_int + if rename(parent, os.fsencode(temporary), parent, os.fsencode(name), 1) != 0: + raise OSError(ctypes.get_errno(), "private fixture publication failed") + + +def publication(parent, owner, *, now=None): + value = decode(file_bytes(parent, "content-provider-adaptive-publication.json", owner, MAX_REPORT)) + require(value["report_kind"] == "volparossa-content-adaptive-provider-seed" + and value["object_sha256"] == OBJECT_SHA and type(value["bytes"]) is int + and value["bytes"] == 60 * CHUNK_BYTES and type(value["chunks"]) is int + and value["chunks"] == 60 and value["publisher_removed"] is True + and value["publisher_private_key_persisted"] is False + and value["recipient_encrypted"] is False) + now = int(time.time()) if now is None else now + created, expires = value["created_unix_seconds"], value["expires_unix_seconds"] + require(type(created) is int and type(expires) is int + and 0 < created <= now < expires and expires-created == 3600) + for label in "abc": + require(type(value[f"replica_{label}_chunks"]) is int and value[f"replica_{label}_chunks"] == 20 + and type(value[f"replica_{label}_bytes"]) is int + and value[f"replica_{label}_bytes"] == 20 * CHUNK_BYTES) + for key in ("manifest_id", "publisher_hex"): + require(type(value[key]) is str and re.fullmatch(r"[0-9a-f]{64}", value[key])) + return dict(manifest_id=value["manifest_id"], publisher_hex=value["publisher_hex"], object_sha256=OBJECT_SHA) + + +class CacheObserver: + def __init__(self, work, owner): + self.owner = owner + self.root = directory("state-client", owner, parent=work) + self.parent = None + self.cache = None + try: + self.parent = directory("content-adaptive-pressure", owner, parent=self.root) + require(stat.S_IMODE(os.fstat(self.parent).st_mode) == 0o700) + require(absent(self.parent, "cache")) + except BaseException: + self.close() + raise + + def sample(self): + # Reopening these fixed directories checks replacement, without taking the + # cache's exclusive mutation lock. No directory scans or index reads. + current = directory("content-adaptive-pressure", self.owner, parent=self.root) + try: + require((os.fstat(current).st_dev, os.fstat(current).st_ino) + == (os.fstat(self.parent).st_dev, os.fstat(self.parent).st_ino)) + finally: + os.close(current) + if absent(self.parent, "cache"): + require(self.cache is None) + return 0 + current = directory("cache", self.owner, parent=self.parent) + if self.cache is None: + self.cache = current + else: + try: + require((os.fstat(current).st_dev, os.fstat(current).st_ino) + == (os.fstat(self.cache).st_dev, os.fstat(self.cache).st_ino)) + finally: + os.close(current) + require(stat.S_IMODE(os.fstat(self.cache).st_mode) == 0o700) + found = 0 + for name in CHUNKS: + try: + raw = file_bytes(self.cache, name, self.owner, CHUNK_BYTES, exact=CHUNK_BYTES) + except FileNotFoundError: + continue + require(hashlib.sha256(raw).hexdigest() == name) + found += 1 + return found + + def close(self): + for name in ("cache", "parent", "root"): + fd = getattr(self, name, None) + if fd is not None: + os.close(fd) + setattr(self, name, None) + + +def guest(work): + require(os.geteuid() == 0 and socket.gethostname() == "volparossa-alpha" + and work.is_absolute() and work.parent == Path("/opt") + and re.fullmatch(r"va\.[0-9a-f]{32}\.[A-Za-z0-9]{6}", work.name) + and work.resolve() == work) + result = subprocess.run(["/usr/bin/systemd-detect-virt", "--vm"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=3, check=False) + require(result.returncode == 0 and result.stdout == b"kvm\n") + worker, agent = pwd.getpwnam("volparossa-worker"), pwd.getpwnam("volparossa") + require(worker.pw_uid > 0 and worker.pw_gid > 0 and agent.pw_uid > 0 + and worker.pw_uid != agent.pw_uid and len(os.sched_getaffinity(0)) >= 2) + return worker, agent.pw_uid + + +def memory_available(): + raw = PSI["read_bounded"](Path("/proc/meminfo")) + rows = [line.split() for line in raw.splitlines() if line.startswith("MemAvailable:")] + require(len(rows) == 1 and len(rows[0]) == 3 and rows[0][2] == "kB" + and rows[0][1].isascii() and rows[0][1].isdigit()) + return int(rows[0][1]) * 1024 >= MEMORY_FLOOR + + +def child(cpu, deadline, parent, uid, gid, acknowledgement, original_mask): + """Called only after the guarded parent forks; never exposed as a CLI action.""" + libc = ctypes.CDLL(None, use_errno=True) + def prctl(option, arg=0): + if libc.prctl(option, arg, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "worker sandbox") + prctl(1, signal.SIGKILL) # PR_SET_PDEATHSIG, before any credential transition. + require(os.getppid() == parent) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + os.setsid() + os.umask(0o077) + prctl(8, 0) # Never retain capabilities across setuid. + prctl(47, 4) # PR_CAP_AMBIENT_CLEAR_ALL. + for capability in range(64): + if libc.prctl(24, capability, 0, 0, 0) != 0 and ctypes.get_errno() != errno.EINVAL: + raise OSError(ctypes.get_errno(), "worker capability drop") + prctl(38, 1) # PR_SET_NO_NEW_PRIVS. + os.setgroups([]) + os.setresgid(gid, gid, gid) + os.setresuid(uid, uid, uid) + # Credential changes clear PDEATHSIG; restore it after dropping identity. + prctl(1, signal.SIGKILL) + require(os.getppid() == parent and os.getresuid() == (uid, uid, uid) + and os.getresgid() == (gid, gid, gid) and os.getgroups() == []) + resource.setrlimit(resource.RLIMIT_AS, (ADDRESS_SPACE, ADDRESS_SPACE)) + resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + resource.setrlimit(resource.RLIMIT_FSIZE, (0, 0)) + os.sched_setaffinity(0, {cpu}) + require(os.sched_getaffinity(0) == {cpu}) + status = Path("/proc/self/status").read_text(encoding="ascii") + for key in ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb"): + require(re.search(rf"^{key}:\s+0+$", status, re.MULTILINE) is not None) + require(re.search(r"^NoNewPrivs:\s+1$", status, re.MULTILINE) is not None) + # Preserve only the one-byte setup acknowledgement; discard all inherited + # fixture/cache descriptors and all I/O channels before burning CPU. + os.dup2(acknowledgement, 3) + maximum = resource.getrlimit(resource.RLIMIT_NOFILE)[0] + require(4 <= maximum <= 1_048_576) + os.closerange(4, maximum) + null = os.open("/dev/null", os.O_RDWR) + for fd in (0, 1, 2): + os.dup2(null, fd) + if null > 3: + os.close(null) + resource.setrlimit(resource.RLIMIT_NOFILE, (4, 4)) + require(time.monotonic_ns() < deadline) + os.write(3, b"1") + os.close(3) + value = 1 + while time.monotonic_ns() < deadline: + for _ in range(1024): + value = (value * 1664525 + 1013904223) & 0xffffffff + + +class Workers: + def __init__(self, worker): + self.worker = worker + self.children = {} + self.ready = 0 + self.forced = 0 + + def start(self, deadline): + cpu, parent = min(os.sched_getaffinity(0)), os.getpid() + for _ in range(2): + require(time.monotonic_ns() < deadline) + read_fd, write_fd = os.pipe2(os.O_CLOEXEC) + try: + # A cancellation must not land between fork and recording the + # exact child we must reap. The child restores the inherited + # mask only after PDEATHSIG and default handlers are installed. + original_mask = signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGTERM, signal.SIGINT}) + pid = None + try: + pid = os.fork() + if pid == 0: + try: + os.close(read_fd) + child(cpu, deadline, parent, self.worker.pw_uid, self.worker.pw_gid, + write_fd, original_mask) + except BaseException: + os._exit(1) + os._exit(0) + self.children[pid] = None + finally: + if pid != 0: + signal.pthread_sigmask(signal.SIG_SETMASK, original_mask) + os.close(write_fd) + write_fd = None + remaining = min(1.0, (deadline-time.monotonic_ns()) / 1e9) + require(remaining > 0 and select.select([read_fd], [], [], remaining)[0]) + require(os.read(read_fd, 2) == b"1") + self.ready += 1 + finally: + os.close(read_fd) + if write_fd is not None: + os.close(write_fd) + + def poll(self): + for pid, code in list(self.children.items()): + if code is None: + reaped, status = os.waitpid(pid, os.WNOHANG) + if reaped: + self.children[pid] = os.waitstatus_to_exitcode(status) + + def alive(self): + self.poll() + return self.ready == 2 and all(code is None for code in self.children.values()) + + def stop(self): + # Normal cancellation reaps our own direct children. If this controller + # is SIGKILLed, PDEATHSIG terminates workers but their reap belongs to + # the guest init; no completed controller receipt is then promised. + previous = {number: signal.signal(number, signal.SIG_IGN) + for number in (signal.SIGTERM, signal.SIGINT)} + try: + self._stop() + finally: + for number, handler in previous.items(): + signal.signal(number, handler) + + def _stop(self): + # These are unreaped direct children, not PIDs recovered from a pathname. + # A PID cannot be reused while its owned child has not been waitpid-reaped. + self.poll() + for pid, code in self.children.items(): + if code is None: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic_ns() + 1_000_000_000 + gentle = deadline - 750_000_000 + while any(code is None for code in self.children.values()) and time.monotonic_ns() < gentle: + self.poll() + time.sleep(0.01) + self.poll() + for pid, code in self.children.items(): + if code is None: + try: + os.kill(pid, signal.SIGKILL) + self.forced += 1 + except ProcessLookupError: + pass + while any(code is None for code in self.children.values()) and time.monotonic_ns() < deadline: + self.poll() + time.sleep(0.01) + self.poll() + require(all(code is not None for code in self.children.values())) + + def receipt(self): + return dict(spawned=len(self.children), ready=self.ready, + reaped=sum(code is not None for code in self.children.values()), + forced_kills=self.forced, exit_codes=list(self.children.values())) + + +def coordinate(observer, workers, *, clock=time.monotonic_ns, sleep=time.sleep, + sampler=PSI["observe"], memory=memory_available): + start = clock() + result = dict(schema_version=1, kind="content-provider-pressure-controller", success=False, + scope=dict(SCOPE), status="invalid", cold_begin=True, started_ns=start, + trigger_ns=None, verified_chunks=0, verified_bytes=0, + load_started_ns=None, load_stopped_ns=None, completed_ns=None, + wait_limit_ms=30000, load_limit_ms=10000, samples=0, + cpu_pressure_observed=False, other_pressure_observed=False, + psi_unavailable=False, initial_observation=None, last_observation=None, + first_pressure_ns=None, cleanup=None) + try: + while clock() < start + WAIT_NS: + require(memory()) + count = observer.sample() + require(type(count) is int and 0 <= count <= 3) + if count == 3: + require(clock() < start + WAIT_NS) + result.update(trigger_ns=clock(), verified_chunks=3, verified_bytes=3*CHUNK_BYTES) + break + sleep(max(0, min(POLL_NS, start + WAIT_NS-clock())) / 1e9) + else: + result["status"] = "trigger-timeout" + return result + initial = sampler() + PSI["validate_observation"](initial) + result["initial_observation"] = initial + result["last_observation"] = initial + if not all(state == "below_threshold" for state in initial.values()): + result["status"] = "resource-unavailable" + return result + require(memory()) + load_started = clock() + require(load_started < start + WAIT_NS) + result["load_started_ns"] = load_started + deadline = result["load_started_ns"] + ACTIVE_NS + workers.start(deadline) + while clock() < deadline: + require(memory()) + if not workers.alive(): + result["status"] = "load-failed" + break + observation = sampler() + PSI["validate_observation"](observation) + observed_ns = clock() + if observed_ns >= deadline: + result["status"] = "observed" if result["cpu_pressure_observed"] else "pressure-missing" + break + result["samples"] += 1 + require(result["samples"] <= 46) + result["last_observation"] = observation + result["cpu_pressure_observed"] |= observation["cpu"] == "pressured" + if observation["cpu"] == "pressured" and result["first_pressure_ns"] is None: + result["first_pressure_ns"] = observed_ns + result["other_pressure_observed"] |= any(observation[key] == "pressured" for key in ("memory", "io")) + result["psi_unavailable"] |= "unavailable" in observation.values() + if result["psi_unavailable"] or result["other_pressure_observed"]: + result["status"] = "resource-unavailable" + break + sleep(max(0, min(POLL_NS, deadline-clock())) / 1e9) + else: + result["status"] = "observed" if result["cpu_pressure_observed"] else "pressure-missing" + except InterruptedError: + result["status"] = "interrupted" + except (OSError, ValueError, KeyError, TypeError): + result["status"] = "invalid" + finally: + try: + workers.stop() + except (OSError, ValueError): + result["status"] = "cleanup-failed" + if result["load_started_ns"] is not None: + result["load_stopped_ns"] = clock() + if result["load_stopped_ns"] - result["load_started_ns"] > LOAD_NS: + result["status"] = "cleanup-failed" + result["cleanup"] = workers.receipt() + if result["status"] == "observed" and ( + result["cleanup"]["spawned"] != 2 or result["cleanup"]["ready"] != 2 + or result["cleanup"]["reaped"] != 2 + or any(code not in (0, -signal.SIGTERM) for code in result["cleanup"]["exit_codes"])): + result["status"] = "load-failed" + result["completed_ns"] = clock() + result["success"] = result["status"] == "observed" + return result + + +def validate(value): + keys = {"schema_version", "kind", "success", "scope", "status", "cold_begin", "started_ns", + "trigger_ns", "verified_chunks", "verified_bytes", "load_started_ns", "load_stopped_ns", + "completed_ns", "wait_limit_ms", "load_limit_ms", "samples", "cpu_pressure_observed", + "other_pressure_observed", "psi_unavailable", "initial_observation", "last_observation", + "first_pressure_ns", "cleanup", "publication"} + require(type(value) is dict and set(value) == keys and type(value["schema_version"]) is int + and value["schema_version"] == 1 and value["kind"] == "content-provider-pressure-controller" + and value["status"] in STATUSES and type(value["success"]) is bool + and value["success"] == (value["status"] == "observed") and value["scope"] == SCOPE + and all(type(v) is bool for v in value["scope"].values()) and value["cold_begin"] is True) + require(value["wait_limit_ms"] == 30000 and type(value["wait_limit_ms"]) is int + and value["load_limit_ms"] == 10000 and type(value["load_limit_ms"]) is int) + for key in ("started_ns", "completed_ns", "verified_chunks", "verified_bytes", "samples"): + require(type(value[key]) is int and 0 <= value[key] <= 2**64-1) + for key in ("trigger_ns", "load_started_ns", "load_stopped_ns", "first_pressure_ns"): + require(value[key] is None or type(value[key]) is int and value["started_ns"] <= value[key] <= value["completed_ns"]) + require(value["completed_ns"] >= value["started_ns"] and value["samples"] <= 46) + for key in ("cpu_pressure_observed", "other_pressure_observed", "psi_unavailable"): + require(type(value[key]) is bool) + require(value["verified_chunks"] in (0, 3) and value["verified_bytes"] == value["verified_chunks"] * CHUNK_BYTES) + require((value["trigger_ns"] is None) == (value["verified_chunks"] == 0)) + if value["trigger_ns"] is not None: + require(value["trigger_ns"] - value["started_ns"] < WAIT_NS) + if value["load_started_ns"] is not None: + require(0 <= value["load_started_ns"] - value["started_ns"] < WAIT_NS) + if value["last_observation"] is not None: + PSI["validate_observation"](value["last_observation"]) + if value["initial_observation"] is not None: + PSI["validate_observation"](value["initial_observation"]) + require((value["first_pressure_ns"] is not None) == value["cpu_pressure_observed"]) + if value["first_pressure_ns"] is not None: + require(value["load_started_ns"] is not None + and value["load_started_ns"] <= value["first_pressure_ns"] + < value["load_started_ns"] + ACTIVE_NS) + if value["load_stopped_ns"] is not None: + require(value["first_pressure_ns"] <= value["load_stopped_ns"]) + p = value["publication"] + require(type(p) is dict and set(p) == {"manifest_id", "publisher_hex", "object_sha256"} + and p["object_sha256"] == OBJECT_SHA + and all(type(p[key]) is str and re.fullmatch(r"[0-9a-f]{64}", p[key]) for key in p)) + c = value["cleanup"] + require(type(c) is dict and set(c) == {"spawned", "ready", "reaped", "forced_kills", "exit_codes"} + and all(type(c[key]) is int and 0 <= c[key] <= 2 for key in c if key != "exit_codes") + and 0 <= c["ready"] <= c["spawned"] and c["reaped"] <= c["spawned"] + and type(c["exit_codes"]) is list and len(c["exit_codes"]) == c["spawned"] + and all(code is None or type(code) is int and -64 <= code <= 255 for code in c["exit_codes"]) + and sum(code is not None for code in c["exit_codes"]) == c["reaped"]) + if value["success"]: + require(value["trigger_ns"] is not None and value["load_started_ns"] is not None + and value["load_stopped_ns"] is not None and value["load_started_ns"] >= value["trigger_ns"] + and 0 < value["load_stopped_ns"] - value["load_started_ns"] <= LOAD_NS + and value["cpu_pressure_observed"] and not value["other_pressure_observed"] + and not value["psi_unavailable"] and value["samples"] > 0 + and value["initial_observation"] == dict.fromkeys(PSI["THRESHOLDS"], "below_threshold") + and c["spawned"] == c["ready"] == c["reaped"] == 2 + and all(code in (0, -signal.SIGTERM) for code in c["exit_codes"])) + return value + + +def observe(work): + worker, agent_uid = guest(work) + root = directory(work, 0) + watcher = None + try: + require(absent(root, PREFIX + "-ready.json") and absent(root, PREFIX + "-controller.json")) + seed = publication(root, 0) + watcher = CacheObserver(root, agent_uid) + require(memory_available()) + ready_ns = time.monotonic_ns() + write_record(root, PREFIX + "-ready.json", dict(schema_version=1, kind="content-provider-pressure-ready", + cold_cache=True, ready_ns=ready_ns, publication=seed)) + print("Disposable guest only: two capless worker-UID CPU processes on one guest CPU, " + "after three verified public chunks; at most ten seconds including drain.", flush=True) + previous = {} + def interrupted(_signum, _frame): + # Further TERM cannot interrupt child drain after the first cancellation. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + raise InterruptedError() + try: + for signum in (signal.SIGTERM, signal.SIGINT): + previous[signum] = signal.signal(signum, interrupted) + result = coordinate(watcher, Workers(worker)) + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler) + result["publication"] = seed + validate(result) + write_record(root, PREFIX + "-controller.json", result) + return result["success"] + finally: + if watcher is not None: + watcher.close() + os.close(root) + + +def main(argv): + try: + require(len(argv) == 3 and argv[1] == "observe") + return 0 if observe(Path(argv[2])) else 1 + except (OSError, ValueError, KeyError, TypeError, UnicodeError): + print("CONTENT_PROVIDER_PRESSURE_INVALID", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/integration/test-content-provider-pressure.py b/tests/integration/test-content-provider-pressure.py new file mode 100644 index 000000000..ce39a6411 --- /dev/null +++ b/tests/integration/test-content-provider-pressure.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Offline fake-clock/process tests. Never fork load workers or stress the host.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import runpy +import signal +import tempfile +import types +import unittest +from unittest.mock import Mock, patch + +HERE = Path(__file__).resolve().parent +C = runpy.run_path(str(HERE / "content-provider-pressure.py")) +G = C["coordinate"].__globals__ + + +def seed(): + return dict(report_kind="volparossa-content-adaptive-provider-seed", + object_sha256=C["OBJECT_SHA"], bytes=60*262144, chunks=60, + publisher_removed=True, publisher_private_key_persisted=False, + recipient_encrypted=False, manifest_id="a"*64, publisher_hex="b"*64, + created_unix_seconds=1000, expires_unix_seconds=4600, + **{f"replica_{label}_{kind}": value for label in "abc" + for kind, value in (("chunks", 20), ("bytes", 20*262144))}) + + +def binding(): + return {key: seed()[key] for key in ("manifest_id", "publisher_hex", "object_sha256")} + + +def psi(**changes): + return dict(dict.fromkeys(("cpu", "memory", "io"), "below_threshold"), **changes) + + +class Clock: + def __init__(self): + self.now = 1_000_000_000 + + def __call__(self): + return self.now + + def sleep(self, seconds): + if seconds < 0: + raise AssertionError("negative sleep") + self.now += round(seconds * 1e9) + + +class Observer: + def __init__(self, values=(3,)): + self.values = iter(values) + self.last = 0 + + def sample(self): + self.last = next(self.values, self.last) + return self.last + + +class Workers: + def __init__(self): + self.started = 0 + self.reaped = 0 + self.trace = [] + self.codes = [] + + def start(self, deadline): + self.trace.append(("start", deadline)) + self.started = 2 + + def alive(self): + return True + + def stop(self): + self.trace.append(("stop",)) + self.reaped = self.started + self.codes = [0] * self.started + + def receipt(self): + return dict(spawned=self.started, ready=self.started, reaped=self.reaped, + forced_kills=0, exit_codes=self.codes) + + +def run(*, observer=None, workers=None, clock=None, sampler=None, memory=lambda: True): + observer, workers, clock = observer or Observer(), workers or Workers(), clock or Clock() + observations = iter([psi(), psi(cpu="pressured")]) + if sampler is None: + def sampler(): + return next(observations, psi(cpu="pressured")) + result = C["coordinate"](observer, workers, clock=clock, sleep=clock.sleep, + sampler=sampler, memory=memory) + result["publication"] = binding() + C["validate"](result) + return result, workers + + +class LifecycleTests(unittest.TestCase): + def test_real_trigger_precedes_load_and_receipt_never_claims_transfer(self): + result, workers = run(observer=Observer([0, 1, 2, 3])) + self.assertTrue(result["success"]) + self.assertEqual(result["verified_bytes"], 786432) + self.assertEqual(result["trigger_ns"]-result["started_ns"], 600_000_000) + self.assertGreaterEqual(result["load_started_ns"], result["trigger_ns"]) + self.assertGreaterEqual(result["first_pressure_ns"], result["load_started_ns"]) + self.assertEqual(result["initial_observation"], psi()) + self.assertEqual(result["load_stopped_ns"]-result["load_started_ns"], C["ACTIVE_NS"]) + self.assertEqual(result["samples"], 45) + self.assertEqual(result["cleanup"]["reaped"], 2) + self.assertEqual([step[0] for step in workers.trace], ["start", "stop"]) + self.assertFalse(result["scope"]["continuation_claimed"]) + self.assertFalse(result["scope"]["transfer_completion_claimed"]) + + def test_no_three_chunks_never_starts_load_or_resets_30s(self): + result, workers = run(observer=Observer([2])) + self.assertEqual(result["status"], "trigger-timeout") + self.assertEqual(result["completed_ns"]-result["started_ns"], C["WAIT_NS"]) + self.assertEqual(workers.trace, [("stop",)]) + self.assertIsNone(result["load_started_ns"]) + + def test_slow_final_sample_cannot_trigger_after_original_deadline(self): + clock = Clock() + observer = Observer() + def sample(): + clock.now += C["WAIT_NS"] + return 3 + observer.sample = sample + result, workers = run(observer=observer, clock=clock) + self.assertEqual(result["status"], "invalid") + self.assertEqual(workers.started, 0) + + def test_absent_or_unrelated_pressure_is_not_success(self): + for observation, expected in ((psi(), "pressure-missing"), + (psi(io="pressured"), "resource-unavailable"), + (psi(cpu="unavailable"), "resource-unavailable")): + with self.subTest(observation=observation): + samples = iter([psi(), observation]) + result, workers = run(sampler=lambda: next(samples, observation)) + self.assertEqual(result["status"], expected) + self.assertEqual(workers.reaped, 2) + self.assertFalse(result["success"]) + + def test_preexisting_pressure_and_memory_shortfall_never_start_load(self): + result, workers = run(sampler=lambda: psi(cpu="pressured")) + self.assertEqual(result["status"], "resource-unavailable") + self.assertEqual(workers.started, 0) + result, workers = run(memory=lambda: False) + self.assertFalse(result["success"]) + self.assertEqual(workers.started, 0) + + def test_failed_second_spawn_and_interrupt_always_reap_first(self): + for error in (OSError("closed"), InterruptedError()): + with self.subTest(error=type(error).__name__): + workers = Workers() + def start(_deadline): + workers.started = 1 + raise error + workers.start = start + result, _ = run(workers=workers) + self.assertEqual(result["status"], "interrupted" if isinstance(error, InterruptedError) else "invalid") + self.assertEqual(result["cleanup"]["reaped"], 1) + + def test_no_new_deadline_after_worker_setup(self): + clock, workers = Clock(), Workers() + original = workers.start + def start(deadline): + original(deadline) + clock.now += 8_000_000_000 + workers.start = start + result, _ = run(clock=clock, workers=workers) + self.assertEqual(result["samples"], 5) + self.assertEqual(result["load_stopped_ns"]-result["load_started_ns"], C["ACTIVE_NS"]) + + def test_preload_decision_and_receipt_share_original_deadline_sample(self): + class BoundaryClock(Clock): + tick_next = False + def __call__(self): + sampled = self.now + if self.tick_next: + self.now += 1 + self.tick_next = False + return sampled + for delta in (-1, 0): + with self.subTest(delta=delta): + clock = BoundaryClock() + start = clock.now + calls = 0 + def memory(): + nonlocal calls + calls += 1 + if calls == 2: + clock.now = start + C["WAIT_NS"] + delta + clock.tick_next = True + return True + result, workers = run(clock=clock, memory=memory) + if delta == -1: + self.assertTrue(result["success"]) + self.assertEqual(result["load_started_ns"], start+C["WAIT_NS"]-1) + else: + self.assertFalse(result["success"]) + self.assertIsNone(result["load_started_ns"]) + self.assertEqual(workers.started, 0) + + def test_delayed_pressure_sample_after_active_deadline_cannot_count(self): + clock = Clock() + calls = 0 + def sample(): + nonlocal calls + calls += 1 + if calls == 1: return psi() + clock.now += C["ACTIVE_NS"] + return psi(cpu="pressured") + result, _ = run(clock=clock, sampler=sample) + self.assertEqual(result["status"], "pressure-missing") + self.assertIsNone(result["first_pressure_ns"]) + self.assertFalse(result["cpu_pressure_observed"]) + + def test_cleanup_timeout_and_failed_child_cannot_report_success(self): + for kind in ("late", "failed", "exception"): + with self.subTest(kind=kind): + clock, workers = Clock(), Workers() + original = workers.stop + def stop(): + original() + if kind == "late": clock.now += 2_000_000_000 + if kind == "failed": workers.codes = [0, 1] + if kind == "exception": raise OSError("cleanup") + workers.stop = stop + result, _ = run(clock=clock, workers=workers) + self.assertFalse(result["success"]) + self.assertEqual(result["status"], "load-failed" if kind == "failed" else "cleanup-failed") + + def test_closed_schema_rejects_invented_claims_and_unreaped_success(self): + result, _ = run() + changes = [lambda r: r.update(url="private"), lambda r: r.update(schema_version=True), + lambda r: r["scope"].update(continuation_claimed=True), + lambda r: r.update(cpu_pressure_observed=False), + lambda r: r["cleanup"].update(reaped=1), + lambda r: r.update(trigger_ns=r["started_ns"]+C["WAIT_NS"]), + lambda r: r.update(load_stopped_ns=r["load_started_ns"]+C["LOAD_NS"]+1)] + for change in changes: + value = copy.deepcopy(result) + change(value) + with self.assertRaises((ValueError, TypeError)): + C["validate"](value) + self.assertLess(len(json.dumps(result)), C["MAX_REPORT"]) + + def test_receipt_rejects_late_load_and_pressure_after_child_drain(self): + result, _ = run() + late = copy.deepcopy(result) + for key in ("load_started_ns", "load_stopped_ns", "first_pressure_ns", "completed_ns"): + late[key] += C["WAIT_NS"] + with self.assertRaises(ValueError): C["validate"](late) + after_drain = copy.deepcopy(result) + after_drain["load_stopped_ns"] = result["load_started_ns"] + 1_000_000_000 + after_drain["first_pressure_ns"] = result["load_started_ns"] + 2_000_000_000 + with self.assertRaises(ValueError): C["validate"](after_drain) + + +class FileTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="pressure-fixture-unit-") + self.addCleanup(temporary.cleanup) + self.work = Path(temporary.name) + self.uid = os.geteuid() + self.root = os.open(self.work, os.O_RDONLY | os.O_DIRECTORY) + self.addCleanup(os.close, self.root) + (self.work / "state-client").mkdir(mode=0o700) + self.parent = self.work / "state-client/content-adaptive-pressure" + self.parent.mkdir(mode=0o700) + + def write(self, path, data): + path.write_bytes(data) + path.chmod(0o600) + + def observer(self): + observer = C["CacheObserver"](self.root, self.uid) + self.addCleanup(observer.close) + return observer + + def test_seed_binding_is_actual_A_B_C_vector_and_rejects_alternatives(self): + source = (HERE.parents[1]/"crates/volparossa-content/examples/content-acceptance-fixture/adaptive.rs").read_text() + self.assertIn("(b'A'..b'A' + 60)", source) + self.assertIn("caches[index % 3].put_verified", source) + self.assertIn(C["OBJECT_SHA"], source) + self.assertEqual(C["CHUNKS"], tuple(hashlib.sha256(bytes([65+i])*262144).hexdigest() for i in range(3))) + path = self.work / "content-provider-adaptive-publication.json" + self.write(path, json.dumps(seed()).encode()) + self.assertEqual(C["publication"](self.root, self.uid, now=1001), binding()) + for key, value in (("object_sha256", "0"*64), ("chunks", 9), ("replica_c_chunks", 19), + ("publisher_removed", False), ("expires_unix_seconds", 999)): + broken = seed(); broken[key] = value + self.write(path, json.dumps(broken).encode()) + with self.assertRaises(ValueError): + C["publication"](self.root, self.uid, now=1001) + + def test_cold_initial_state_real_complete_hashes_no_writes_or_store_lock(self): + watcher = self.observer() + self.assertEqual(watcher.sample(), 0) + cache = self.parent / "cache"; cache.mkdir(mode=0o700) + for i, name in enumerate(C["CHUNKS"]): + self.write(cache/name, bytes([65+i])*262144) + self.assertEqual(watcher.sample(), i+1) + self.assertEqual(sorted(p.name for p in cache.iterdir()), sorted(C["CHUNKS"])) + with self.assertRaises(ValueError): + C["CacheObserver"](self.root, self.uid) + + def test_wrong_hash_size_symlink_hardlink_fifo_and_mode_fail_closed(self): + watcher = self.observer() + cache = self.parent / "cache"; cache.mkdir(mode=0o700) + chunk = cache/C["CHUNKS"][0] + for kind in ("hash", "size", "symlink", "hardlink", "fifo", "mode"): + with self.subTest(kind=kind): + if kind == "symlink": chunk.symlink_to("missing") + elif kind == "fifo": os.mkfifo(chunk, 0o600) + else: + self.write(chunk, b"B"*262144 if kind == "hash" else b"A"*(262143 if kind == "size" else 262144)) + if kind == "hardlink": os.link(chunk, cache/"alias") + if kind == "mode": chunk.chmod(0o644) + with self.assertRaises((OSError, ValueError)): + watcher.sample() + chunk.unlink() + if (cache/"alias").exists(): (cache/"alias").unlink() + + def test_directory_replacement_and_symlink_rejected(self): + watcher = self.observer() + cache = self.parent / "cache"; cache.mkdir(mode=0o700) + self.assertEqual(watcher.sample(), 0) + cache.rename(self.parent/"old") + cache.mkdir(mode=0o700) + with self.assertRaises(ValueError): watcher.sample() + cache.rmdir(); cache.symlink_to("old", target_is_directory=True) + with self.assertRaises(OSError): watcher.sample() + + def test_same_descriptor_read_detects_in_place_change_and_closes(self): + target = self.work/"public" + self.write(target, b"abc") + actual = os.read + descriptors = [] + def mutate(fd, count): + descriptors.append(fd) + data = actual(fd, count) + if data: + target.write_bytes(b"xyz") + os.utime(target, ns=(1, 1)) + return data + with patch.object(G["os"], "read", side_effect=mutate): + with self.assertRaises(ValueError): C["file_bytes"](self.root, "public", self.uid, 3) + with self.assertRaises(OSError): os.fstat(descriptors[0]) + + def test_closed_receipt_is_private_exclusive_and_no_raw_paths(self): + result, _ = run() + name = C["PREFIX"]+"-controller.json" + C["write_record"](self.root, name, result) + data = C["file_bytes"](self.root, name, self.uid, C["MAX_REPORT"]) + self.assertEqual(json.loads(data), result) + self.assertNotIn(str(self.work).encode(), data) + with self.assertRaises(FileExistsError): C["write_record"](self.root, name, result) + + def test_publication_exposes_only_complete_json_and_cleans_failed_stage(self): + result, _ = run() + name = C["PREFIX"]+"-controller.json" + actual = C["publish"] + def inspect(parent, temporary, final): + self.assertFalse((self.work/final).exists()) + self.assertEqual(json.loads(C["file_bytes"](parent, temporary, self.uid, C["MAX_REPORT"])), result) + actual(parent, temporary, final) + with patch.dict(G, {"publish": inspect}): C["write_record"](self.root, name, result) + self.assertEqual(json.loads((self.work/name).read_bytes()), result) + self.assertFalse((self.work/("."+name+".part")).exists()) + ready = C["PREFIX"]+"-ready.json" + with patch.dict(G, {"publish": Mock(side_effect=OSError("simulated"))}): + with self.assertRaises(OSError): C["write_record"](self.root, ready, {}) + self.assertFalse((self.work/ready).exists()) + self.assertFalse((self.work/("."+ready+".part")).exists()) + + +class ProcessContractTests(unittest.TestCase): + def test_host_guard_rejects_before_any_fork(self): + with patch.object(G["os"], "fork", side_effect=AssertionError("must never fork")), \ + patch.object(G["os"], "geteuid", return_value=1000): + with self.assertRaises(ValueError): C["guest"](Path("/tmp/not-a-guest")) + with patch.dict(G, {"observe": Mock(side_effect=AssertionError("no worker CLI"))}): + self.assertEqual(C["main"](["script", "load", "1"]), 1) + + def test_actual_owned_child_reap_only_signals_unreaped_children(self): + workers = C["Workers"](types.SimpleNamespace(pw_uid=985, pw_gid=985)) + workers.children = {11: None, 12: None}; workers.ready = 2 + responses = {11: [(11, 0)], 12: [(0, 0), (12, signal.SIGTERM)]} + def waitpid(pid, flags): return responses[pid].pop(0) + with patch.object(G["os"], "waitpid", side_effect=waitpid), \ + patch.object(G["os"], "kill") as kill, \ + patch.object(G["time"], "sleep"): + workers.stop() + kill.assert_called_once_with(12, signal.SIGTERM) + self.assertEqual(workers.receipt()["reaped"], 2) + + def test_failed_handshake_keeps_exact_child_owned_for_cleanup(self): + workers = C["Workers"](types.SimpleNamespace(pw_uid=985, pw_gid=985)) + with patch.object(G["os"], "fork", return_value=12345), \ + patch.object(G["select"], "select", return_value=([], [], [])), \ + patch.object(G["time"], "monotonic_ns", return_value=1): + with self.assertRaises(ValueError): workers.start(10_000_000_000) + self.assertEqual(workers.children, {12345: None}) + self.assertEqual(workers.ready, 0) + + def test_fork_cancellation_cannot_precede_child_ownership(self): + workers = C["Workers"](types.SimpleNamespace(pw_uid=985, pw_gid=985)) + calls = [] + def mask(how, value): + calls.append((how, value)) + if how == signal.SIG_SETMASK: + self.assertEqual(workers.children, {12345: None}) + raise InterruptedError() + self.assertEqual(value, {signal.SIGTERM, signal.SIGINT}) + return {signal.SIGUSR1} + with patch.object(G["os"], "fork", return_value=12345), \ + patch.object(G["signal"], "pthread_sigmask", side_effect=mask), \ + patch.object(G["time"], "monotonic_ns", return_value=1): + with self.assertRaises(InterruptedError): workers.start(10_000_000_000) + self.assertEqual(calls[-1], (signal.SIG_SETMASK, {signal.SIGUSR1})) + self.assertEqual(workers.ready, 0) + + def test_child_sandbox_restores_parent_death_signal_after_uid_drop(self): + # Run setup with every process-mutating operation mocked and a clock + # already past the loop deadline after its acknowledgement: zero load. + events = [] + libc = types.SimpleNamespace(prctl=lambda option, arg, *_: events.append(("prctl", option, arg)) or 0) + status = "".join(f"{key}:\t0000000000000000\n" for key in + ("CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb")) + "NoNewPrivs:\t1\n" + names = ("setsid", "umask", "setgroups", "setresgid", "setresuid", "sched_setaffinity", + "dup2", "closerange", "close", "write") + patches = [patch.object(G["os"], name, side_effect=lambda *args, n=name: events.append((n, *args))) + for name in names] + for item in patches: item.start() + try: + with patch.object(G["ctypes"], "CDLL", return_value=libc), \ + patch.object(G["os"], "getppid", return_value=123), \ + patch.object(G["os"], "getresuid", return_value=(985,)*3), \ + patch.object(G["os"], "getresgid", return_value=(985,)*3), \ + patch.object(G["os"], "getgroups", return_value=[]), \ + patch.object(G["os"], "sched_getaffinity", return_value={1}), \ + patch.object(G["os"], "open", return_value=4), \ + patch.object(G["signal"], "signal"), \ + patch.object(G["signal"], "pthread_sigmask", side_effect=lambda *args: events.append(("sigmask", *args))), \ + patch.object(G["resource"], "setrlimit") as limit, \ + patch.object(G["resource"], "getrlimit", return_value=(1024, 1024)), \ + patch.object(G["Path"], "read_text", return_value=status), \ + patch.object(G["time"], "monotonic_ns", side_effect=[0, 100]): + C["child"](1, 100, 123, 985, 985, 9, {signal.SIGUSR1}) + self.assertGreaterEqual(limit.call_count, 5) + finally: + for item in reversed(patches): item.stop() + death = [i for i, event in enumerate(events) if event == ("prctl", 1, signal.SIGKILL)] + uid = events.index(("setresuid", 985, 985, 985)) + self.assertEqual(len(death), 2) + self.assertLess(death[0], uid); self.assertGreater(death[1], uid) + self.assertGreater(events.index(("sigmask", signal.SIG_SETMASK, {signal.SIGUSR1})), death[0]) + self.assertIn(("prctl", 38, 1), events) + self.assertIn(("write", 3, b"1"), events) + + +if __name__ == "__main__": + unittest.main() From 52a3a7876647f1abba8715efb34bce1fc7f5a2fd Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 06:52:29 +0200 Subject: [PATCH 47/49] test: wire protected cache recovery under bounded CPU pressure --- .github/workflows/alpha-topology.yml | 6 + .github/workflows/quality.yml | 6 + docs/IMPLEMENTATION_STATUS.md | 72 ++++- .../content-provider-adaptive-smoke.sh | 1 + .../content-provider-pressure-smoke.py | 290 ++++++++++++++++++ .../content-provider-pressure-smoke.sh | 260 ++++++++++++++++ .../content-provider-resource-ready.py | 3 +- tests/integration/content-provider-smoke.py | 49 ++- tests/integration/content-provider-smoke.sh | 27 +- .../content-provider-worker-budget.py | 11 +- tests/integration/kvm-alpha-topology.sh | 15 + tests/integration/run-alpha-topology-vm.sh | 33 +- .../test-content-provider-pressure-shell.py | 177 +++++++++++ .../test-content-provider-pressure-smoke.py | 258 ++++++++++++++++ .../test-content-provider-pressure-wiring.py | 226 ++++++++++++++ .../test-content-provider-resource-ready.py | 44 ++- .../test-content-provider-smoke.py | 96 +++++- .../test-content-provider-worker-budget.py | 92 +++++- 18 files changed, 1629 insertions(+), 37 deletions(-) create mode 100644 tests/integration/content-provider-pressure-smoke.py create mode 100644 tests/integration/content-provider-pressure-smoke.sh create mode 100644 tests/integration/test-content-provider-pressure-shell.py create mode 100644 tests/integration/test-content-provider-pressure-smoke.py create mode 100644 tests/integration/test-content-provider-pressure-wiring.py diff --git a/.github/workflows/alpha-topology.yml b/.github/workflows/alpha-topology.yml index 114e3c09b..bd99f5df1 100644 --- a/.github/workflows/alpha-topology.yml +++ b/.github/workflows/alpha-topology.yml @@ -181,6 +181,12 @@ jobs: python3 -B tests/integration/test-content-provider-filter-smoke.py python3 -B tests/integration/test-content-provider-smoke.py python3 -B tests/integration/test-content-provider-worker-budget.py + python3 -B tests/integration/test-content-provider-resource-ready.py + python3 -B tests/integration/test-content-provider-pressure.py + python3 -B tests/integration/test-content-provider-pressure-smoke.py + python3 -B tests/integration/test-content-provider-pressure-shell.py + python3 -B tests/integration/test-content-provider-pressure-wiring.py + sh -n tests/integration/content-provider-pressure-smoke.sh python3 -B tests/integration/test-content-cancellation-smoke.py sh -n tests/integration/content-provider-late-recovery.sh python3 -B tests/integration/test-content-provider-late-recovery.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 21820d8fe..6626c5f14 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -74,6 +74,12 @@ jobs: - name: Check closed provider and Weather diagnostics without network access run: | python3 -B tests/integration/test-content-provider-worker-budget.py + python3 -B tests/integration/test-content-provider-resource-ready.py + python3 -B tests/integration/test-content-provider-pressure.py + python3 -B tests/integration/test-content-provider-pressure-smoke.py + python3 -B tests/integration/test-content-provider-pressure-shell.py + python3 -B tests/integration/test-content-provider-pressure-wiring.py + sh -n tests/integration/content-provider-pressure-smoke.sh node --version node -e 'if (Number(process.versions.node.split(".")[0]) < 22) process.exit(1)' node --test tests/integration/test-weather-source-observer.mjs diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 6e287e6cd..2c0a61f87 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,17 +29,19 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Preserve existing flows when a new socket is definitively rejected, then resume joined Weather delivery. | Run `38199501353` on `0e23140b` passes the standalone sequence. The later joined run on `9b132f6c` stops at named-cache fetch: one definite kernel rejection incorrectly destroys the shared route. The original socket errno remains unknown. Pressure retirement and continuation still require a separate protected trial. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Prove pressure retirement and continuation, then resume joined Weather delivery. | The route-preservation correction passes the standalone sequence in run `38204038877` on `9441b451`. The failed joined run and unknown original kernel errno remain separately recorded. The new pressure phase is wired but has not yet passed its protected trial. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run stops before Weather acquisition, so it makes no new weather-source requests. An earlier exact-source probe encountered ECMWF HTTP 429. Closed retention includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests or automatic retries. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover joined delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR13 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | -The standalone provider proof passes without weather downloads. The latest joined -trial uses the corrected Overmorrow bundle but exposes an earlier shared-route -failure. Correct that failure before another joined trial; do not automatically -retry source requests or route them through other peers to evade a source limit. +The corrected standalone provider proof passes without weather downloads. The +latest joined trial uses the corrected Overmorrow bundle but stops at the earlier +shared-route failure. The new standalone result clears that prerequisite, not +the joined delivery gate. Do not automatically retry source requests or route +them through other peers to evade a source limit. Pressure retirement and continuation remain separate evidence gaps, not implied -by successful parallel delivery. Their test helper is being prepared separately -and is not part of the failed run. +by successful parallel delivery. A separate disposable-guest phase is now wired +after the healthy native and HTTPS transfers. It is not part of either retained +standalone result or the failed joined run. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM @@ -74,8 +76,9 @@ still require route retirement. Eleven targeted tests pass: seven retirement tests, three helper-protocol tests and the route-loss classification test. The tests exercise the production error conversion, real local sibling socket ownership and malformed replies; they do not substitute for MPTCP dataplane -evidence. Formatting and independent review pass. The correction has not yet -passed a new protected network trial. +evidence. Formatting and independent review pass. The correction also passes +the protected standalone trial below; the original failed joined run remains +failed. Failure evidence is retained at `.git/ci-evidence/weather-overmorrow-joined.f58CgX/`. The source archive hash is @@ -86,9 +89,58 @@ Cleanup completes with zero owned objects; the VM, private scratch directory and forwarded test port are gone. Readable development-host network measurements are unchanged; firewall access was unavailable before and after, not verified equal. +### Verified corrected provider baseline + +[Run `38204038877`](https://github.com/VOLPAROSSA/volparossa/actions/runs/38204038877) +passes on exact source `9441b4514ae0c66c6353158fdeb45579773c32fa`. +Independent validation with that revision's schema-three checker reconstructs +the complete aggregate from its raw evidence. Named fetch delivers 2,097,275 +bytes in nine chunks from two providers. Native and HTTPS adaptive phases each +deliver 15,728,640 bytes in sixty chunks from three providers, with measured +three-provider overlaps of 5.139 and 4.655 seconds. HTTPS uses one authenticated +HEAD and no origin body or range requests. + +Late recovery completes in 11.096 seconds, with its gate released after 4.975 +seconds. Requester cancellation preserves independent cache-only delivery +(165 ms), closes control IPC (7 ms), and drains the original agent and helper. +All owned objects are removed; the disposable guest-root network state is +unchanged. This is guest evidence, not a development-host comparison. + +Evidence is retained at +`.git/ci-evidence/content-provider-hosted-9441.M6pG5O/`; the aggregate SHA-256 is +`0d8cbd696240dd7d12f73545c4ef305b5b73e6a845620388a88bad3819d449a3`. +Worker allowance remains twelve, with a peak of three leases and no pressure or +continuation events. The run proves neither pressure recovery nor joined Weather +delivery. CodeQL passes for this exact revision; Quality checks are still running. + +### Prepared pressure retirement trial + +The new phase uses the original sixty-chunk publication, three disjoint provider +caches and the same protected route, but a fresh client cache. After observing +three complete, hash-verified chunks, a guest-only controller starts two capless +CPU workers on one guest CPU for at most ten seconds, including child cleanup. +Three fixed 2 Mbit/s provider egress limits leave work in flight; the fixture +restores only its own qdiscs before the enclosing route teardown. It does not +alter production resource observations, admission rules, deadlines or device +affinity. + +Passing requires real CPU-pressure observations, at least three initial worker +leases followed by contraction to at most one, useful serial continuation after +the first batch drains, full reconstruction, unchanged protected paths and +privacy checks. The evidence does not claim that pressure has already fallen +before continuation, or that owner-task performance has improved. The +controller, separate production diagnostics and final object verification must +agree; no one receipt substitutes for the others. + +Current provider reports use schema four and require this separate component. +Earlier schema-one through schema-three results remain verifiable only through +the explicit historical-report path; their original claims are unchanged. +Focused controller, component, diagnostic, wiring and aggregate tests pass. +These are test-harness checks, not a completed live pressure trial. + ### Verified cache delivery and remaining integration -The latest [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38199501353) +The earlier [hosted cache trial](https://github.com/VOLPAROSSA/volparossa/actions/runs/38199501353) on `0e23140b` **passes** the complete standalone provider sequence. Independent validation of the retained schema-three report and reconstructed component evidence confirms: diff --git a/tests/integration/content-provider-adaptive-smoke.sh b/tests/integration/content-provider-adaptive-smoke.sh index 5d202b330..79dcd26f6 100644 --- a/tests/integration/content-provider-adaptive-smoke.sh +++ b/tests/integration/content-provider-adaptive-smoke.sh @@ -329,6 +329,7 @@ content_provider_adaptive_run() { client_forwarded_discovery:$discovered}' >"$WORK/content-provider-adaptive-object.json" content_provider_adaptive_https_run + content_provider_pressure_run PHASE=content-provider-adaptive-cleanup for adaptive_node in relay3 relay4 relay5; do diff --git a/tests/integration/content-provider-pressure-smoke.py b/tests/integration/content-provider-pressure-smoke.py new file mode 100644 index 000000000..d606e7b46 --- /dev/null +++ b/tests/integration/content-provider-pressure-smoke.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Disposable pressure-retirement component; not owner-goodput or PSI-recovery proof. + +The controller observes real guest CPU PSI after three verified cold-cache chunks. +Production diagnostics independently prove admission contraction and useful serial +continuation. Exact reconstruction and protected-path captures are still mandatory. +""" +import os +from pathlib import Path +import re +import runpy +import sys +import time + +HERE = Path(__file__).resolve().parent +ADAPTIVE = runpy.run_path(str(HERE / "content-provider-adaptive-smoke.py")) +CONTROLLER = runpy.run_path(str(HERE / "content-provider-pressure.py")) +BUDGET = runpy.run_path(str(HERE / "content-provider-worker-budget.py")) +RESOURCE = runpy.run_path(str(HERE / "content-provider-resource-ready.py")) +BASE = ADAPTIVE["BASE"] +require, NODES, ROLES = ADAPTIVE["require"], ADAPTIVE["NODES"], ADAPTIVE["ROLES"] +PREFIX = "content-provider-pressure" +BYTES, CHUNKS, SHA = (ADAPTIVE[key] for key in ("BYTES", "CHUNKS", "SHA")) +MAX_FILE = 65536 +MAX_EVIDENCE = 262144 +SCOPE = dict(disposable_guest=True, real_cpu_pressure=True, synthetic_budget=False, + pressure_recovery_claimed=False, owner_goodput_claimed=False) +FIELDS = {"schema_version", "kind", "success", "scope", "publication", "expected_peers", + "layout", "original_selected_route", "selected_before", "selected_route", + "status_before", "status_after", "fetch", "output", "privacy", "control_underlay", + "control_filter", "providers", "resource_ready", "ready", "controller", + "worker_budget", "continuation", "qdisc", "cleanup"} +QDISC_PROFILE = dict(profile="fixed-provider-egress-2mbit", + devices={node: f"r{node[-1]}x" for node in NODES}, handle="884:", + rate_bits_per_second=2000000, burst_bytes=131072, queue_latency_ms=250) +CLEANUP = dict(controller_reaped=True, fetch_reaped=True, owned_output_removed=True, + cache_teardown_deferred_to_topology=True, qdisc_restored=True, route_retained=True) + + +def integer(value, low=0, high=2**64-1): + return type(value) is int and low <= value <= high + + +def publication(value): + keys = {"report_kind", "publisher_hex", "manifest_id", "object_sha256", "bytes", "chunks", + "created_unix_seconds", "expires_unix_seconds", "publisher_removed", + "publisher_private_key_persisted", "recipient_encrypted"} + keys |= {f"replica_{label}_{kind}" for label in "abc" for kind in ("chunks", "bytes")} + require(type(value) is dict and set(value) == keys, "unexpected publication envelope") + require(value["report_kind"] == "volparossa-content-adaptive-provider-seed" + and value["object_sha256"] == SHA and integer(value["bytes"], BYTES, BYTES) + and integer(value["chunks"], CHUNKS, CHUNKS) + and value["publisher_removed"] is True and value["publisher_private_key_persisted"] is False + and value["recipient_encrypted"] is False, "original public adaptive object missing") + require(all(type(value[key]) is str and re.fullmatch(r"[0-9a-f]{64}", value[key]) + for key in ("manifest_id", "publisher_hex")), "original publisher/manifest binding invalid") + require(integer(value["created_unix_seconds"], 1) and integer(value["expires_unix_seconds"], 1) + and value["expires_unix_seconds"] - value["created_unix_seconds"] == 3600, + "original publication lifetime changed") + for label in "abc": + require(integer(value[f"replica_{label}_chunks"], 20, 20) + and integer(value[f"replica_{label}_bytes"], 5242880, 5242880), + "original disjoint provider shard changed") + return {key: value[key] for key in ("manifest_id", "publisher_hex", "object_sha256")} + + +def validate_ready(value, seed): + require(type(value) is dict and set(value) == + {"schema_version", "kind", "cold_cache", "ready_ns", "publication"} + and integer(value["schema_version"], 1, 1) + and value["kind"] == "content-provider-pressure-ready" + and value["cold_cache"] is True and integer(value["ready_ns"], 1) + and value["publication"] == publication(seed), "controller readiness is not bound to cold original input") + return value + + +def diagnostic(value, kind, validator): + require(type(value) is dict and set(value) == + {"schema_version", "kind", "phase", "phase_closed", "complete", "status", "records"} + and integer(value["schema_version"], 1, 1) and value["kind"] == kind + and value["phase"] == "pressure" and value["phase_closed"] is True + and value["complete"] is True and value["status"] == "observed" + and type(value["records"]) is list and len(value["records"]) == 1, + "one complete phase-bound production diagnostic required") + return validator(value["records"][0]) + + +def validate_diagnostics(evidence): + ready = RESOURCE["validate"](evidence["resource_ready"]) + require(ready["phase"] == "pressure" and ready["status"] == "ready", + "pressure phase lacks real initial resource readiness") + budget = diagnostic(evidence["worker_budget"], "content-worker-budget-diagnostic", BUDGET["summary"]) + require(budget["outcome"] == "returned" + and integer(budget["initial_allowance"], 3) + and integer(budget["min_allowance"], 0, 1) + and integer(budget["max_allowance"], budget["initial_allowance"]) + and integer(budget["min_memory_workers"], 3) and integer(budget["min_fd_workers"], 3) + and budget["peak_leases"] >= 3 and budget["active_leases"] == 0 + and budget["causes"] == {key: key == "cpu_pressure" for key in BUDGET["CAUSES"]}, + "three admitted workers did not contract under observed CPU-only pressure") + continuation = diagnostic(evidence["continuation"], "content-provider-continuation-diagnostic", + BUDGET["continuation_summary"]) + require(continuation["outcome"] == "returned" + and 1 <= continuation["returned_useful_continuations"] + <= continuation["admitted_continuations"] <= continuation["prepared_continuations"] + <= continuation["retired_candidates"] <= 3 + and 1 <= continuation["returned_verified_chunks"] <= CHUNKS + and continuation["returned_verified_bytes"] == continuation["returned_verified_chunks"] * 262144 + and continuation["initial_active_workers"] == continuation["final_active_workers"] == 0 + and continuation["peak_active_workers"] == 1, + "drained original batch lacks useful single-use serial continuation") + + +def validate_qdisc(value): + require(type(value) is dict and set(value) == {"profile", "before", "limited", "after"} + and value["profile"] == QDISC_PROFILE, "fixed disposable shaping profile changed") + for key in ("rate_bits_per_second", "burst_bytes", "queue_latency_ms"): + require(type(value["profile"][key]) is int, "invalid shaping profile number") + for phase in ("before", "limited", "after"): + require(type(value[phase]) is dict and set(value[phase]) == set(NODES), "shaping coverage incomplete") + for node in NODES: + rows = value[phase][node] + require(type(rows) is list and len(rows) == 1 and type(rows[0]) is dict, + "unexpected disposable egress qdisc") + row = rows[0] + require(row["root"] is True, "qdisc is not the selected egress root") + if "dev" in row: + require(row["dev"] == QDISC_PROFILE["devices"][node], "qdisc device changed") + if phase == "limited": + require(row["kind"] == "tbf" and row["handle"] == "884:" + and type(row["options"]) is dict + and integer(row["options"]["rate"], 250000, 250000), "fixed egress rate absent") + for key, expected in (("burst", 131072), ("lat", 250000)): + if key in row["options"]: + require(integer(row["options"][key], expected, expected), "observed qdisc option changed") + else: + require(row["kind"] == "noqueue" and row["handle"] == "0:", "original noqueue not restored") + for node in NODES: + # Packet counters/refcounts may legitimately change; root configuration may not. + config = lambda row: {key: row.get(key) for key in ("kind", "handle", "root", "options", "dev")} + require(config(value["before"][node][0]) == config(value["after"][node][0]), + "preexisting egress configuration not restored") + + +def validate(evidence): + require(type(evidence) is dict and set(evidence) == FIELDS + and integer(evidence["schema_version"], 1, 1) + and evidence["kind"] == "content-provider-pressure-smoke" and evidence["success"] is True + and evidence["scope"] == SCOPE and all(type(v) is bool for v in evidence["scope"].values()), + "invalid closed pressure component") + seed, peers, layout = (evidence[key] for key in ("publication", "expected_peers", "layout")) + binding = publication(seed) + ready = validate_ready(evidence["ready"], seed) + controller = CONTROLLER["validate"](evidence["controller"]) + require(controller["success"] is True and controller["publication"] == binding + and ready["ready_ns"] <= controller["started_ns"], "real controller completion missing or substituted") + validate_diagnostics(evidence) + require(layout["provider_nodes"] == list(NODES) and len(set(peers.values())) == len(peers), + "three independent provider identities missing") + control = layout["control_relay_peer_id"] + nodes = [node for node in ROLES[1:4] if peers[node] == control] + require(len(nodes) == 1 and control not in {peers[node] for node in NODES}, "broker not independent") + require(all(evidence[key]["control_relay_peer_id"] == control + for key in ("status_before", "status_after", "fetch")), "control authority changed") + fetch, output = evidence["fetch"], evidence["output"] + require(fetch["operation"] == "native_content" and integer(fetch["bytes"], BYTES, BYTES) + and integer(fetch["chunks"], CHUNKS, CHUNKS) and integer(fetch["providers_used"], 3, 3) + and type(fetch["provider_peer_ids"]) is list and len(fetch["provider_peer_ids"]) == 3 + and set(fetch["provider_peer_ids"]) == {peers[node] for node in NODES} + and integer(fetch["peer_bytes"], BYTES, BYTES) and integer(fetch["origin_body_bytes"], 0, 0) + and integer(fetch["origin_range_requests"], 0, 0) and fetch["origin_authenticated"] is False, + "full native transfer from three useful protected providers missing") + require(output["sha256"] == SHA and integer(output["bytes"], BYTES, BYTES) + and all(output[key] is True for key in ("client_cache_initially_absent", "client_mount_positive_control", + "client_mount_cannot_read_replica_stores", "publisher_process_exited_before_fetch")) + and integer(output["event_baseline_unix_ms"], 1) + and integer(output["generic_dht_queries"], 1) and integer(output["authenticated_upstream_offers"], 3) + and integer(output["client_forwarded_discovery"], 1), "cold verified reconstruction/isolation absent") + for key in ("started_monotonic_ns", "completed_monotonic_ns", "started_unix_ms", "completed_unix_ms"): + require(integer(output[key], 1), "invalid original fetch clock") + require(ready["ready_ns"] <= output["started_monotonic_ns"] <= controller["trigger_ns"] + <= controller["first_pressure_ns"] <= output["completed_monotonic_ns"] + and 0 < output["completed_monotonic_ns"] - output["started_monotonic_ns"] <= 120_000_000_000 + and seed["created_unix_seconds"] * 1000 <= output["started_unix_ms"] + <= output["completed_unix_ms"] < seed["expires_unix_seconds"] * 1000, + "CPU event occurred outside the original bounded, live-publication fetch") + original, before, live = (evidence[key] for key in + ("original_selected_route", "selected_before", "selected_route")) + def route_identity(value): + return dict(value, paths=[{key: item for key, item in path.items() + if key not in {"reported_bytes", "smoothed_rtt_us", "state"}} + for path in value["paths"]]) + require(original == before and route_identity(before) == route_identity(live), + "original protected route was replaced or rebound") + BASE["validate_path"](evidence, peers, list(NODES), False) + for node in NODES: + count = evidence["privacy"]["exit"]["provider_application"][node]["response_payload_bytes"] + require(integer(count, ADAPTIVE["SHARD_BYTES"], ADAPTIVE["SHARD_BYTES"] + 131072), + "provider payload differs from its original disjoint shard") + ADAPTIVE["validate_control"](evidence, nodes[0]) + ADAPTIVE["validate_filter"](evidence["control_filter"]) + require(type(evidence["providers"]) is dict and set(evidence["providers"]) == set(NODES), + "provider lifecycle coverage incomplete") + for provider in evidence["providers"].values(): + require(type(provider) is dict and set(provider) == {"reset", "serve", "status", "stop"}, + "unexpected provider lifecycle records") + for operation in ("reset", "stop"): + require(provider[operation]["serving"] is False + and integer(provider[operation]["publications"], 0, 0), "provider stop incomplete") + for operation in ("serve", "status"): + require(provider[operation]["serving"] is True + and integer(provider[operation]["publications"], 1, 1) + and provider[operation]["replication_enabled"] is False, "original isolated service missing") + validate_qdisc(evidence["qdisc"]) + require(evidence["cleanup"] == CLEANUP and all(type(v) is bool for v in evidence["cleanup"].values()), + "pressure-owned resources not reaped/restored or enclosing cache ownership lost") + return evidence + + +def read_at(directory, filename, maximum=MAX_FILE): + return BUDGET["decode"](CONTROLLER["file_bytes"](directory, filename, os.geteuid(), maximum)) + + +def ready(work): + work = Path(work) + require(work.is_absolute() and work.resolve() == work, "unsafe fixture root") + directory = CONTROLLER["directory"](work, os.geteuid()) + try: + seed = read_at(directory, "content-provider-adaptive-publication.json", 4096) + value = validate_ready(read_at(directory, PREFIX + "-ready.json", 4096), seed) + require(seed["created_unix_seconds"] <= int(time.time()) < seed["expires_unix_seconds"] + and 0 <= time.monotonic_ns() - value["ready_ns"] < 30_000_000_000, + "stale ready receipt or expired original publication") + return value + finally: + os.close(directory) + + +def build_evidence(work): + work = Path(work) + require(work.is_absolute() and work.resolve() == work, "unsafe fixture root") + directory = CONTROLLER["directory"](work, os.geteuid()) + try: + evidence = dict(schema_version=1, kind="content-provider-pressure-smoke", success=True, scope=dict(SCOPE)) + for key, filename in (("publication", "content-provider-adaptive-publication.json"), + ("expected_peers", "a01-expected-peers.json"), + ("layout", "content-provider-adaptive-layout.json"), + ("original_selected_route", "content-provider-adaptive-live-selection.json"), + ("control_filter", "content-provider-adaptive-control-filter.json")): + evidence[key] = read_at(directory, filename) + for key, suffix in (("selected_before", "selection"), ("selected_route", "live-selection"), + ("status_before", "status-before"), ("status_after", "status-after"), ("fetch", "fetch"), + ("output", "object"), ("ready", "ready"), ("controller", "controller"), + ("resource_ready", "resource-ready"), ("worker_budget", "worker-budget"), + ("continuation", "continuation"), ("cleanup", "cleanup")): + evidence[key] = read_at(directory, f"{PREFIX}-{suffix}.json") + evidence["privacy"] = {role: read_at(directory, f"{PREFIX}-privacy-{role}.json") for role in ROLES} + evidence["control_underlay"] = dict(capture=read_at(directory, f"{PREFIX}-control-privacy.json"), + routes={node: {direction: read_at(directory, f"content-provider-adaptive-control-{node}-{direction}.json") + for direction in ("out", "back")} for node in NODES}) + evidence["providers"] = {node: {operation: read_at(directory, f"{PREFIX}-{node}-{operation}.json") + for operation in ("reset", "serve", "status", "stop")} for node in NODES} + evidence["qdisc"] = {phase: read_at(directory, f"{PREFIX}-qdisc-{phase}.json") + for phase in ("profile", "before", "limited", "after")} + return validate(evidence) + finally: + os.close(directory) + + +build = build_evidence + + +def main(argv): + try: + if len(argv) == 3 and argv[1] == "ready": + ready(Path(argv[2])) + elif len(argv) == 4 and argv[1] == "evidence": + value = build_evidence(Path(argv[2])) + BUDGET["write_private"](Path(argv[3]), value, MAX_EVIDENCE, exclusive=True) + else: + raise ValueError("usage: pressure-smoke.py ready WORK | evidence WORK OUTPUT") + return 0 + except (ValueError, KeyError, TypeError, OSError, UnicodeError, RecursionError): + print("CONTENT_PROVIDER_PRESSURE_EVIDENCE_INVALID", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/integration/content-provider-pressure-smoke.sh b/tests/integration/content-provider-pressure-smoke.sh new file mode 100644 index 000000000..6b2e82742 --- /dev/null +++ b/tests/integration/content-provider-pressure-smoke.sh @@ -0,0 +1,260 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only +# Sourced only by the disposable content-provider KVM topology. +# shellcheck disable=SC2154,SC2034 + +content_provider_pressure_node() { + case $1 in + relay3) pp_ns=$R3; pp_dev=r3x; pp_owned=${CONTENT_PROVIDER_PRESSURE_R3_OWNED:-no} ;; + relay4) pp_ns=$R4; pp_dev=r4x; pp_owned=${CONTENT_PROVIDER_PRESSURE_R4_OWNED:-no} ;; + relay5) pp_ns=$R5; pp_dev=r5x; pp_owned=${CONTENT_PROVIDER_PRESSURE_R5_OWNED:-no} ;; + *) return 1 ;; + esac +} + +content_provider_pressure_snapshot() { + case $1 in before|limited|after) ;; *) return 1 ;; esac + for pp_node in relay3 relay4 relay5; do + content_provider_pressure_node "$pp_node" || return 1 + ip netns exec "$pp_ns" tc -j qdisc show dev "$pp_dev" \ + >"$WORK/.content-provider-pressure-$pp_node-$1.json" || return 1 + done + jq -n --slurpfile r3 "$WORK/.content-provider-pressure-relay3-$1.json" \ + --slurpfile r4 "$WORK/.content-provider-pressure-relay4-$1.json" \ + --slurpfile r5 "$WORK/.content-provider-pressure-relay5-$1.json" \ + '{relay3:$r3[0],relay4:$r4[0],relay5:$r5[0]}' \ + >"$WORK/content-provider-pressure-qdisc-$1.json" +} + +content_provider_pressure_unshape() { + pp_unshape_result=0 + for pp_node in relay3 relay4 relay5; do + content_provider_pressure_node "$pp_node" || return 1 + [ "$pp_owned" = yes ] || continue + # Never remove an unrelated qdisc, even while unwinding a failed trial. + if ! ip netns exec "$pp_ns" tc -j qdisc show dev "$pp_dev" \ + >"$WORK/.content-provider-pressure-$pp_node-final.json" \ + || ! jq -e 'length == 1 and .[0].kind == "tbf" and .[0].root == true + and .[0].handle == "884:" and .[0].options.rate == 250000' \ + "$WORK/.content-provider-pressure-$pp_node-final.json" >/dev/null; then + pp_unshape_result=1 + continue + fi + printf '%s\n' "Removing only owned TBF 884: from disposable namespace $pp_ns interface $pp_dev." + if ! ip netns exec "$pp_ns" tc qdisc del dev "$pp_dev" root handle 884:; then + pp_unshape_result=1 + continue + fi + case $pp_node in + relay3) CONTENT_PROVIDER_PRESSURE_R3_OWNED=no ;; + relay4) CONTENT_PROVIDER_PRESSURE_R4_OWNED=no ;; + relay5) CONTENT_PROVIDER_PRESSURE_R5_OWNED=no ;; + esac + done + return "$pp_unshape_result" +} + +content_provider_pressure_cleanup() { + # These are our unreaped timeout supervisors, never PID files from an agent. + # timeout forwards TERM to its own process group and bounds forced drain. + if [ -n "${CONTENT_PROVIDER_PRESSURE_FETCH_PID:-}" ]; then + kill -TERM "$CONTENT_PROVIDER_PRESSURE_FETCH_PID" 2>/dev/null || true + wait "$CONTENT_PROVIDER_PRESSURE_FETCH_PID" 2>/dev/null || true + CONTENT_PROVIDER_PRESSURE_FETCH_PID= + fi + if [ -n "${CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID:-}" ]; then + kill -TERM "$CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID" 2>/dev/null || true + wait "$CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID" 2>/dev/null || true + CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID= + fi + content_provider_pressure_unshape +} + +content_provider_pressure_shape() { + # Controller readiness already checked the exact disposable KVM/root boundary. + # Check all three links before making any change; never replace a preexisting qdisc. + pp_parent_ns=$(readlink /proc/self/ns/net) || return 1 + for pp_node in relay3 relay4 relay5; do + content_provider_pressure_node "$pp_node" || return 1 + [ "$pp_owned" = no ] || return 1 + pp_child_ns=$(ip netns exec "$pp_ns" readlink /proc/self/ns/net) || return 1 + [ -n "$pp_child_ns" ] && [ "$pp_child_ns" != "$pp_parent_ns" ] || return 1 + done + content_provider_pressure_snapshot before || return 1 + jq -e 'keys == ["relay3","relay4","relay5"] and + all(.[]; length == 1 and .[0].kind == "noqueue" and .[0].root == true)' \ + "$WORK/content-provider-pressure-qdisc-before.json" >/dev/null || return 1 + for pp_node in relay3 relay4 relay5; do + content_provider_pressure_node "$pp_node" || return 1 + printf '%s\n' "Adding fixed TBF 884: rate 2mbit burst 128kb latency 250ms to disposable namespace $pp_ns interface $pp_dev; remove before route teardown." + ip netns exec "$pp_ns" tc qdisc add dev "$pp_dev" root handle 884: \ + tbf rate 2mbit burst 128kb latency 250ms || return 1 + case $pp_node in + relay3) CONTENT_PROVIDER_PRESSURE_R3_OWNED=yes ;; + relay4) CONTENT_PROVIDER_PRESSURE_R4_OWNED=yes ;; + relay5) CONTENT_PROVIDER_PRESSURE_R5_OWNED=yes ;; + esac + done + content_provider_pressure_snapshot limited || return 1 + jq -e 'all(.[]; length == 1 and .[0].kind == "tbf" and .[0].root == true + and .[0].handle == "884:" and .[0].options.rate == 250000)' \ + "$WORK/content-provider-pressure-qdisc-limited.json" >/dev/null || return 1 + jq -n '{profile:"fixed-provider-egress-2mbit", + devices:{relay3:"r3x",relay4:"r4x",relay5:"r5x"},handle:"884:", + rate_bits_per_second:2000000,burst_bytes:131072,queue_latency_ms:250}' \ + >"$WORK/content-provider-pressure-qdisc-profile.json" +} + +content_provider_pressure_run() { + PHASE=content-provider-pressure-setup + pp_prefix=content-provider-pressure + pp_client=$WORK/state-client/content-adaptive-pressure + if [ -e "$pp_client" ] || [ -L "$pp_client" ] \ + || [ -n "${CONTENT_PROVIDER_PRESSURE_FETCH_PID:-}" ] \ + || [ -n "${CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID:-}" ]; then + fail CONTENT_PROVIDER_PRESSURE_NOT_COLD + fi + install -d -o "$AGENT_UID" -g "$AGENT_GID" -m 0700 "$pp_client" + # Use the original native publication and route, not the HTTPS index identities. + cp -- "$WORK/content-provider-adaptive-live-selection.json" "$WORK/$pp_prefix-selection.json" + for pp_provider in relay3 relay4 relay5; do + case $pp_provider in + relay3) pp_address=48.164.4.1; pp_hostname=provider-c.volparossa.test ;; + relay4) pp_address=49.165.5.1; pp_hostname=provider-a.volparossa.test ;; + relay5) pp_address=50.166.6.1; pp_hostname=provider-b.volparossa.test ;; + esac + "$binary_directory/volparossa" --control-socket "$WORK/runtime-$pp_provider/control/agent.sock" \ + content stop >"$WORK/$pp_prefix-$pp_provider-reset.json" \ + || fail CONTENT_PROVIDER_PRESSURE_PROVIDER_RESET_FAILED + "$binary_directory/volparossa" --control-socket "$WORK/runtime-$pp_provider/control/agent.sock" \ + content serve --manifest "$adaptive_manifest" \ + --publisher-key "$adaptive_publisher" --cache "$WORK/state-$pp_provider/content-adaptive/cache" \ + --bind "$pp_address:18080" --advertised-hostname "$pp_hostname" \ + >"$WORK/$pp_prefix-$pp_provider-serve.json" \ + || fail CONTENT_PROVIDER_PRESSURE_PROVIDER_SERVE_FAILED + "$binary_directory/volparossa" --control-socket "$WORK/runtime-$pp_provider/control/agent.sock" \ + content status >"$WORK/$pp_prefix-$pp_provider-status.json" \ + || fail CONTENT_PROVIDER_PRESSURE_PROVIDER_STATUS_FAILED + jq -e '.serving == true and .publications == 1 and .replication_enabled == false' \ + "$WORK/$pp_prefix-$pp_provider-status.json" >/dev/null \ + || fail CONTENT_PROVIDER_PRESSURE_PROVIDER_NOT_READY + done + "$binary_directory/volparossa" --control-socket "$WORK/runtime-client/control/agent.sock" \ + content status >"$WORK/$pp_prefix-status-before.json" \ + || fail CONTENT_PROVIDER_PRESSURE_STATUS_FAILED + # Repeat the positive mount control and all provider-store denials for this phase. + nsenter --target "$adaptive_pid" --mount setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" \ + --clear-groups --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$adaptive_manifest" || fail CONTENT_PROVIDER_PRESSURE_MOUNT_UNAVAILABLE + for pp_private in "$WORK/state-relay3" "$WORK/state-relay4" "$WORK/state-relay5" "$adaptive_root"; do + if nsenter --target "$adaptive_pid" --mount setpriv --reuid="$AGENT_UID" --regid="$AGENT_GID" \ + --clear-groups --inh-caps=-all --ambient-caps=-all --bounding-set=-all --no-new-privs \ + -- test -r "$pp_private"; then fail CONTENT_PROVIDER_PRESSURE_LOCAL_SHORTCUT; fi + done + PHASE=content-provider-pressure-resource-ready + python3 -B "$source_directory/tests/integration/content-provider-resource-ready.py" "$WORK" pressure \ + || fail CONTENT_PROVIDER_PRESSURE_RESOURCE_UNAVAILABLE + capture_product_logs + pp_baseline_ms=$(client_log_baseline_ms) || fail CONTENT_PROVIDER_PRESSURE_BASELINE_UNAVAILABLE + start_privacy_observers "$pp_prefix-privacy" || fail CONTENT_PROVIDER_PRESSURE_CAPTURE_FAILED + content_provider_adaptive_start_control_observer "$pp_prefix-control-privacy" \ + || fail CONTENT_PROVIDER_PRESSURE_CONTROL_FAILED + PHASE=content-provider-pressure-observer + timeout --signal=TERM --kill-after=5s 45s python3 -B \ + "$WORK/bin/content-provider-pressure.py" observe "$WORK" \ + >"$WORK/$pp_prefix-controller.log" 2>&1 & + CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID=$! + pp_ready_attempt=0 + while [ "$pp_ready_attempt" -lt 50 ]; do + if [ -e "$WORK/$pp_prefix-ready.json" ]; then break; fi + kill -0 "$CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID" 2>/dev/null \ + || fail CONTENT_PROVIDER_PRESSURE_OBSERVER_EXITED + sleep 0.1 + pp_ready_attempt=$((pp_ready_attempt + 1)) + done + python3 -B "$source_directory/tests/integration/content-provider-pressure-smoke.py" ready "$WORK" \ + || fail CONTENT_PROVIDER_PRESSURE_OBSERVER_NOT_READY + content_provider_pressure_shape || fail CONTENT_PROVIDER_PRESSURE_QDISC_FAILED + + PHASE=content-provider-pressure-fetch + content_provider_worker_budget_capture mark pressure + pp_started_mono=$(python3 -B -c 'import time; print(time.monotonic_ns())') + pp_started_unix=$(date +%s%3N) + # Ordinary CLI: unchanged production admission/PSI, deadlines and continuation. + timeout --signal=TERM --kill-after=5s 120s "$binary_directory/volparossa" \ + --control-socket "$WORK/runtime-client/control/agent.sock" \ + content fetch --manifest "$adaptive_manifest" --publisher-key "$adaptive_publisher" \ + --cache "$pp_client/cache" --output "$pp_client/object.bin" \ + >"$WORK/$pp_prefix-fetch.json" 2>"$WORK/$pp_prefix-fetch.err" & + CONTENT_PROVIDER_PRESSURE_FETCH_PID=$! + pp_fetch_status=0 + wait "$CONTENT_PROVIDER_PRESSURE_FETCH_PID" || pp_fetch_status=$? + CONTENT_PROVIDER_PRESSURE_FETCH_PID= + pp_completed_mono=$(python3 -B -c 'import time; print(time.monotonic_ns())') + pp_completed_unix=$(date +%s%3N) + if [ "$pp_fetch_status" -eq 0 ]; then + content_provider_worker_budget_capture finish pressure + else + content_provider_worker_budget_capture pending pressure + fail CONTENT_PROVIDER_PRESSURE_FETCH_FAILED + fi + pp_controller_status=0 + wait "$CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID" || pp_controller_status=$? + CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID= + [ "$pp_controller_status" -eq 0 ] || fail CONTENT_PROVIDER_PRESSURE_NOT_OBSERVED + benchmark_capture_paths "$pp_prefix-live" mptcp || fail CONTENT_PROVIDER_PRESSURE_PATHS_FAILED + jq -e --arg context "$adaptive_context" '.route_context_id == $context' \ + "$WORK/$pp_prefix-live-selection.json" >/dev/null || fail CONTENT_PROVIDER_PRESSURE_ROUTE_CHANGED + "$binary_directory/volparossa" --control-socket "$WORK/runtime-client/control/agent.sock" \ + content status >"$WORK/$pp_prefix-status-after.json" || fail CONTENT_PROVIDER_PRESSURE_STATUS_FAILED + stop_privacy_observers || fail CONTENT_PROVIDER_PRESSURE_CAPTURE_NOT_DRAINED + content_provider_stop_control_observer || fail CONTENT_PROVIDER_PRESSURE_CONTROL_NOT_DRAINED + capture_product_logs + pp_queries=0 + pp_offers=0 + for pp_node in relay0 relay1 relay2 relay3 relay4 relay5; do + pp_queries=$((pp_queries + $(awk -v baseline="$pp_baseline_ms" \ + '$1 > baseline && $3 == "event=CONTENT_PROVIDER_QUERY_STARTED" {n++} END {print n+0}' "$WORK/logs-$pp_node.txt"))) + pp_offers=$((pp_offers + $(awk -v baseline="$pp_baseline_ms" \ + '$1 > baseline && $3 == "event=CONTENT_PROVIDER_OFFER_VERIFIED" {n++} END {print n+0}' "$WORK/logs-$pp_node.txt"))) + done + pp_discovered=$(awk -v baseline="$pp_baseline_ms" \ + '$1 > baseline && $3 == "event=CONTENT_DISCOVERY_COMPLETED" {n++} END {print n+0}' "$WORK/logs-client.txt") + if [ -L "$pp_client/object.bin" ] \ + || [ "$(stat -Lc '%a:%u:%g:%h' "$pp_client/object.bin")" != "600:$AGENT_UID:$AGENT_GID:1" ]; then + fail CONTENT_PROVIDER_PRESSURE_OUTPUT_UNSAFE + fi + jq -n --arg sha256 "$(sha256sum "$pp_client/object.bin" | awk '{print $1}')" \ + --argjson bytes "$(stat -Lc '%s' "$pp_client/object.bin")" \ + --argjson baseline "$pp_baseline_ms" --argjson queries "$pp_queries" \ + --argjson offers "$pp_offers" --argjson discovered "$pp_discovered" \ + --argjson started_mono "$pp_started_mono" --argjson completed_mono "$pp_completed_mono" \ + --argjson started_unix "$pp_started_unix" --argjson completed_unix "$pp_completed_unix" \ + '{sha256:$sha256,bytes:$bytes,client_cache_initially_absent:true, + client_mount_positive_control:true,client_mount_cannot_read_replica_stores:true, + publisher_process_exited_before_fetch:true,event_baseline_unix_ms:$baseline, + generic_dht_queries:$queries,authenticated_upstream_offers:$offers, + client_forwarded_discovery:$discovered,started_monotonic_ns:$started_mono, + completed_monotonic_ns:$completed_mono,started_unix_ms:$started_unix,completed_unix_ms:$completed_unix}' \ + >"$WORK/$pp_prefix-object.json" + PHASE=content-provider-pressure-cleanup + for pp_provider in relay3 relay4 relay5; do + "$binary_directory/volparossa" --control-socket "$WORK/runtime-$pp_provider/control/agent.sock" \ + content stop >"$WORK/$pp_prefix-$pp_provider-stop.json" \ + || fail CONTENT_PROVIDER_PRESSURE_PROVIDER_STOP_FAILED + done + content_provider_pressure_cleanup || fail CONTENT_PROVIDER_PRESSURE_CLEANUP_FAILED + content_provider_pressure_snapshot after || fail CONTENT_PROVIDER_PRESSURE_QDISC_CLEANUP_FAILED + jq -e 'all(.[]; length == 1 and .[0].kind == "noqueue" and .[0].root == true)' \ + "$WORK/$pp_prefix-qdisc-after.json" >/dev/null || fail CONTENT_PROVIDER_PRESSURE_QDISC_NOT_RESTORED + # As with native adaptive cache, whole-topology teardown owns cache removal. + rm -- "$WORK/state-client/content-adaptive-pressure/object.bin" + if [ -e "$pp_client/object.bin" ] || [ -L "$pp_client/object.bin" ]; then + fail CONTENT_PROVIDER_PRESSURE_OUTPUT_CLEANUP_FAILED + fi + jq -n '{controller_reaped:true,fetch_reaped:true,owned_output_removed:true, + cache_teardown_deferred_to_topology:true,qdisc_restored:true,route_retained:true}' \ + >"$WORK/$pp_prefix-cleanup.json" + python3 -B "$source_directory/tests/integration/content-provider-pressure-smoke.py" \ + evidence "$WORK" "$WORK/$pp_prefix-evidence.json" || fail CONTENT_PROVIDER_PRESSURE_EVIDENCE_INVALID +} diff --git a/tests/integration/content-provider-resource-ready.py b/tests/integration/content-provider-resource-ready.py index 42e064c1a..1a0a7e77a 100644 --- a/tests/integration/content-provider-resource-ready.py +++ b/tests/integration/content-provider-resource-ready.py @@ -26,7 +26,8 @@ "fd_capacity_checked": False, "cgroup_capacity_checked": False, "worker_capacity_proven": False, "production_policy_overridden": False} OUTPUTS = {"native": "content-provider-adaptive-resource-ready.json", - "https": "content-provider-adaptive-https-resource-ready.json"} + "https": "content-provider-adaptive-https-resource-ready.json", + "pressure": "content-provider-pressure-resource-ready.json"} def require(condition): diff --git a/tests/integration/content-provider-smoke.py b/tests/integration/content-provider-smoke.py index a261d301c..3e9bb38af 100644 --- a/tests/integration/content-provider-smoke.py +++ b/tests/integration/content-provider-smoke.py @@ -164,7 +164,16 @@ def build_named_publication(work, nodes): serves={node: read(work / f"content-provider-named-{node}-serve.json") for node in nodes}) -def validate_transfer(evidence, *, historical=False, historical_late=False): +def route_identity(selection): + # Fresh selection captures legitimately update these path observations. + # Keep every other field, including any policy binding, exact; never infer + # a new route or accept a different Exit/Relay/path from matching counters. + return dict(selection, paths=[{key: value for key, value in path.items() + if key not in {"reported_bytes", "smoothed_rtt_us", "state"}} + for path in selection["paths"]]) + + +def validate_transfer(evidence, *, historical=False, historical_late=False, historical_pressure=False): publication, output, fetch = evidence["publication"], evidence["output"], evidence["fetch"] require(publication["bytes"] == output["bytes"] == fetch["bytes"] == BYTES and publication["object_sha256"] == output["sha256"] == SHA @@ -305,6 +314,20 @@ def validate_transfer(evidence, *, historical=False, historical_late=False): and adaptive["publication"]["publisher_hex"] != publication["publisher_hex"] and adaptive["publication"]["manifest_id"] != publication["manifest_id"], "adaptive proof substituted the topology or reused the earlier route/publication") + if historical_pressure: + require("pressure" not in evidence, + "historical transfer must not claim the later pressure phase") + else: + pressure = evidence.get("pressure") + require(type(pressure) is dict and pressure.get("success") is True, + "current transfer requires actual pressure-retirement and continuation proof") + pressure_check = runpy.run_path(str(Path(__file__).with_name("content-provider-pressure-smoke.py"))) + pressure_check["validate"](pressure) + require(pressure["expected_peers"] == peers + and pressure["publication"] == adaptive["publication"] + and route_identity(pressure["selected_route"]) == route_identity(adaptive["selected_route"]) + and pressure["selected_before"] == adaptive["selected_route"], + "pressure proof substituted the adaptive publication, topology or protected route") # Historical component evidence predates this phase; only current aggregates # acceptance requires it. If supplied, it must validate and bind to this run. if "late_provider_recovery" in evidence: @@ -360,6 +383,14 @@ def build_late_provider_recovery(work): return rebuilt +def build_pressure(work): + checker = runpy.run_path(str(Path(__file__).with_name("content-provider-pressure-smoke.py"))) + rebuilt = checker["build_evidence"](work) + require(rebuilt == read(work / "content-provider-pressure-evidence.json"), + "pressure component differs from its raw transfer, pressure, continuation or cleanup evidence") + return rebuilt + + def build_evidence(work): layout = read(work / "content-provider-layout.json") evidence = dict(success=True, publication=read(work / "content-provider-publication.json"), @@ -379,6 +410,7 @@ def build_evidence(work): site_publication=build_site_publication(work), filter_snapshot=build_filter_snapshot(work), adaptive_workers=build_adaptive_workers(work), + pressure=build_pressure(work), late_provider_recovery=build_late_provider_recovery(work), requester_cancellation=read(work / "content-provider-cancellation.json"), expected_peers=read(work / "a01-expected-peers.json"), @@ -430,9 +462,17 @@ def validate_control_underlay(evidence, control_node, provider_nodes): def validate_report(report, revision, *, historical=False): - require(type(report.get("schema_version")) is int - and report["schema_version"] in ((1, 2) if historical else (3,)), + require(type(historical) is bool and type(report.get("schema_version")) is int + and report["schema_version"] in ((1, 2, 3) if historical else (4,)), "wrong aggregate report version for current or historical validation") + if historical: + require("pressure" not in report and "pressure" not in report["transfer"], + "historical aggregate must not claim the later pressure phase") + else: + require(report.get("pressure") is True + and type(report["transfer"].get("pressure")) is dict + and report["transfer"]["pressure"].get("success") is True, + "current aggregate report requires the separate pressure proof") legacy_without_late = historical and report["schema_version"] == 1 if legacy_without_late: require("late_provider_recovery" not in report @@ -459,7 +499,8 @@ def validate_report(report, revision, *, historical=False): and re.fullmatch(r"[0-9a-f]{64}", report["host_state"]["before_sha256"]) and all(report[flag] is False for flag in SCOPE), "wrong source, cleanup, host state or scope") validate_transfer(report["transfer"], historical=legacy_without_late, - historical_late=historical and report["schema_version"] == 2) + historical_late=historical and report["schema_version"] == 2, + historical_pressure=historical) if __name__ == "__main__": diff --git a/tests/integration/content-provider-smoke.sh b/tests/integration/content-provider-smoke.sh index 0c7be8225..e6b5fab32 100644 --- a/tests/integration/content-provider-smoke.sh +++ b/tests/integration/content-provider-smoke.sh @@ -132,7 +132,7 @@ content_provider_adaptive_control_underlay() { content_provider_adaptive_start_control_observer() { provider_control_prefix=$1 - case $provider_control_prefix in content-provider-adaptive-*) ;; *) return 1 ;; esac + case $provider_control_prefix in content-provider-adaptive-*|content-provider-pressure-control-privacy) ;; *) return 1 ;; esac [ -z "$PROVIDER_CONTROL_PID" ] || return 1 ip netns exec "$provider_control_ns" python3 "$WORK/bin/privacy-observer.py" \ content-control "$WORK/$provider_control_prefix.json" \ @@ -341,6 +341,23 @@ content_provider_finalize_report() { # large native plus HTTPS capture bundle in a single operating-system argument. for provider_artifact in "$WORK"/content-provider-*.json "$WORK"/content-provider-*.txt \ "$WORK"/content-provider-*.log "$WORK"/content-provider-*.out "$WORK"/content-provider-*.err; do + case $(basename -- "$provider_artifact") in + content-provider-pressure-*) + case $(basename -- "$provider_artifact") in + content-provider-pressure-worker-budget.json|content-provider-pressure-continuation.json|content-provider-pressure-resource-ready.json|\ + content-provider-pressure-evidence.json|content-provider-pressure-ready.json|content-provider-pressure-controller.json|\ + content-provider-pressure-cleanup.json|content-provider-pressure-object.json|content-provider-pressure-fetch.json|\ + content-provider-pressure-selection.json|content-provider-pressure-live-selection.json|\ + content-provider-pressure-status-before.json|content-provider-pressure-status-after.json|\ + content-provider-pressure-qdisc-before.json|content-provider-pressure-qdisc-limited.json|content-provider-pressure-qdisc-after.json|\ + content-provider-pressure-qdisc-profile.json|content-provider-pressure-control-privacy.json|\ + content-provider-pressure-privacy-client.json|content-provider-pressure-privacy-relay[012].json|content-provider-pressure-privacy-exit.json|\ + content-provider-pressure-relay[345]-reset.json|content-provider-pressure-relay[345]-serve.json|\ + content-provider-pressure-relay[345]-status.json|content-provider-pressure-relay[345]-stop.json) ;; + *) continue ;; + esac + ;; + esac [ ! -f "$provider_artifact" ] || [ -L "$provider_artifact" ] || \ install -o "$OUTPUT_UID" -g "$OUTPUT_GID" -m 0600 "$provider_artifact" \ "$output_directory/$(basename -- "$provider_artifact")" @@ -356,17 +373,18 @@ content_provider_finalize_report() { --argjson complete "$CLEANUP_COMPLETE" \ --argjson remaining "$REMAINING_OWNED_OBJECTS" ' $evidence_input[0] as $evidence | $host_input[0] as $host | - {schema_version:3,report_kind:"volparossa-native-content-providers", + {schema_version:4,report_kind:"volparossa-native-content-providers", source_revision:$revision,run_id:$run_id,phase:$phase, success:($status == 0 and $evidence.success == true and $evidence.ordinary_publication.success == true and $evidence.site_publication.success == true and $evidence.filter_snapshot.success == true and - $evidence.adaptive_workers.success == true and $evidence.late_provider_recovery.success == true and + $evidence.adaptive_workers.success == true and $evidence.pressure.success == true and + $evidence.late_provider_recovery.success == true and $evidence.requester_cancellation.success == true and $complete and $remaining == 0 and $host.unchanged == true),transfer:$evidence, runner_exit_status:$status,observed_blocker:(if $blocker == "NONE" then null else $blocker end), cleanup:{complete:$complete,remaining_owned_objects:$remaining},host_state:($host | del(.acceptance_id)), - scope:"explicit native publication and cooperative-origin HTTPS for the same object, normal named static-site HTTP assets, exact public filter export and networkless cache reuse, two policy-authorized providers via generic DHT/control-relay discovery and protected MPTCP/TLS/WireGuard, complete peers and missing origin ranges; separate late complementary-provider native recovery; named requester cancellation and agent shutdown during pending bootstrap", + scope:"explicit native publication and cooperative-origin HTTPS for the same object, normal named static-site HTTP assets, exact public filter export and networkless cache reuse, two policy-authorized providers via generic DHT/control-relay discovery and protected MPTCP/TLS/WireGuard, complete peers and missing origin ranges; three-provider admission under real bounded CPU pressure and single-worker continuation; separate late complementary-provider native recovery; named requester cancellation and agent shutdown during pending bootstrap", general_nat_reachability_claimed:false,full_c02_claimed:false, explicit_origin_authenticated_https:($evidence.https.success == true), normal_user_publication:($evidence.ordinary_publication.success == true), @@ -374,6 +392,7 @@ content_provider_finalize_report() { native_static_site:($evidence.site_publication.success == true), public_filter_snapshot:($evidence.filter_snapshot.success == true), adaptive_provider_workers:($evidence.adaptive_workers.success == true), + pressure:($evidence.pressure.success == true), late_provider_recovery:($evidence.late_provider_recovery.success == true), requester_cancellation:($evidence.requester_cancellation.success == true), browser_integration_claimed:false,arbitrary_https_integration_claimed:false, diff --git a/tests/integration/content-provider-worker-budget.py b/tests/integration/content-provider-worker-budget.py index 43ede7945..2245b5eda 100644 --- a/tests/integration/content-provider-worker-budget.py +++ b/tests/integration/content-provider-worker-budget.py @@ -21,7 +21,10 @@ CODE = "CONTENT_WORKER_BUDGET_SUMMARY" CONTINUATION_TARGET = "volparossa_agent::content::parallel::continuation" CONTINUATION_CODE = "CONTENT_PROVIDER_CONTINUATION_SUMMARY" -PHASES = ("native", "https") +PHASES = ("native", "https", "pressure") +OUTPUT_PREFIXES = {"native": "content-provider-adaptive", + "https": "content-provider-adaptive-https", + "pressure": "content-provider-pressure"} MAX_INPUT = 8 * 1024 * 1024 MAX_LINE = 16384 MAX_LINES = 8192 @@ -171,15 +174,13 @@ def write_private(path, value, maximum, *, exclusive=False): def paths(work, phase): require(phase in PHASES) - suffix = "" if phase == "native" else "-https" return (work / ".content-worker-budget-{}.part".format(phase), - work / f"content-provider-adaptive{suffix}-worker-budget.json") + work / f"{OUTPUT_PREFIXES[phase]}-worker-budget.json") def continuation_path(work, phase): require(phase in PHASES) - suffix = "" if phase == "native" else "-https" - return work / f"content-provider-adaptive{suffix}-continuation.json" + return work / f"{OUTPUT_PREFIXES[phase]}-continuation.json" def mark(work, phase): diff --git a/tests/integration/kvm-alpha-topology.sh b/tests/integration/kvm-alpha-topology.sh index c80b5eb5e..90a3c3f8b 100755 --- a/tests/integration/kvm-alpha-topology.sh +++ b/tests/integration/kvm-alpha-topology.sh @@ -1361,6 +1361,7 @@ if [ "$scenario" = content-provider ]; then content-provider-filter-smoke.sh content-provider-filter-smoke.py \ content-provider-adaptive-smoke.sh content-provider-adaptive-smoke.py content-provider-worker-budget.py \ content-provider-resource-ready.py \ + content-provider-pressure-smoke.sh content-provider-pressure-smoke.py content-provider-pressure.py \ content-provider-late-recovery.sh content-provider-late-recovery.py content-cancellation-smoke.py; do if [ ! -f "$source_directory/tests/integration/$provider_fixture" ] \ || [ -L "$source_directory/tests/integration/$provider_fixture" ]; then @@ -2157,6 +2158,10 @@ cleanup() { [ "$FINALIZED" = no ] || exit "$original_status" FINALIZED=yes trap - EXIT HUP INT TERM + # Stop the owned CPU workers/fetch and restore fixture shaping before teardown. + if [ "$scenario" = content-provider ] && command -v content_provider_pressure_cleanup >/dev/null 2>&1; then + content_provider_pressure_cleanup || original_status=1 + fi # Release the fixture gate and reap its exact owned fetch before services or namespaces. if [ "$scenario" = content-provider ] && command -v content_provider_late_recovery_cleanup >/dev/null 2>&1; then content_provider_late_recovery_cleanup || original_status=1 @@ -2701,6 +2706,8 @@ if [ "$scenario" = content-provider ]; then . "$source_directory/tests/integration/content-provider-filter-smoke.sh" # shellcheck source=tests/integration/content-provider-adaptive-smoke.sh . "$source_directory/tests/integration/content-provider-adaptive-smoke.sh" + # shellcheck source=tests/integration/content-provider-pressure-smoke.sh + . "$source_directory/tests/integration/content-provider-pressure-smoke.sh" # shellcheck source=tests/integration/content-provider-late-recovery.sh . "$source_directory/tests/integration/content-provider-late-recovery.sh" if [ -n "$weather_client_bundle" ]; then @@ -2954,6 +2961,10 @@ if [ "$scenario" = content-provider ]; then install -o root -g root -m 0555 "$source_directory/tests/integration/$late_recovery_helper" \ "$WORK/bin/$late_recovery_helper" done + for pressure_helper in content-provider-pressure.py content-provider-resource-ready.py; do + install -o root -g root -m 0555 "$source_directory/tests/integration/$pressure_helper" \ + "$WORK/bin/$pressure_helper" + done fi if [ "$scenario" = content-custody ] || [ "$scenario" = agent-artifact ] || [ "$scenario" = agent-jobs ]; then for custody_script in content-custody-smoke.py content-retain-smoke.py content-provider-smoke.py content-provider-https-smoke.py content-network-smoke.py; do @@ -4736,6 +4747,7 @@ provider_timing_enabled = (content_provider_mode and role == "exit" and os.path.basename(output_path) in { "content-provider-privacy-exit.json", "content-provider-adaptive-privacy-exit.json", + "content-provider-pressure-privacy-exit.json", "content-provider-adaptive-https-privacy-exit.json"}) # Linux UAPI SO_TIMESTAMPNS_NEW reports __kernel_timespec, two signed 64-bit fields. # Capture packet arrival, not socket-drain time: fair queue draining can reorder interfaces. @@ -4743,6 +4755,7 @@ provider_timing_enabled = (content_provider_mode and role == "exit" SO_TIMESTAMPNS_NEW = 64 provider_payload_milestones = ([262144, 3932160] if os.path.basename(output_path) in {"content-provider-adaptive-privacy-exit.json", + "content-provider-pressure-privacy-exit.json", "content-provider-adaptive-https-privacy-exit.json"} else [65536, 983040]) provider_payload_timing = dict(enabled=provider_timing_enabled, clock="linux-so-timestampns-new", @@ -6226,6 +6239,8 @@ start_privacy_observers() { [ "$scenario" = content-provider ] || return 1 ;; content-provider-late-recovery-privacy) [ "$scenario" = content-provider ] || return 1 ;; + content-provider-pressure-privacy) + [ "$scenario" = content-provider ] || return 1 ;; content-provider-privacy|content-provider-adaptive-privacy|content-provider-adaptive-https-privacy|content-provider-https-complete-privacy|content-provider-https-missing-privacy|content-provider-https-baseline-privacy|content-provider-https-origin-only-privacy|content-provider-https-auto-privacy|content-provider-https-digest-origin-only-privacy|content-provider-https-digest-peers-first-privacy|content-provider-https-limited-origin-only-privacy|content-provider-https-limited-peers-first-privacy|content-provider-https-limited-auto-privacy|content-provider-https-checksum-origin-only-privacy|content-provider-https-checksum-peers-first-privacy|content-provider-https-checksum-mismatch-privacy|content-provider-user-privacy|content-provider-named-privacy|content-provider-site-privacy|content-provider-site-cache-only-privacy|content-provider-filter-cold-privacy|content-provider-filter-warm-privacy|weather-client-delivery-privacy) [ "$scenario" = content-provider ] || return 1 ;; content-message-publication-privacy) diff --git a/tests/integration/run-alpha-topology-vm.sh b/tests/integration/run-alpha-topology-vm.sh index f832954bb..16868c94f 100755 --- a/tests/integration/run-alpha-topology-vm.sh +++ b/tests/integration/run-alpha-topology-vm.sh @@ -1117,6 +1117,16 @@ WEATHER_PRODUCTS_FAILURE_NAMES = {f"weather-products-delivery-{suffix}.json" for WEATHER_PRODUCTS_HELPERS = ("weather-products-delivery.py", "weather-source-observer.mjs", *WEATHER_CLIENT_HELPERS) LATE_PROVIDER_NAMES = {f"content-provider-late-recovery-{suffix}.json" for suffix in ( "first-snapshot", "final-snapshot", "object", "run-cleanup", "cleanup", "evidence", "failure", "readiness")} +PRESSURE_PROVIDER_NAMES = {f"content-provider-pressure-{suffix}.json" for suffix in ( + "worker-budget", "continuation", "resource-ready", "evidence", "ready", "controller", "cleanup", + "object", "fetch", "selection", "live-selection", "status-before", "status-after", + "qdisc-before", "qdisc-limited", "qdisc-after", "qdisc-profile", "control-privacy")} +PRESSURE_PROVIDER_NAMES.update(f"content-provider-pressure-privacy-{node}.json" + for node in ("client", "relay0", "relay1", "relay2", "exit")) +PRESSURE_PROVIDER_NAMES.update(f"content-provider-pressure-{node}-{operation}.json" + for node in ("relay3", "relay4", "relay5") for operation in ("reset", "serve", "status", "stop")) +PRESSURE_PRIORITY_NAMES = {f"content-provider-pressure-{suffix}.json" for suffix in ( + "worker-budget", "continuation", "resource-ready", "evidence", "ready", "controller", "cleanup", "object", "fetch")} def joined_client_helper(work, names, entrypoint): @@ -1357,6 +1367,12 @@ def collect(home, opt, revision, scenario, guest_status, if len(roots) == 5: break if scenario == "content-provider": + # Keep the aggregate outcome/terminal phase even if later diagnostics + # fill the unchanged emergency file budget. + for root, label in roots: + if root.is_dir() and not root.is_symlink(): + candidates.extend((root / name, f"{label}/{name}") + for name in ("content-provider-smoke.json", "current-phase")) # Prioritize already validated exact Browser receipts before generic # diagnostics consume the existing file-count/byte limits. This is # recovery evidence only, even when an inner receipt says passed=true. @@ -1378,6 +1394,12 @@ def collect(home, opt, revision, scenario, guest_status, for root, label in roots: if root.is_dir() and not root.is_symlink(): candidates.extend((root / name, f"{label}/{name}") for name in sorted(LATE_PROVIDER_NAMES)) + # Preserve the closed pressure outcome first; the remaining exact raw + # evidence is best effort under the same 64-file/8MiB emergency bounds. + for names in (PRESSURE_PRIORITY_NAMES, PRESSURE_PROVIDER_NAMES - PRESSURE_PRIORITY_NAMES): + for root, label in roots: + if root.is_dir() and not root.is_symlink(): + candidates.extend((root / name, f"{label}/{name}") for name in sorted(names)) for root, label in roots: if root.is_symlink() or not root.is_dir(): continue @@ -1528,7 +1550,9 @@ def collect(home, opt, revision, scenario, guest_status, candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("content-*"))[:48] if re.fullmatch(r"content-[a-z0-9-]+\.(json|txt|log|err|out)", path.name) and (not path.name.startswith("content-provider-late-recovery-") - or scenario == "content-provider" and path.name in LATE_PROVIDER_NAMES)) + or scenario == "content-provider" and path.name in LATE_PROVIDER_NAMES) + and (not path.name.startswith("content-provider-pressure-") + or scenario == "content-provider" and path.name in PRESSURE_PROVIDER_NAMES)) candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("agent-public-task-*"))[:32] if re.fullmatch(r"agent-public-task-[a-z0-9-]+\.(json|err|log)", path.name)) candidates.extend((path, f"{label}/{path.name}") for path in sorted(root.glob("agent-public-document-*"))[:32] @@ -1546,8 +1570,13 @@ def collect(home, opt, revision, scenario, guest_status, if observed is None: continue data, original_size = observed - if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES | WEATHER_PRODUCTS_FAILURE_NAMES | LATE_PROVIDER_NAMES and original_size > len(data): + if path.name in BROWSER_FILTER_NAMES | WEATHER_CLIENT_NAMES | WEATHER_PRODUCTS_NAMES | WEATHER_PRODUCTS_FAILURE_NAMES | LATE_PROVIDER_NAMES | PRESSURE_PROVIDER_NAMES and original_size > len(data): continue # Preserve complete validated original bytes, never a JSON tail. + if path.name in PRESSURE_PROVIDER_NAMES: + try: + json.loads(data) + except (ValueError, UnicodeError, RecursionError): + continue # Interrupted/malformed pressure JSON is not raw-log authority. if scenario == "browser-network" and path.name in browser_names and original_size > len(data): # A truncated JSON tail is not reconstructible structured evidence. continue diff --git a/tests/integration/test-content-provider-pressure-shell.py b/tests/integration/test-content-provider-pressure-shell.py new file mode 100644 index 000000000..d328e4660 --- /dev/null +++ b/tests/integration/test-content-provider-pressure-shell.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert execution of pressure shell control flow; no network, load or signals.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +SCRIPT = HERE / "content-provider-pressure-smoke.sh" + +# Keep the production jq expressions and functions, but replace every boundary +# that could change networking or signal/wait for a real process. The shell +# mock accepts only the fixture's exact namespace/interface/operation tuples. +MOCKS = r''' +R3=ns3; R4=ns4; R5=ns5 +SHAPED3=no; SHAPED4=no; SHAPED5=no +FAIL_ADD=; FAIL_DELETE=; WRONG_KIND=; SAME_NAMESPACE= +readlink() { + [ "$*" = /proc/self/ns/net ] || return 93 + printf 'net:[parent]\n' +} +ip() { + printf 'ip %s\n' "$*" >> "$CALLS" + [ "$1" = netns ] && [ "$2" = exec ] || return 93 + mock_ns=$3 + case $mock_ns in + ns3) mock_dev=r3x; mock_shaped=$SHAPED3 ;; + ns4) mock_dev=r4x; mock_shaped=$SHAPED4 ;; + ns5) mock_dev=r5x; mock_shaped=$SHAPED5 ;; + *) return 93 ;; + esac + shift 3 + if [ "$*" = 'readlink /proc/self/ns/net' ]; then + if [ "$mock_ns" = "$SAME_NAMESPACE" ]; then + printf 'net:[parent]\n' + else + printf 'net:[%s]\n' "$mock_ns" + fi + elif [ "$*" = "tc -j qdisc show dev $mock_dev" ]; then + if [ "$mock_ns" = "$WRONG_KIND" ]; then + printf '[{"kind":"fq_codel","handle":"884:","root":true}]\n' + elif [ "$mock_shaped" = yes ]; then + printf '[{"kind":"tbf","handle":"884:","root":true,"options":{"rate":250000}}]\n' + else + printf '[{"kind":"noqueue","handle":"0:","root":true}]\n' + fi + elif [ "$*" = "tc qdisc add dev $mock_dev root handle 884: tbf rate 2mbit burst 128kb latency 250ms" ]; then + [ "$mock_ns" != "$FAIL_ADD" ] || return 1 + case $mock_ns in ns3) SHAPED3=yes ;; ns4) SHAPED4=yes ;; ns5) SHAPED5=yes ;; esac + elif [ "$*" = "tc qdisc del dev $mock_dev root handle 884:" ]; then + [ "$mock_ns" != "$FAIL_DELETE" ] || return 1 + case $mock_ns in ns3) SHAPED3=no ;; ns4) SHAPED4=no ;; ns5) SHAPED5=no ;; esac + else + return 93 + fi +} +kill() { printf 'kill %s\n' "$*" >> "$CALLS"; return 1; } +wait() { printf 'wait %s\n' "$*" >> "$CALLS"; return 127; } +''' + + +class PressureShell(unittest.TestCase): + def shell(self, body): + with tempfile.TemporaryDirectory(prefix="pressure-shell-") as directory: + work = Path(directory) + calls = work / "calls" + calls.touch(mode=0o600) + script = 'WORK=$1\nCALLS=$2\n. "$3"\n' + MOCKS + '\n' + body + result = subprocess.run(["sh", "-eu", "-c", script, "pressure-shell", + str(work), str(calls), str(SCRIPT)], + capture_output=True, timeout=3, check=False) + recorded = calls.read_text().splitlines() + self.assertEqual(result.returncode, 0, result.stderr.decode()) + return result.stdout.decode(), recorded + + def test_shape_fixed_profile_and_restore_exact_owned_devices(self): + _, calls = self.shell(''' +content_provider_pressure_shape +[ "$CONTENT_PROVIDER_PRESSURE_R3_OWNED:$CONTENT_PROVIDER_PRESSURE_R4_OWNED:$CONTENT_PROVIDER_PRESSURE_R5_OWNED" = yes:yes:yes ] +jq -e '.rate_bits_per_second == 2000000 and .burst_bytes == 131072 and .queue_latency_ms == 250' "$WORK/content-provider-pressure-qdisc-profile.json" >/dev/null +content_provider_pressure_cleanup +[ "$CONTENT_PROVIDER_PRESSURE_R3_OWNED:$CONTENT_PROVIDER_PRESSURE_R4_OWNED:$CONTENT_PROVIDER_PRESSURE_R5_OWNED" = no:no:no ] +content_provider_pressure_cleanup +''') + adds = [line for line in calls if "qdisc add" in line] + deletes = [line for line in calls if "qdisc del" in line] + self.assertEqual(len(adds), 3) + self.assertEqual(deletes, [f"ip netns exec ns{n} tc qdisc del dev r{n}x root handle 884:" + for n in (3, 4, 5)]) + + def test_partial_add_failure_restores_only_successfully_owned_qdisc(self): + _, calls = self.shell(''' +FAIL_ADD=ns4 +if content_provider_pressure_shape; then exit 90; fi +[ "$CONTENT_PROVIDER_PRESSURE_R3_OWNED" = yes ] +[ "${CONTENT_PROVIDER_PRESSURE_R4_OWNED:-no}:${CONTENT_PROVIDER_PRESSURE_R5_OWNED:-no}" = no:no ] +content_provider_pressure_cleanup +[ "$CONTENT_PROVIDER_PRESSURE_R3_OWNED" = no ] +''') + self.assertEqual([line for line in calls if "qdisc del" in line], + ["ip netns exec ns3 tc qdisc del dev r3x root handle 884:"]) + self.assertFalse(any("ns5 tc qdisc add" in line for line in calls)) + + def test_existing_qdisc_or_parent_namespace_prevents_all_mutations(self): + for setting in ("WRONG_KIND=ns5", "SAME_NAMESPACE=ns5", + "CONTENT_PROVIDER_PRESSURE_R5_OWNED=yes"): + with self.subTest(setting=setting): + _, calls = self.shell(setting + '\nif content_provider_pressure_shape; then exit 90; fi\n') + self.assertFalse(any("qdisc add" in line or "qdisc del" in line for line in calls)) + + def test_replaced_qdisc_is_not_deleted_and_other_owned_links_still_restore(self): + _, calls = self.shell(''' +content_provider_pressure_shape +WRONG_KIND=ns4 +if content_provider_pressure_cleanup; then exit 90; fi +[ "$CONTENT_PROVIDER_PRESSURE_R3_OWNED:$CONTENT_PROVIDER_PRESSURE_R4_OWNED:$CONTENT_PROVIDER_PRESSURE_R5_OWNED" = no:yes:no ] +''') + self.assertEqual([line for line in calls if "qdisc del" in line], + [f"ip netns exec ns{n} tc qdisc del dev r{n}x root handle 884:" for n in (3, 5)]) + + def test_failed_delete_keeps_owned_flag_and_remains_retryable(self): + _, calls = self.shell(''' +content_provider_pressure_shape +FAIL_DELETE=ns4 +if content_provider_pressure_cleanup; then exit 90; fi +[ "$CONTENT_PROVIDER_PRESSURE_R3_OWNED:$CONTENT_PROVIDER_PRESSURE_R4_OWNED:$CONTENT_PROVIDER_PRESSURE_R5_OWNED" = no:yes:no ] +FAIL_DELETE= +content_provider_pressure_cleanup +[ "$CONTENT_PROVIDER_PRESSURE_R4_OWNED" = no ] +''') + self.assertEqual([line for line in calls if "qdisc del" in line], + [f"ip netns exec ns{n} tc qdisc del dev r{n}x root handle 884:" for n in (3, 4, 5, 4)]) + + def test_cleanup_reaps_exact_supervisors_before_network_restore_even_on_failed_wait(self): + _, calls = self.shell(''' +content_provider_pressure_shape +CONTENT_PROVIDER_PRESSURE_FETCH_PID=730001 +CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID=730002 +content_provider_pressure_cleanup +[ -z "$CONTENT_PROVIDER_PRESSURE_FETCH_PID$CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID" ] +content_provider_pressure_cleanup +''') + lifecycle = [line for line in calls if line.startswith(("kill ", "wait ")) or "qdisc del" in line] + self.assertEqual(lifecycle, ["kill -TERM 730001", "wait 730001", "kill -TERM 730002", "wait 730002"] + + [f"ip netns exec ns{n} tc qdisc del dev r{n}x root handle 884:" for n in (3, 4, 5)]) + + def test_provider_original_manifest_and_diagnostic_deadlines_keep_phase_order(self): + source = SCRIPT.read_text() + flow = source.split("content_provider_pressure_run() {", 1)[1] + steps = [ + 'content stop >"$WORK/$pp_prefix-$pp_provider-reset.json"', + 'content serve --manifest "$adaptive_manifest"', + '"$WORK/bin/content-provider-pressure.py" observe "$WORK"', + 'ready "$WORK"', + 'content_provider_pressure_shape || fail', + 'content_provider_worker_budget_capture mark pressure', + 'content fetch --manifest "$adaptive_manifest"', + 'wait "$CONTENT_PROVIDER_PRESSURE_FETCH_PID"', + 'pp_completed_mono=', + 'content_provider_worker_budget_capture finish pressure', + 'content_provider_worker_budget_capture pending pressure', + 'wait "$CONTENT_PROVIDER_PRESSURE_CONTROLLER_PID"', + 'benchmark_capture_paths "$pp_prefix-live"', + 'content_provider_pressure_cleanup || fail', + 'evidence "$WORK"', + ] + offsets = [flow.index(step) for step in steps] + self.assertEqual(offsets, sorted(offsets)) + self.assertIn('timeout --signal=TERM --kill-after=5s 45s python3 -B', flow) + self.assertIn('timeout --signal=TERM --kill-after=5s 120s "$binary_directory/volparossa"', flow) + self.assertIn('content_provider_worker_budget_capture pending pressure\n fail CONTENT_PROVIDER_PRESSURE_FETCH_FAILED', flow) + self.assertNotIn('rm -r', source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-content-provider-pressure-smoke.py b/tests/integration/test-content-provider-pressure-smoke.py new file mode 100644 index 000000000..efabc8885 --- /dev/null +++ b/tests/integration/test-content-provider-pressure-smoke.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Pure closed-evidence tests; synthetic receipts never claim real pressure or networking.""" +import copy +import json +import os +from pathlib import Path +import runpy +import tempfile +import unittest +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +CHECK = runpy.run_path(str(HERE / "content-provider-pressure-smoke.py")) +ADAPTIVE = runpy.run_path(str(HERE / "test-content-provider-adaptive-smoke.py")) +CONTROLLER = runpy.run_path(str(HERE / "test-content-provider-pressure.py")) +RESOURCE = CHECK["RESOURCE"] + + +def fixture(): + original = ADAPTIVE["fixture"]() + controller, _ = CONTROLLER["run"]() + controller["publication"] = CHECK["publication"](original["publication"]) + budget = dict(schema_version=1, outcome="returned", initial_allowance=8, min_allowance=1, + max_allowance=8, min_memory_workers=8, min_fd_workers=64, peak_shared_held=3, + admitted_leases=5, denied_leases=0, peak_leases=3, active_leases=0, + causes={key: key == "cpu_pressure" for key in CHECK["BUDGET"]["CAUSES"]}) + continuation = dict(schema_version=1, outcome="returned", retired_candidates=2, + prepared_continuations=2, admitted_continuations=2, + returned_useful_continuations=2, returned_verified_chunks=36, + returned_verified_bytes=36*262144, initial_active_workers=0, + peak_active_workers=1, final_active_workers=0) + def diagnostic(kind, record): + return dict(schema_version=1, kind=kind, phase="pressure", phase_closed=True, + complete=True, status="observed", records=[record]) + selected = original["selected_route"] + output = dict(original["output"], started_monotonic_ns=controller["started_ns"], + completed_monotonic_ns=12_000_000_000, started_unix_ms=1_001_000, + completed_unix_ms=1_012_000) + providers = {node: dict(reset=dict(serving=False, publications=0), + serve=dict(serving=True, publications=1, replication_enabled=False), + status=dict(serving=True, publications=1, replication_enabled=False), + stop=dict(serving=False, publications=0)) for node in CHECK["NODES"]} + return dict(schema_version=1, kind="content-provider-pressure-smoke", success=True, + scope=dict(CHECK["SCOPE"]), publication=original["publication"], + expected_peers=original["expected_peers"], layout=original["layout"], + original_selected_route=copy.deepcopy(selected), selected_before=copy.deepcopy(selected), + selected_route=selected, status_before=original["status_before"], + status_after=original["status_after"], fetch=original["fetch"], output=output, + privacy=original["privacy"], control_underlay=original["control_underlay"], + control_filter=original["control_filter"], providers=providers, + ready=dict(schema_version=1, kind="content-provider-pressure-ready", cold_cache=True, + ready_ns=controller["started_ns"]-1, publication=dict(controller["publication"])), + controller=controller, + resource_ready=dict(schema_version=1, kind="content-provider-psi-readiness", phase="pressure", + scope=dict(RESOURCE["SCOPE"]), status="ready", deadline_ms=30000, + sample_interval_ms=1000, required_consecutive=3, samples=3, consecutive=3, + elapsed_ms=2000, last_observation=dict.fromkeys(RESOURCE["THRESHOLDS"], "below_threshold")), + worker_budget=diagnostic("content-worker-budget-diagnostic", budget), + continuation=diagnostic("content-provider-continuation-diagnostic", continuation), + qdisc=dict(profile=copy.deepcopy(CHECK["QDISC_PROFILE"]), + before={node: [dict(kind="noqueue", handle="0:", root=True, bytes=0)] for node in CHECK["NODES"]}, + limited={node: [dict(kind="tbf", handle="884:", root=True, + options=dict(rate=250000, burst=131072, lat=250000))] for node in CHECK["NODES"]}, + after={node: [dict(kind="noqueue", handle="0:", root=True, bytes=100)] for node in CHECK["NODES"]}), + cleanup=dict(CHECK["CLEANUP"])) + + +def raw_files(evidence): + prefix = CHECK["PREFIX"] + files = {name: evidence[key] for key, name in ( + ("publication", "content-provider-adaptive-publication.json"), + ("expected_peers", "a01-expected-peers.json"), ("layout", "content-provider-adaptive-layout.json"), + ("original_selected_route", "content-provider-adaptive-live-selection.json"), + ("control_filter", "content-provider-adaptive-control-filter.json"))} + files.update({f"{prefix}-{suffix}.json": evidence[key] for key, suffix in ( + ("selected_before", "selection"), ("selected_route", "live-selection"), + ("status_before", "status-before"), ("status_after", "status-after"), ("fetch", "fetch"), + ("output", "object"), ("ready", "ready"), ("controller", "controller"), + ("resource_ready", "resource-ready"), ("worker_budget", "worker-budget"), + ("continuation", "continuation"), ("cleanup", "cleanup"))}) + for node, operations in evidence["providers"].items(): + for operation, value in operations.items(): + files[f"{prefix}-{node}-{operation}.json"] = value + files.update({f"{prefix}-privacy-{role}.json": value for role, value in evidence["privacy"].items()}) + files[f"{prefix}-control-privacy.json"] = evidence["control_underlay"]["capture"] + for node, directions in evidence["control_underlay"]["routes"].items(): + for direction, value in directions.items(): + files[f"content-provider-adaptive-control-{node}-{direction}.json"] = value + for phase, value in evidence["qdisc"].items(): + files[f"{prefix}-qdisc-{phase}.json"] = value + return files + + +def write_files(work, evidence): + for name, value in raw_files(evidence).items(): + with (work/name).open("x", encoding="ascii") as output: + json.dump(value, output) + (work/name).chmod(0o600) + + +class PressureEvidence(unittest.TestCase): + def rejected(self, change): + evidence = fixture() + change(evidence) + with self.assertRaises((ValueError, KeyError, TypeError)): + CHECK["validate"](evidence) + + def test_complete_pressure_and_serial_useful_continuation(self): + evidence = fixture() + self.assertEqual(CHECK["validate"](evidence), evidence) + self.assertEqual(evidence["controller"]["last_observation"]["cpu"], "pressured") + self.assertFalse(evidence["scope"]["pressure_recovery_claimed"]) + self.assertFalse(evidence["scope"]["owner_goodput_claimed"]) + + def test_contraction_requires_three_actual_leases_and_available_resources(self): + for key, value in (("initial_allowance", 2), ("peak_leases", 2), ("min_allowance", 2), + ("min_memory_workers", 2), ("min_fd_workers", None), ("active_leases", 1), + ("outcome", "interrupted"), ("initial_allowance", True)): + with self.subTest(key=key): + self.rejected(lambda e: e["worker_budget"]["records"][0].__setitem__(key, value)) + + def test_cpu_only_pressure_and_observation_are_mandatory(self): + for cause in CHECK["BUDGET"]["CAUSES"]: + self.rejected(lambda e: e["worker_budget"]["records"][0]["causes"].__setitem__( + cause, cause != "cpu_pressure")) + for key, value in (("success", False), ("cold_begin", False), ("publication", {})): + self.rejected(lambda e: e["controller"].__setitem__(key, value)) + + def test_no_progress_or_undrained_parallel_continuation_is_not_completion(self): + for key, value in (("returned_useful_continuations", 0), ("retired_candidates", 4), + ("returned_verified_chunks", 0), ("returned_verified_bytes", 1), + ("initial_active_workers", 1), ("final_active_workers", 1), + ("peak_active_workers", 2), ("outcome", "failed")): + with self.subTest(key=key): + self.rejected(lambda e: e["continuation"]["records"][0].__setitem__(key, value)) + + def test_complete_phase_bounded_diagnostic_not_other_batch_required(self): + for name in ("worker_budget", "continuation"): + for key, value in (("phase", "native"), ("complete", False), ("phase_closed", False), + ("status", "missing"), ("records", [])): + self.rejected(lambda e: e[name].__setitem__(key, value)) + self.rejected(lambda e: e[name]["records"].append(copy.deepcopy(e[name]["records"][0]))) + self.rejected(lambda e: e["resource_ready"].__setitem__("phase", "native")) + + def test_ready_seed_and_bounded_real_cpu_event_are_bound_to_fetch(self): + self.rejected(lambda e: e["ready"].__setitem__("cold_cache", False)) + self.rejected(lambda e: e["ready"]["publication"].__setitem__("manifest_id", "f"*64)) + self.rejected(lambda e: e["ready"].__setitem__("ready_ns", 20_000_000_000)) + self.rejected(lambda e: e["output"].__setitem__("started_monotonic_ns", 3_000_000_000)) + self.rejected(lambda e: e["output"].__setitem__("completed_monotonic_ns", 999_999_999)) + self.rejected(lambda e: e["output"].__setitem__("completed_monotonic_ns", 122_000_000_000)) + self.rejected(lambda e: e["output"].__setitem__("completed_unix_ms", 4_600_000)) + + def test_full_correct_object_and_three_useful_providers_still_required(self): + for key, value in (("bytes", CHECK["BYTES"]-1), ("chunks", 59), ("providers_used", 2), + ("peer_bytes", 0), ("origin_body_bytes", 1), ("origin_authenticated", True)): + self.rejected(lambda e: e["fetch"].__setitem__(key, value)) + self.rejected(lambda e: e["output"].__setitem__("sha256", "0"*64)) + self.rejected(lambda e: e["output"].__setitem__("client_cache_initially_absent", False)) + self.rejected(lambda e: e["privacy"]["exit"]["provider_application"]["relay5"].__setitem__( + "response_payload_bytes", 0)) + + def test_original_route_and_existing_privacy_gates_remain_mandatory(self): + self.rejected(lambda e: e["selected_route"].__setitem__("route_context_id", "c"*32)) + self.rejected(lambda e: e["selected_before"].__setitem__("new", "unexpected")) + self.rejected(lambda e: e["privacy"]["client"].__setitem__("direct_client_exit_packets", 1)) + self.rejected(lambda e: e["privacy"]["relay0"].__setitem__("client_leg_wireguard_data_datagrams", 0)) + self.rejected(lambda e: e["privacy"]["exit"].__setitem__("packet_socket_drops", 1)) + evidence = fixture() + evidence["selected_route"]["paths"][0]["reported_bytes"] = 999 + CHECK["validate"](evidence) # Counters may grow; ownership and path identities may not. + + def test_exact_owned_qdiscs_restore_config_not_packet_statistics(self): + self.rejected(lambda e: e["qdisc"]["limited"]["relay3"][0]["options"].__setitem__("rate", 500000)) + self.rejected(lambda e: e["qdisc"]["limited"]["relay4"][0].__setitem__("handle", "885:")) + self.rejected(lambda e: e["qdisc"]["after"]["relay5"][0].__setitem__("kind", "tbf")) + self.rejected(lambda e: e["qdisc"]["profile"].__setitem__("queue_latency_ms", 500)) + evidence = fixture() + for row in evidence["qdisc"]["limited"].values(): + del row[0]["options"]["lat"] + del row[0]["options"]["burst"] + CHECK["validate"](evidence) # Kernel may omit fields; exact commanded profile is retained. + + def test_provider_stop_and_owned_cleanup_are_independent_gates(self): + for key in CHECK["CLEANUP"]: + self.rejected(lambda e: e["cleanup"].__setitem__(key, False)) + self.rejected(lambda e: e["providers"]["relay4"]["stop"].__setitem__("serving", True)) + self.rejected(lambda e: e["providers"]["relay5"]["serve"].__setitem__("publications", 2)) + + def test_scope_and_unknown_fields_fail_closed(self): + self.rejected(lambda e: e.__setitem__("unknown", "unexpected")) + self.rejected(lambda e: e["scope"].__setitem__("owner_goodput_claimed", True)) + self.rejected(lambda e: e["publication"].__setitem__("unknown", "unexpected")) + self.rejected(lambda e: e["fetch"].__setitem__("origin_body_bytes", False)) + self.rejected(lambda e: e["selected_route"].__setitem__("source", "substituted")) + + def test_builder_reconstructs_only_raw_phase_receipts(self): + evidence = fixture() + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + write_files(work, evidence) + self.assertEqual(CHECK["build_evidence"](work), evidence) + target = work/(CHECK["PREFIX"] + "-worker-budget.json") + target.chmod(0o666) + with self.assertRaises(ValueError): + CHECK["build_evidence"](work) + + def test_ready_rejects_duplicate_keys_oversize_and_unpublished_partial(self): + evidence = fixture() + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + write_files(work, evidence) + target = work/(CHECK["PREFIX"] + "-ready.json") + target.write_text('{"schema_version":1,"schema_version":1}', encoding="ascii") + with self.assertRaises(ValueError): + CHECK["ready"](work) + target.write_text(" " * 4097, encoding="ascii") + with self.assertRaises(ValueError): + CHECK["ready"](work) + target.rename(work/("." + target.name + ".part")) + with self.assertRaises(FileNotFoundError): + CHECK["ready"](work) + + def test_ready_checks_same_descriptor_modes_links_lifetime_and_atomic_name(self): + evidence = fixture() + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + write_files(work, evidence) + with patch.object(CHECK["time"], "time", return_value=1001), \ + patch.object(CHECK["time"], "monotonic_ns", return_value=1_500_000_000): + self.assertEqual(CHECK["ready"](work), evidence["ready"]) + target = work/(CHECK["PREFIX"] + "-ready.json") + original = work/"original-ready.json" + target.rename(original) + target.symlink_to(original.name) + with self.assertRaises(OSError): + CHECK["ready"](work) + target.unlink() + os.link(original, target) + with self.assertRaises(ValueError): + CHECK["ready"](work) + target.unlink() + with self.assertRaises(FileNotFoundError): + CHECK["ready"](work) + original.rename(target) + with patch.object(CHECK["time"], "time", return_value=4600), \ + patch.object(CHECK["time"], "monotonic_ns", return_value=1_500_000_000): + with self.assertRaises(ValueError): + CHECK["ready"](work) + with patch.object(CHECK["time"], "time", return_value=1001), \ + patch.object(CHECK["time"], "monotonic_ns", return_value=31_000_000_000): + with self.assertRaises(ValueError): + CHECK["ready"](work) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-content-provider-pressure-wiring.py b/tests/integration/test-content-provider-pressure-wiring.py new file mode 100644 index 000000000..c63201fc3 --- /dev/null +++ b/tests/integration/test-content-provider-pressure-wiring.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-only +"""Inert wiring/export checks: no VM, stress worker, privileged command or network.""" +import json +from pathlib import Path +import re +import subprocess +import tarfile +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +TOPOLOGY = (HERE / "kvm-alpha-topology.sh").read_text() +RUNNER = (HERE / "run-alpha-topology-vm.sh").read_text() +SUFFIXES = ("worker-budget", "continuation", "resource-ready", "evidence", "ready", "controller", + "cleanup", "object", "fetch", "selection", "live-selection", "status-before", "status-after", + "qdisc-before", "qdisc-limited", "qdisc-after", "qdisc-profile", "control-privacy") +ROLES = ("client", "relay0", "relay1", "relay2", "exit") +NAMES = {f"content-provider-pressure-{suffix}.json" for suffix in SUFFIXES} +NAMES |= {f"content-provider-pressure-privacy-{role}.json" for role in ROLES} +NAMES |= {f"content-provider-pressure-{node}-{operation}.json" + for node in ("relay3", "relay4", "relay5") for operation in ("reset", "serve", "status", "stop")} + + +def shell(script, *args): + return subprocess.run(["sh", "-eu", "-c", script, "sh", *args], + capture_output=True, timeout=3, check=False) + + +def collector(): + embedded = RUNNER.split("<<'GUEST_DIAGNOSTICS_PYTHON'\n", 1)[1].split("\nGUEST_DIAGNOSTICS_PYTHON\n", 1)[0] + api = {"__name__": "inert_pressure_collector"} + exec(compile(embedded, "pressure-collector", "exec"), api) + return api + + +class PressureWiring(unittest.TestCase): + def test_presence_symlinks_source_and_staged_controller_closure(self): + preflight = "for provider_fixture in " + TOPOLOGY.split("for provider_fixture in ", 1)[1].split("\n done", 1)[0] + "\n done\n" + names = preflight.split("; do", 1)[0].removeprefix("for provider_fixture in ").replace("\\\n", " ").split() + pressure = ("content-provider-pressure-smoke.sh", "content-provider-pressure-smoke.py", "content-provider-pressure.py") + for name in pressure: + self.assertIn(name, names) + self.assertTrue((HERE / name).is_file()) + self.assertIn('. "$source_directory/tests/integration/content-provider-pressure-smoke.sh"', TOPOLOGY) + stage = TOPOLOGY.split("for pressure_helper in ", 1)[1].split("\n done", 1)[0] + self.assertEqual(stage.split("; do", 1)[0].split(), + ["content-provider-pressure.py", "content-provider-resource-ready.py"]) + self.assertIn('install -o root -g root -m 0555', stage) + self.assertIn('"$WORK/bin/$pressure_helper"', stage) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / "tests/integration" + target.mkdir(parents=True) + for name in names: + (target / name).write_text("fixture\n") + command = 'source_directory=$1\n' + preflight + self.assertEqual(shell(command, str(root)).returncode, 0) + for name in pressure: + path = target / name + path.unlink() + self.assertEqual(shell(command, str(root)).returncode, 69) + path.symlink_to(target / "content-provider-smoke.py") + self.assertEqual(shell(command, str(root)).returncode, 69) + path.unlink() + path.write_text("fixture\n") + + def test_early_cleanup_retains_failure_and_runs_remaining_hooks(self): + cleanup = TOPOLOGY.split("cleanup() {\n", 1)[1].split("\n}\n", 1)[0] + self.assertLess(cleanup.index("content_provider_pressure_cleanup"), cleanup.index("retire_unit")) + early = cleanup.split(' if [ "$scenario" = dns-cache ]', 1)[0] + for failure in (0, 1): + for scenario in ("content-provider", "content"): + prefix = ('scenario=$1\nPHASE=pressure-failed\nOBSERVED_BLOCKER=pressure-blocker\nFINALIZED=no\n' + f'content_provider_pressure_cleanup() {{ printf "pressure\\n"; return {failure}; }}\n' + 'content_provider_late_recovery_cleanup() { printf "late\\n"; }\n' + 'content_provider_cancellation_resume() { printf "cancel\\n"; }\n') + command = prefix + 'cleanup_prelude() {\n' + early + '\n}\ntrue\ncleanup_prelude\n' + command += 'printf "%s:%s:%s\\n" "$original_status" "$failure_phase" "$failure_reason"\n' + result = shell(command, scenario) + self.assertEqual(result.returncode, 0, result.stderr) + expected = f"pressure\nlate\ncancel\n{failure}" if scenario == "content-provider" else "0" + self.assertEqual(result.stdout.decode(), expected + ":pressure-failed:pressure-blocker\n") + # No phase was entered: an unavailable hook must not make cleanup fail. + command = ('scenario=content-provider\nPHASE=preflight\nOBSERVED_BLOCKER=none\nFINALIZED=no\n' + 'cleanup_prelude() {\n' + early + '\n}\ntrue\ncleanup_prelude\n') + self.assertEqual(shell(command).returncode, 0) + + def test_pressure_phase_runs_once_after_https_before_shared_teardown(self): + adaptive = (HERE / "content-provider-adaptive-smoke.sh").read_text() + matches = list(re.finditer(r"^ content_provider_pressure_run$", adaptive, re.M)) + self.assertEqual(len(matches), 1) + start = matches[0].start() + self.assertLess(adaptive.index("\n content_provider_adaptive_https_run\n"), start) + self.assertLess(start, adaptive.index("\n PHASE=content-provider-adaptive-cleanup\n")) + pressure = (HERE / "content-provider-pressure-smoke.sh").read_text() + self.assertIn('"$WORK/bin/content-provider-pressure.py" observe "$WORK"', pressure) + self.assertIn('"$source_directory/tests/integration/content-provider-pressure-smoke.py"', pressure) + for operation in ("mark", "finish", "pending"): + self.assertIn(f"content_provider_worker_budget_capture {operation} pressure", pressure) + + def test_privacy_registration_is_exact_and_timing_matches_adaptive(self): + registration = TOPOLOGY.split("start_privacy_observers() {\n", 1)[1].split(" set --\n", 1)[0] + command = 'registered() {\n' + registration + '}\nscenario=$1\nregistered "$2"\n' + for scenario in ("content-provider", "content", "content-https"): + for prefix in ("content-provider-pressure-privacy", "content-provider-pressure-other", "../pressure-privacy"): + self.assertEqual(shell(command, scenario, prefix).returncode == 0, + scenario == "content-provider" and prefix == "content-provider-pressure-privacy") + import os + code = "provider_timing_enabled = " + TOPOLOGY.split("provider_timing_enabled = ", 1)[1].split("provider_payload_timing =", 1)[0] + for role in ("exit", "client"): + for enabled in (True, False): + for name in ("adaptive", "adaptive-https", "pressure", "pressure-other"): + scope = dict(os=os, role=role, content_provider_mode=enabled, + output_path=f"/fixture/content-provider-{name}-privacy-exit.json") + exec(compile(code, "inert-timing-selection", "exec"), scope) + self.assertEqual(scope["provider_timing_enabled"], enabled and role == "exit" and name != "pressure-other") + if name != "pressure-other": + self.assertEqual(scope["provider_payload_milestones"], [262144, 3932160]) + + def test_workflows_run_pure_checks_and_current_not_historical_acceptance(self): + for name in ("quality.yml", "alpha-topology.yml"): + workflow = (ROOT / ".github/workflows" / name).read_text() + for test in ("pressure", "pressure-smoke", "pressure-wiring", "worker-budget", "resource-ready"): + self.assertIn(f"python3 -B tests/integration/test-content-provider-{test}.py", workflow) + self.assertIn("sh -n tests/integration/content-provider-pressure-smoke.sh", workflow) + workflow = (ROOT / ".github/workflows/alpha-topology.yml").read_text() + step = workflow.split("- name: Require actual native provider discovery and protected fetch\n", 1)[1].split("\n - name:", 1)[0] + self.assertIn('content-provider-smoke.py report "$report" "$GITHUB_SHA"', step) + self.assertNotIn("historical-report", step) + self.assertIn('test "$TOPOLOGY_EXIT_CODE" = 0', step) + + def test_closed_emergency_names_and_unchanged_caps(self): + api = collector() + self.assertEqual(api["PRESSURE_PROVIDER_NAMES"], NAMES) + self.assertEqual(api["PRESSURE_PRIORITY_NAMES"], {f"content-provider-pressure-{suffix}.json" + for suffix in ("worker-budget", "continuation", "resource-ready", "evidence", "ready", "controller", "cleanup", "object", "fetch")}) + self.assertEqual((api["FILE_LIMIT"], api["TOTAL_LIMIT"], api["FILE_COUNT_LIMIT"]), (131072, 8388608, 64)) + for phase in ("adaptive", "adaptive-https"): + for suffix in ("worker-budget", "continuation", "resource-ready"): + self.assertIn(f"content-provider-{phase}-{suffix}.json", api["SAFE_NAMES"]) + + def test_normal_and_emergency_pressure_export_allowlists_match(self): + source = (HERE / "content-provider-smoke.sh").read_text() + finalize = source.split("content_provider_finalize_report() {\n", 1)[1] + selection = ' case $(basename -- "$provider_artifact") in' + finalize.split( + ' case $(basename -- "$provider_artifact") in', 1)[1].split(' [ ! -f "$provider_artifact" ]', 1)[0] + command = 'for provider_artifact do\n' + selection + '\nprintf "%s\\n" "$provider_artifact"\ndone\n' + denied = {"content-provider-pressure-unknown.json", "content-provider-pressure-fetch.err", + "content-provider-pressure-controller.log", "content-provider-pressure-fetch.state", + "content-provider-pressure-privacy-relay3.json", "content-provider-pressure-relay2-serve.json", + "content-provider-pressure-privacy-bootstrap1.json"} + result = shell(command, *sorted(NAMES | denied)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(set(result.stdout.decode().splitlines()), NAMES) + + def test_emergency_priority_keeps_terminal_and_outcomes_across_duplicate_roots(self): + api = collector() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home, opt, proc, cgroups = (root / name for name in ("home", "opt", "proc", "cgroups")) + for path in (home, opt, proc, cgroups): path.mkdir() + work = opt / ("va." + "a" * 32 + ".ABC123") + work.mkdir() + published = home / "alpha-output" + published.mkdir() + for path in (published, work): + (path / "content-provider-smoke.json").write_text('{"success":false}') + (path / "current-phase").write_text("content-provider-pressure-fetch\n") + for name in NAMES | api["LATE_PROVIDER_NAMES"]: (path / name).write_text('{"observed":true}') + for index in range(60): (path / f"content-before-{index:02}.json").write_text("{}") + archive = api["collect"](home, opt, "b" * 40, "content-provider", 124, cgroups, proc) + with tarfile.open(archive, "r:gz") as bundle: + files = set(bundle.getnames()) + summary = json.load(bundle.extractfile("vm-incomplete.json")) + for label in ("published", "work-1"): + for name in api["PRESSURE_PRIORITY_NAMES"] | {"content-provider-smoke.json", "current-phase"}: + self.assertIn(f"{label}/{name}", files) + self.assertEqual(len(summary["diagnostics"]["files"]), 64) + self.assertTrue(summary["diagnostics"]["partial"]) + self.assertFalse(summary["success"]) + + def test_emergency_recovers_only_complete_exact_json_with_no_success_claim(self): + api = collector() + for scenario in ("content-provider", "content"): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home, opt, proc, cgroups = (root / name for name in ("home", "opt", "proc", "cgroups")) + for path in (home, opt, proc, cgroups): path.mkdir() + work = opt / ("va." + "a" * 32 + ".ABC123") + work.mkdir() + published = home / "alpha-output" + published.mkdir() + (published / "content-provider-smoke.json").write_text('{"success":false}') + (published / "current-phase").write_text("content-provider-pressure-fetch\n") + for index in range(60): (work / f"content-before-{index:02}.json").write_text("{}") + for name in NAMES | api["LATE_PROVIDER_NAMES"]: (work / name).write_text('{"observed":true}') + oversized = "content-provider-pressure-evidence.json" + (work / oversized).write_bytes(b" " * (api["FILE_LIMIT"] + 1)) + alias = "content-provider-pressure-cleanup.json" + (work / alias).unlink() + (work / alias).symlink_to(work / "content-before-00.json") + malformed = "content-provider-pressure-controller.json" + (work / malformed).write_text('{"interrupted":') + for name in ("content-provider-pressure-fetch.state", "content-provider-pressure-unknown.json", + "content-provider-pressure-fetch.err", ".content-provider-pressure-relay3-final.json"): + (work / name).write_text("PRIVATE RAW INPUT") + archive = api["collect"](home, opt, "b" * 40, scenario, 124, cgroups, proc) + with tarfile.open(archive, "r:gz") as bundle: + files = {member.name: bundle.extractfile(member).read() for member in bundle.getmembers()} + selected = {name.rsplit("/", 1)[-1] for name in files if "/content-provider-pressure-" in name} + self.assertEqual(selected, NAMES - {oversized, alias, malformed} if scenario == "content-provider" else set()) + if scenario == "content-provider": + self.assertIn("published/content-provider-smoke.json", files) + self.assertIn("published/current-phase", files) + self.assertFalse(any(b"PRIVATE" in value for value in files.values())) + summary = json.loads(files["vm-incomplete.json"]) + self.assertIs(summary["success"], False) + self.assertEqual(summary["cleanup"], {"complete": False, "verified": False}) + self.assertLessEqual(len(summary["diagnostics"]["files"]), 64) + self.assertLessEqual(summary["diagnostics"]["captured_bytes"], 8388608) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test-content-provider-resource-ready.py b/tests/integration/test-content-provider-resource-ready.py index ffed38200..41632f45f 100644 --- a/tests/integration/test-content-provider-resource-ready.py +++ b/tests/integration/test-content-provider-resource-ready.py @@ -72,6 +72,48 @@ def test_requires_three_consecutive_samples_a_second_apart(self): self.assertEqual(result["scope"], CHECK["SCOPE"]) self.assertFalse(result["scope"]["worker_capacity_proven"]) + def test_pressure_phase_keeps_identical_thresholds_timing_and_failure_policy(self): + outcomes = (([observation()] * 3, "ready", 3, 2000), + ([observation(), observation(cpu="unavailable")], "unavailable", 2, 1000), + ([observation(io="pressured")] * 30, "timeout", 30, 30000)) + for values, status, samples, elapsed in outcomes: + with self.subTest(status=status): + legacy = wait(values) + pressure = wait(values, phase="pressure") + self.assertEqual(pressure, dict(legacy, phase="pressure")) + self.assertEqual((pressure["status"], pressure["samples"], pressure["elapsed_ms"]), + (status, samples, elapsed)) + self.assertFalse(pressure["scope"]["worker_capacity_proven"]) + self.assertFalse(pressure["scope"]["production_policy_overridden"]) + + def test_pressure_file_is_separate_exclusive_and_phase_bound(self): + self.assertEqual(CHECK["OUTPUTS"], { + "native": "content-provider-adaptive-resource-ready.json", + "https": "content-provider-adaptive-https-resource-ready.json", + "pressure": "content-provider-pressure-resource-ready.json"}) + retained = {} + for phase in ("native", "https", "pressure"): + expected = wait(phase=phase) + self.assertTrue(CHECK["run"](self.work, phase, waiter=lambda _, v=expected: v)) + output = self.work / CHECK["OUTPUTS"][phase] + retained[output] = output.read_bytes() + self.assertEqual(json.loads(retained[output]), expected) + self.assertEqual(output.stat().st_mode & 0o777, 0o600) + self.assertLess(len(retained[output]), 2048) + with self.assertRaises(ValueError): + CHECK["run"](self.work, "pressure", waiter=lambda _: self.fail("must not sample twice")) + for output, raw in retained.items(): + self.assertEqual(output.read_bytes(), raw) + + def test_pressure_rejects_mismatched_receipt_and_unsafe_target_before_export(self): + target = self.work / CHECK["OUTPUTS"]["pressure"] + with self.assertRaises(ValueError): + CHECK["run"](self.work, "pressure", waiter=lambda _: wait(phase="native")) + self.assertFalse(target.exists()) + target.symlink_to(self.work / "absent") + with self.assertRaises(ValueError): + CHECK["run"](self.work, "pressure", waiter=lambda _: self.fail("unsafe pressure target")) + def test_pressure_resets_sequence_without_resetting_deadline(self): result = wait([observation(), observation(), observation(io="pressured"), observation(), observation(), observation()]) @@ -244,7 +286,7 @@ def test_shell_places_both_gates_after_setup_before_captures_and_retains_exports self.assertIn("|| fail CONTENT_PROVIDER_ADAPTIVE_HTTPS_RESOURCE_UNAVAILABLE", https) runner = (HERE / "run-alpha-topology-vm.sh").read_text() safe_names = runner[runner.index("SAFE_NAMES ="):runner.index("def ", runner.index("SAFE_NAMES ="))] - for name in CHECK["OUTPUTS"].values(): + for name in (CHECK["OUTPUTS"][phase] for phase in ("native", "https")): self.assertIn(f'"{name}"', safe_names) topology = (HERE / "kvm-alpha-topology.sh").read_text() provider_preflight = topology[topology.index("for provider_fixture in "):] diff --git a/tests/integration/test-content-provider-smoke.py b/tests/integration/test-content-provider-smoke.py index 11892df37..665b0dcf8 100644 --- a/tests/integration/test-content-provider-smoke.py +++ b/tests/integration/test-content-provider-smoke.py @@ -20,7 +20,7 @@ CHECK = runpy.run_path(str(HERE / "content-provider-smoke.py")) -def fixture(control_node="relay2", *, include_late=True): +def fixture(control_node="relay2", *, include_late=True, include_pressure=True): peers = {name: f"peer-{name}" for name in (*CHECK["ROLES"], "relay3", "relay4", "relay5")} provider_nodes = [node for node in CHECK["CANDIDATES"] if node != control_node][:2] paths = [dict(route_context_id="a" * 32, path_id=index + 1, @@ -93,6 +93,8 @@ def fixture(control_node="relay2", *, include_late=True): evidence["site_publication"] = runpy.run_path(str(HERE / "test-content-provider-site-smoke.py"))["fixture"](control_node) evidence["filter_snapshot"] = runpy.run_path(str(HERE / "test-content-provider-filter-smoke.py"))["fixture"](control_node) evidence["adaptive_workers"] = runpy.run_path(str(HERE / "test-content-provider-adaptive-smoke.py"))["fixture"]() + if include_pressure: + evidence["pressure"] = runpy.run_path(str(HERE / "test-content-provider-pressure-smoke.py"))["fixture"]() if include_late: evidence["late_provider_recovery"] = runpy.run_path( str(HERE / "test-content-provider-late-recovery.py"))["fixture"]() @@ -213,7 +215,8 @@ def test_late_phase_hooks_preflight_and_cleanup_cannot_be_skipped(self): self.assertIn("content_provider_adaptive_run\n content_provider_late_recovery_run\n" " content_provider_cancellation_run", runtime) self.assertIn("$evidence.late_provider_recovery.success == true", runtime) - self.assertIn("{schema_version:3,report_kind:", runtime) + self.assertIn("$evidence.pressure.success == true", runtime) + self.assertIn("{schema_version:4,report_kind:", runtime) preflight = source.split('for provider_fixture in ', 1)[1].split('; do', 1)[0] for name in ("content-provider-late-recovery.sh", "content-provider-late-recovery.py"): self.assertIn(name, preflight) @@ -222,20 +225,26 @@ def test_late_phase_hooks_preflight_and_cleanup_cannot_be_skipped(self): hook = "content_provider_late_recovery_cleanup || original_status=1" self.assertLess(cleanup.index(hook), cleanup.index('kill -TERM')) self.assertLess(cleanup.index(hook), cleanup.index('retire_unit')) + pressure_hook = "content_provider_pressure_cleanup || original_status=1" + self.assertLess(cleanup.index(pressure_hook), cleanup.index('kill -TERM')) + self.assertLess(cleanup.index(pressure_hook), cleanup.index('retire_unit')) early = cleanup.split(' if [ "$scenario" = dns-cache ]', 1)[0] # Execute the actual trap prelude with inert hooks: failure is retained, # cancellation cleanup still runs, and no service/network tool is called. script = ('set -eu\nscenario=content-provider\nPHASE=late-failure\nOBSERVED_BLOCKER=late-blocker\n' 'FINALIZED=no\ncontent_provider_late_recovery_cleanup() { printf "late\\n"; return "$1"; }\n') # The hook has no arguments; bake the synthetic result into its body. - for result in (0, 1): + for result, pressure_result in ((0, 0), (1, 0), (0, 1), (1, 1)): prefix = script.replace('return "$1"', f'return {result}') + prefix += ('content_provider_pressure_cleanup() { printf "pressure\\n"; ' + f'return {pressure_result}; }}\n') prefix += 'content_provider_cancellation_resume() { printf "cancel\\n"; }\n' body = prefix + 'cleanup_prelude() {\n' + early + '\n}\ntrue\ncleanup_prelude\n' body += 'printf "%s:%s:%s\\n" "$original_status" "$failure_phase" "$failure_reason"\n' checked = subprocess.run(["sh", "-eu", "-c", body], capture_output=True, timeout=2, check=False) self.assertEqual(checked.returncode, 0, checked.stderr) - self.assertEqual(checked.stdout.decode(), f"late\ncancel\n{result}:late-failure:late-blocker\n") + self.assertEqual(checked.stdout.decode(), + f"pressure\nlate\ncancel\n{max(result, pressure_result)}:late-failure:late-blocker\n") stage = source.split('for late_recovery_helper in ', 1)[1].split('\n done', 1)[0] for name in ("content-provider-late-recovery.py", "content-provider-https-smoke.py", "content-network-smoke.py"): self.assertIn(name, stage) @@ -313,6 +322,34 @@ def test_adaptive_workers_are_required_and_bound_to_real_component_evidence(self with self.assertRaises((ValueError, KeyError)): CHECK["validate_transfer"](changed) + def test_pressure_is_separate_required_and_bound_to_existing_adaptive_proof(self): + evidence = fixture() + CHECK["validate_transfer"](evidence) + changed_statistics = copy.deepcopy(evidence) + changed_statistics["pressure"]["selected_route"]["paths"][0].update( + reported_bytes=123456, smoothed_rtt_us=456, state=1) + CHECK["validate_transfer"](changed_statistics) + for mutate in ( + lambda value: value.pop("pressure"), + lambda value: value.update(pressure=None), + lambda value: value["pressure"].update(success=False), + lambda value: value["pressure"]["expected_peers"].update(client="different-client"), + lambda value: value["pressure"]["publication"].update(manifest_id="f" * 64), + lambda value: value["pressure"]["selected_route"].update(route_context_id="f" * 32), + lambda value: value["pressure"]["selected_route"]["paths"][0].update(exit_peer_id="different-exit"), + lambda value: value["pressure"]["selected_route"].update(policy_hash="different-policy"), + lambda value: value["pressure"]["scope"].update(owner_goodput_claimed=True), + lambda value: value["adaptive_workers"].update(provider_payload_overlap_ns=0), + ): + changed = copy.deepcopy(evidence) + mutate(changed) + with self.assertRaises((ValueError, KeyError)): + CHECK["validate_transfer"](changed) + old = fixture(include_pressure=False) + CHECK["validate_transfer"](old, historical_pressure=True) + with self.assertRaises(ValueError): CHECK["validate_transfer"](old) + with self.assertRaises(ValueError): CHECK["validate_transfer"](evidence, historical_pressure=True) + def test_site_proof_is_required_and_bound_to_parent_topology_and_exact_bundle(self): evidence = fixture() CHECK["validate_transfer"](evidence) @@ -481,6 +518,7 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): "site_publication": "site-evidence", "filter_snapshot": "filter-evidence", "adaptive_workers": "adaptive-evidence", + "pressure": "pressure-evidence", "late_provider_recovery": "late-recovery-evidence", "requester_cancellation": "cancellation", } @@ -495,6 +533,12 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): if name in files: self.assertEqual(files[name], value, "component fixture cannot replace earlier proof inputs") files[name] = value + pressure_raw = runpy.run_path(str(HERE / "test-content-provider-pressure-smoke.py"))["raw_files"]( + evidence["pressure"]) + for name, value in pressure_raw.items(): + if name in files: + self.assertEqual(files[name], value, "pressure cannot replace earlier adaptive proof inputs") + files[name] = value late_raw = runpy.run_path(str(HERE / "test-content-provider-late-recovery.py"))["raw_files"]( evidence["late_provider_recovery"]) for name, value in late_raw.items(): @@ -522,6 +566,7 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): work = Path(directory) for name, value in files.items(): (work / name).write_text(value if isinstance(value, str) else json.dumps(value), encoding="ascii") + (work / name).chmod(0o600) self.assertEqual(CHECK["build_evidence"](work), evidence) # The site must be rebuilt from raw receipts/captures, not accepted by its summary flag. site_report = work / "content-provider-site-evidence.json" @@ -548,6 +593,24 @@ def test_actual_kernel_route_array_files_are_read_and_bound(self): with self.assertRaises(ValueError): CHECK["build_evidence"](work) adaptive_report.write_text(json.dumps(evidence["adaptive_workers"]), encoding="ascii") + pressure_report = work / "content-provider-pressure-evidence.json" + pressure_report.unlink() + with self.assertRaises(OSError): + CHECK["build_evidence"](work) + substituted = copy.deepcopy(evidence["pressure"]) + substituted["success"] = False + pressure_report.write_text(json.dumps(substituted), encoding="ascii") + with self.assertRaises(ValueError): + CHECK["build_evidence"](work) + pressure_report.write_text(json.dumps(evidence["pressure"]), encoding="ascii") + controller = work / "content-provider-pressure-controller.json" + original_controller = controller.read_text(encoding="ascii") + tampered = json.loads(original_controller) + tampered["verified_bytes"] = 0 + controller.write_text(json.dumps(tampered), encoding="ascii") + with self.assertRaises(ValueError): + CHECK["build_evidence"](work) + controller.write_text(original_controller, encoding="ascii") late_report = work / "content-provider-late-recovery-evidence.json" late_report.unlink() with self.assertRaises(OSError): @@ -590,10 +653,10 @@ def test_current_control_relay_is_excluded_without_changing_roles(self): self.assertEqual(fixture("relay4")["layout"]["provider_nodes"], ["relay5", "relay3"]) def test_exact_scoped_report(self): - report = dict(schema_version=3, report_kind="volparossa-native-content-providers", source_revision="a" * 40, + report = dict(schema_version=4, report_kind="volparossa-native-content-providers", source_revision="a" * 40, explicit_origin_authenticated_https=True, normal_user_publication=True, native_name_retrieval=True, native_static_site=True, public_filter_snapshot=True, - adaptive_provider_workers=True, late_provider_recovery=True, requester_cancellation=True, + adaptive_provider_workers=True, pressure=True, late_provider_recovery=True, requester_cancellation=True, success=True, runner_exit_status=0, cleanup=dict(complete=True, remaining_owned_objects=0), host_state=dict(unchanged=True, before_sha256="b" * 64, after_sha256="b" * 64), transfer=fixture(), **{name: False for name in CHECK["SCOPE"]}) @@ -605,10 +668,15 @@ def test_exact_scoped_report(self): lambda item: item.update(native_static_site=False), lambda item: item.update(public_filter_snapshot=False), lambda item: item.update(adaptive_provider_workers=False), + lambda item: item.update(pressure=False), + lambda item: item.pop("pressure"), + lambda item: item["transfer"].pop("pressure"), + lambda item: item["transfer"]["pressure"].update(success=False), lambda item: item.update(late_provider_recovery=False), lambda item: item.pop("late_provider_recovery"), lambda item: item["transfer"].pop("late_provider_recovery"), lambda item: item.update(schema_version=1), + lambda item: item.update(schema_version=3), lambda item: item.update(schema_version=2.0), lambda item: item.pop("schema_version"), lambda item: item.update(requester_cancellation=False), @@ -622,7 +690,19 @@ def test_exact_scoped_report(self): mutate(wrong) with self.assertRaises(ValueError): CHECK["validate_report"](wrong, "a" * 40) - historical = copy.deepcopy(report) + older_current = copy.deepcopy(report) + older_current["schema_version"] = 3 + older_current.pop("pressure") + older_current["transfer"].pop("pressure") + CHECK["validate_report"](older_current, "a" * 40, historical=True) + with self.assertRaises(ValueError): CHECK["validate_report"](older_current, "a" * 40) + for mutate in (lambda value: value.update(pressure=True), + lambda value: value["transfer"].update(pressure=report["transfer"]["pressure"])): + retroactive = copy.deepcopy(older_current) + mutate(retroactive) + with self.assertRaises(ValueError): + CHECK["validate_report"](retroactive, "a" * 40, historical=True) + historical = copy.deepcopy(older_current) historical["schema_version"] = 1 historical.pop("late_provider_recovery") historical["transfer"].pop("late_provider_recovery") @@ -634,7 +714,7 @@ def test_exact_scoped_report(self): CHECK["validate_report"](historical, "a" * 40) with self.assertRaises(ValueError): CHECK["validate_report"](report, "a" * 40, historical=True) - older_late = copy.deepcopy(report) + older_late = copy.deepcopy(older_current) older_late["schema_version"] = 2 older_late["transfer"]["late_provider_recovery"]["schema_version"] = 1 older_late["transfer"]["late_provider_recovery"].pop("readiness") diff --git a/tests/integration/test-content-provider-worker-budget.py b/tests/integration/test-content-provider-worker-budget.py index e6abf26a9..45e68fdcd 100644 --- a/tests/integration/test-content-provider-worker-budget.py +++ b/tests/integration/test-content-provider-worker-budget.py @@ -84,6 +84,94 @@ def test_phase_boundaries_prevent_native_https_and_later_event_bleed(self): self.assertEqual([item["outcome"] for item in https["records"]], ["failed"]) self.assertNotIn("success", native) + def test_pressure_has_separate_paths_and_preserves_historical_names(self): + expected = {"native": "content-provider-adaptive", "https": "content-provider-adaptive-https", + "pressure": "content-provider-pressure"} + self.assertEqual(set(CHECK["PHASES"]), set(expected)) + for phase, prefix in expected.items(): + marker, output = CHECK["paths"](self.work, phase) + self.assertEqual(marker.name, f".content-worker-budget-{phase}.part") + self.assertEqual(output.name, f"{prefix}-worker-budget.json") + self.assertEqual(CHECK["continuation_path"](self.work, phase).name, f"{prefix}-continuation.json") + for phase in ("", "PRESSURE", "../pressure", "native-pressure"): + with self.subTest(phase=phase), self.assertRaises(ValueError): + CHECK["paths"](self.work, phase) + with self.subTest(phase=phase), self.assertRaises(ValueError): + CHECK["continuation_path"](self.work, phase) + + def test_three_phase_ranges_remain_independent_when_finalized_later(self): + retained = {} + for phase, allowance in (("native", 12), ("https", 8), ("pressure", 3)): + self.marker(phase) + sample = budget(initial_allowance=allowance, max_allowance=allowance) + sample["causes"]["cpu_pressure"] = phase == "pressure" + progress = continuation(returned_verified_chunks=allowance) + self.append(event(sample) + continuation_event(progress)) + report = self.finish(phase) + self.assertEqual(report["phase"], phase) + self.assertEqual(report["records"], [sample]) + for path in (self.report(phase), CHECK["continuation_path"](self.work, phase)): + retained[path] = path.read_bytes() + parsed = json.loads(CHECK["continuation_path"](self.work, phase).read_bytes()) + self.assertEqual(parsed["records"], [progress]) + self.assertEqual(parsed["phase"], phase) + self.append(event(budget(outcome="failed")) + continuation_event(continuation(outcome="failed"))) + CHECK["finalize"](self.work) + for path, raw in retained.items(): + self.assertEqual(path.read_bytes(), raw) + + def test_pressure_pending_then_cancelled_records_finalize_without_other_phases(self): + self.marker("pressure") + pending = self.finish("pressure", pending=True) + target = CHECK["continuation_path"](self.work, "pressure") + self.assertEqual((pending["phase_closed"], pending["complete"], pending["status"]), + (False, False, "incomplete")) + self.assertFalse(json.loads(target.read_bytes())["complete"]) + interrupted = continuation(outcome="interrupted", returned_useful_continuations=0, + returned_verified_chunks=0, returned_verified_bytes=0) + self.append(event(budget(outcome="interrupted")) + continuation_event(interrupted)) + CHECK["finalize"](self.work) + final = json.loads(self.report("pressure").read_bytes()) + self.assertEqual((final["phase_closed"], final["complete"], final["status"]), + (True, True, "observed")) + self.assertEqual(final["records"][0]["outcome"], "interrupted") + self.assertEqual(json.loads(target.read_bytes())["records"], [interrupted]) + for phase in ("native", "https"): + self.assertFalse(self.report(phase).exists()) + self.assertFalse(CHECK["continuation_path"](self.work, phase).exists()) + + def test_pressure_rejects_cross_phase_marker_and_keeps_other_closed_evidence(self): + self.marker("native") + self.append(event() + continuation_event()) + self.finish("native") + retained = self.report("native").read_bytes() + marker, _ = CHECK["paths"](self.work, "pressure") + original, _ = CHECK["paths"](self.work, "native") + marker.write_bytes(original.read_bytes()) + marker.chmod(0o600) + self.append(event(budget(outcome="failed"))) + CHECK["finalize"](self.work) + self.assertEqual(self.report("native").read_bytes(), retained) + for path in (self.report("pressure"), CHECK["continuation_path"](self.work, "pressure")): + parsed = json.loads(path.read_bytes()) + self.assertEqual((parsed["phase"], parsed["status"], parsed["complete"], parsed["records"]), + ("pressure", "invalid", False, [])) + + def test_pressure_cli_lifecycle_is_closed_and_preserves_invalid_payload_isolation(self): + for action in ("mark", "pending"): + result = subprocess.run([sys.executable, "-B", str(HELPER), action, str(self.work), "pressure"], + capture_output=True, timeout=5) + self.assertEqual((result.returncode, result.stdout), (0, b""), result.stderr) + self.append(event(budget(peer_id="not-exportable")) + continuation_event()) + result = subprocess.run([sys.executable, "-B", str(HELPER), "finalize", str(self.work)], + capture_output=True, timeout=5) + self.assertEqual((result.returncode, result.stdout), (0, b""), result.stderr) + invalid = json.loads(self.report("pressure").read_bytes()) + self.assertEqual((invalid["status"], invalid["records"]), ("invalid", [])) + continued = json.loads(CHECK["continuation_path"](self.work, "pressure").read_bytes()) + self.assertEqual((continued["status"], continued["records"]), ("observed", [continuation()])) + self.assertNotIn("not-exportable", self.report("pressure").read_text()) + def test_unfinished_failed_command_is_finalized_after_summary_arrives(self): self.marker("native") pending = self.finish("native", pending=True) @@ -299,13 +387,13 @@ def test_glue_opt_in_cleanup_order_and_closed_emergency_allowlist(self): self.assertLess(retire, finalize) self.assertLess(finalize, topology.index(" copy_artifacts || original_status=1", finalize)) shell = (HERE / "content-provider-adaptive-smoke.sh").read_text() - for phase in CHECK["PHASES"]: + for phase in ("native", "https"): self.assertIn(f"content_provider_worker_budget_capture mark {phase}", shell) self.assertIn(f"content_provider_worker_budget_capture pending {phase}", shell) self.assertIn(f"content_provider_worker_budget_capture finish {phase}", shell) self.assertIn('|| fail CONTENT_PROVIDER_ADAPTIVE_EVIDENCE_INVALID', shell) emergency = (HERE / "run-alpha-topology-vm.sh").read_text() - for phase in CHECK["PHASES"]: + for phase in ("native", "https"): self.assertIn('"' + self.report(phase).name + '"', emergency) self.assertIn('"' + CHECK["continuation_path"](self.work, phase).name + '"', emergency) for name in ("kvm-alpha-topology.sh", "run-alpha-topology-vm.sh", "content-provider-adaptive-smoke.sh"): From 0f391c77a2b9ee72ec2df26cb398a03041ed314b Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:22:34 +0200 Subject: [PATCH 48/49] test: distinguish authenticated refresh in pressure proof --- docs/IMPLEMENTATION_STATUS.md | 73 +++++++++++++--- .../content-provider-pressure-smoke.py | 36 ++++++-- .../content-provider-pressure-smoke.sh | 42 +++++++--- .../test-content-provider-pressure-smoke.py | 84 ++++++++++++++++++- 4 files changed, 206 insertions(+), 29 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 2c0a61f87..3be788473 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,19 +29,20 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Prove pressure retirement and continuation, then resume joined Weather delivery. | The route-preservation correction passes the standalone sequence in run `38204038877` on `9441b451`. The failed joined run and unknown original kernel errno remain separately recorded. The new pressure phase is wired but has not yet passed its protected trial. | -| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined run stops before Weather acquisition, so it makes no new weather-source requests. An earlier exact-source probe encountered ECMWF HTTP 429. Closed retention includes bounded Retry-After diagnostics and passes eighty focused checks, without extra requests or automatic retries. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover joined delivery. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Prove pressure retirement and continuation, then resume joined Weather delivery. | The corrected standalone baseline passes on `9441b451`. The first pressure run observes real contraction and serial continuation but fails its discovery-evidence check: the phase uses targeted refresh, not generic DHT discovery. Correct that distinction and rerun the complete sequence; retain the failed result. | +| Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined attempt on `9441b451` stops during the guest build because of development-host memory pressure, before any Weather request. Prepare execution away from the development host without uploading private inputs or bypassing source limits. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover joined delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR13 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | The corrected standalone provider proof passes without weather downloads. The -latest joined trial uses the corrected Overmorrow bundle but stops at the earlier -shared-route failure. The new standalone result clears that prerequisite, not -the joined delivery gate. Do not automatically retry source requests or route -them through other peers to evade a source limit. +earlier joined trial stops at a shared-route failure; the later attempt with the +corrected core stops during its build because of host memory pressure. Neither +reaches Weather acquisition. Do not automatically retry source requests or route +them through other peers to evade a source limit. The earlier ECMWF HTTP 429 and +its bounded Retry-After diagnostics remain relevant to the next acquisition. Pressure retirement and continuation remain separate evidence gaps, not implied -by successful parallel delivery. A separate disposable-guest phase is now wired -after the healthy native and HTTPS transfers. It is not part of either retained -standalone result or the failed joined run. +by successful parallel delivery. The separate pressure phase has reached real +transfer and cleanup, but its first complete report fails validation. It is not +part of either passing standalone result or either joined attempt. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM @@ -111,9 +112,32 @@ Evidence is retained at `0d8cbd696240dd7d12f73545c4ef305b5b73e6a845620388a88bad3819d449a3`. Worker allowance remains twelve, with a peak of three leases and no pressure or continuation events. The run proves neither pressure recovery nor joined Weather -delivery. CodeQL passes for this exact revision; Quality checks are still running. +delivery. CodeQL and Quality checks both pass for this exact revision. -### Prepared pressure retirement trial +### Joined attempt stopped by host resource protection + +The next joined attempt uses the same proven core revision `9441b451` and the +unchanged, verified Overmorrow input bundle. It exits with status 143 during the +guest build after observations of memory pressure and falling available +host memory. No topology or Weather-source request is reached. The attempt is +aborted, not passed, and is not automatically restarted. + +The owned runner, VM and SSH supervisor are gone, the forwarded port is free, +and the temporary VM files are removed. The base image and tracked source remain +unchanged. DNS, routes, rules, addresses, forwarding settings, namespace names +and WireGuard interface names match their initial observations. Full host-network +equality is not proven: firewall access is unavailable and the link hash differs. +That hash includes dynamic bridge timers, but the raw initial values were not +retained, so the cause of the difference is not established. + +Evidence is retained at `.git/ci-evidence/weather-overmorrow-9441.PSl9Ws/`. +The next preparation step is a reproducible hosted input recipe: the current +workflow has no joined Weather input, and pinned public Node and Firefox +acquisition plus decoder-build verification remain missing. Keep the existing +private bundle outside published artifacts; a fresh build needs its own browser +proof and bundle hash. + +### First pressure retirement trial The new phase uses the original sixty-chunk publication, three disjoint provider caches and the same protected route, but a fresh client cache. After observing @@ -136,7 +160,32 @@ Current provider reports use schema four and require this separate component. Earlier schema-one through schema-three results remain verifiable only through the explicit historical-report path; their original claims are unchanged. Focused controller, component, diagnostic, wiring and aggregate tests pass. -These are test-harness checks, not a completed live pressure trial. +The first [live pressure run](https://github.com/VOLPAROSSA/volparossa/actions/runs/38205593860) +on `52a3a787` nevertheless fails at the component-evidence check. The checker +requires generic DHT discovery events, while this phase performs three targeted +provider refreshes. Its original raw evidence reproduces that rejection; it has +not been rewritten as a successful report. + +The raw observations show a complete 15,728,640-byte, sixty-chunk transfer from +three providers without origin fallback. Under CPU-only pressure, allowance +falls from ten to one after a peak of three leases. Two serial continuations +return thirty-one verified chunks (8,126,464 bytes). Both pressure workers are +reaped, cleanup removes all owned objects, and the guest-root network state is +unchanged. These observations do not constitute a passing aggregate: late +recovery and cancellation are not reached in this run. + +Evidence is retained at `.git/ci-evidence/content-pressure-hosted.ao2q3z/`. +The report SHA-256 is +`c1bacb39e85b50e2fd2093e4aff00cd8f87e8b3b3561cdf3e92f0c1c383848ef`. +The corrected pressure component uses schema two, with separate counters for +generic queries, targeted lookup starts and finishes, and verified nonempty +client responses. Counters cover only the actual fetch interval; an exact lookup +finish alone cannot satisfy the check. The earlier generic discovery proof and +all cold-cache, three-provider, pressure, continuation, route, privacy and +cleanup requirements remain mandatory. A targeted lookup is not a DHT query. +The correction passes 117 focused fixture checks and independent review, but +still requires a new full protected trial. The old raw result lacks these new +counters and remains rejected rather than being upgraded after the fact. ### Verified cache delivery and remaining integration diff --git a/tests/integration/content-provider-pressure-smoke.py b/tests/integration/content-provider-pressure-smoke.py index d606e7b46..3be5f3b2f 100644 --- a/tests/integration/content-provider-pressure-smoke.py +++ b/tests/integration/content-provider-pressure-smoke.py @@ -5,6 +5,8 @@ The controller observes real guest CPU PSI after three verified cold-cache chunks. Production diagnostics independently prove admission contraction and useful serial continuation. Exact reconstruction and protected-path captures are still mandatory. +Schema 2 distinguishes generic discovery from authenticated recent-provider refresh; +an exact-lookup completion alone is not proof of a nonempty verified response. """ import os from pathlib import Path @@ -36,12 +38,36 @@ rate_bits_per_second=2000000, burst_bytes=131072, queue_latency_ms=250) CLEANUP = dict(controller_reaped=True, fetch_reaped=True, owned_output_removed=True, cache_teardown_deferred_to_topology=True, qdisc_restored=True, route_retained=True) +DISCOVERY_COUNTS = {"generic_dht_queries", "authenticated_upstream_offers", "client_forwarded_discovery", + "exact_lookup_started", "exact_lookup_finished", "client_nonempty_verified_responses"} +OUTPUT_FIELDS = DISCOVERY_COUNTS | {"sha256", "bytes", "client_cache_initially_absent", + "client_mount_positive_control", "client_mount_cannot_read_replica_stores", + "publisher_process_exited_before_fetch", "event_baseline_unix_ms", "started_monotonic_ns", + "completed_monotonic_ns", "started_unix_ms", "completed_unix_ms"} def integer(value, low=0, high=2**64-1): return type(value) is int and low <= value <= high +def validate_discovery(output): + require(type(output) is dict and set(output) == OUTPUT_FIELDS + and all(integer(output[key], 0, 2**32-1) for key in DISCOVERY_COUNTS), + "closed phase-bound discovery counters required") + queries, offers, completed = (output[key] for key in + ("generic_dht_queries", "authenticated_upstream_offers", "client_forwarded_discovery")) + started, finished, nonempty = (output[key] for key in + ("exact_lookup_started", "exact_lookup_finished", "client_nonempty_verified_responses")) + require(finished <= started and 1 <= nonempty <= completed, + "fresh verified broker response absent or counters inconsistent") + generic = queries >= 1 and offers >= 3 + # Recent hints are only identities, scoped to this route/policy/control broker. + # The production client independently checks authority, targets, signatures and + # expiry before emitting RECEIVED_OFFERS. Exact FINISHED may instead be empty. + exact = queries == offers == 0 and 3 <= nonempty <= finished <= started + require(generic or exact, "neither generic discovery nor three authenticated exact refreshes proved") + + def publication(value): keys = {"report_kind", "publisher_hex", "manifest_id", "object_sha256", "bytes", "chunks", "created_unix_seconds", "expires_unix_seconds", "publisher_removed", @@ -145,7 +171,7 @@ def validate_qdisc(value): def validate(evidence): require(type(evidence) is dict and set(evidence) == FIELDS - and integer(evidence["schema_version"], 1, 1) + and integer(evidence["schema_version"], 2, 2) and evidence["kind"] == "content-provider-pressure-smoke" and evidence["success"] is True and evidence["scope"] == SCOPE and all(type(v) is bool for v in evidence["scope"].values()), "invalid closed pressure component") @@ -164,6 +190,7 @@ def validate(evidence): require(all(evidence[key]["control_relay_peer_id"] == control for key in ("status_before", "status_after", "fetch")), "control authority changed") fetch, output = evidence["fetch"], evidence["output"] + validate_discovery(output) require(fetch["operation"] == "native_content" and integer(fetch["bytes"], BYTES, BYTES) and integer(fetch["chunks"], CHUNKS, CHUNKS) and integer(fetch["providers_used"], 3, 3) and type(fetch["provider_peer_ids"]) is list and len(fetch["provider_peer_ids"]) == 3 @@ -174,15 +201,14 @@ def validate(evidence): require(output["sha256"] == SHA and integer(output["bytes"], BYTES, BYTES) and all(output[key] is True for key in ("client_cache_initially_absent", "client_mount_positive_control", "client_mount_cannot_read_replica_stores", "publisher_process_exited_before_fetch")) - and integer(output["event_baseline_unix_ms"], 1) - and integer(output["generic_dht_queries"], 1) and integer(output["authenticated_upstream_offers"], 3) - and integer(output["client_forwarded_discovery"], 1), "cold verified reconstruction/isolation absent") + and integer(output["event_baseline_unix_ms"], 1), "cold verified reconstruction/isolation absent") for key in ("started_monotonic_ns", "completed_monotonic_ns", "started_unix_ms", "completed_unix_ms"): require(integer(output[key], 1), "invalid original fetch clock") require(ready["ready_ns"] <= output["started_monotonic_ns"] <= controller["trigger_ns"] <= controller["first_pressure_ns"] <= output["completed_monotonic_ns"] and 0 < output["completed_monotonic_ns"] - output["started_monotonic_ns"] <= 120_000_000_000 and seed["created_unix_seconds"] * 1000 <= output["started_unix_ms"] + and output["event_baseline_unix_ms"] <= output["started_unix_ms"] <= output["completed_unix_ms"] < seed["expires_unix_seconds"] * 1000, "CPU event occurred outside the original bounded, live-publication fetch") original, before, live = (evidence[key] for key in @@ -242,7 +268,7 @@ def build_evidence(work): require(work.is_absolute() and work.resolve() == work, "unsafe fixture root") directory = CONTROLLER["directory"](work, os.geteuid()) try: - evidence = dict(schema_version=1, kind="content-provider-pressure-smoke", success=True, scope=dict(SCOPE)) + evidence = dict(schema_version=2, kind="content-provider-pressure-smoke", success=True, scope=dict(SCOPE)) for key, filename in (("publication", "content-provider-adaptive-publication.json"), ("expected_peers", "a01-expected-peers.json"), ("layout", "content-provider-adaptive-layout.json"), diff --git a/tests/integration/content-provider-pressure-smoke.sh b/tests/integration/content-provider-pressure-smoke.sh index 6b2e82742..c26158a33 100644 --- a/tests/integration/content-provider-pressure-smoke.sh +++ b/tests/integration/content-provider-pressure-smoke.sh @@ -105,6 +105,33 @@ content_provider_pressure_shape() { >"$WORK/content-provider-pressure-qdisc-profile.json" } +content_provider_pressure_event_count() { + # Millisecond event timestamps include the whole fetch, but no setup/history + # before it or unrelated later work. Equal timestamps at its start are valid. + awk -v baseline="$pp_baseline_ms" -v started="$pp_started_unix" \ + -v completed="$pp_completed_unix" -v event="event=$1" \ + '$1 > baseline && $1 >= started && $1 <= completed && $3 == event {n++} END {print n+0}' "$2" +} + +content_provider_pressure_discovery_counts() { + pp_queries=0 + pp_offers=0 + pp_exact_started=0 + pp_exact_finished=0 + for pp_node in relay0 relay1 relay2 relay3 relay4 relay5; do + pp_queries=$((pp_queries + $(content_provider_pressure_event_count \ + CONTENT_PROVIDER_QUERY_STARTED "$WORK/logs-$pp_node.txt"))) + pp_offers=$((pp_offers + $(content_provider_pressure_event_count \ + CONTENT_PROVIDER_OFFER_VERIFIED "$WORK/logs-$pp_node.txt"))) + pp_exact_started=$((pp_exact_started + $(content_provider_pressure_event_count \ + CONTENT_EXACT_LOOKUP_STARTED "$WORK/logs-$pp_node.txt"))) + pp_exact_finished=$((pp_exact_finished + $(content_provider_pressure_event_count \ + CONTENT_EXACT_LOOKUP_FINISHED "$WORK/logs-$pp_node.txt"))) + done + pp_discovered=$(content_provider_pressure_event_count CONTENT_DISCOVERY_COMPLETED "$WORK/logs-client.txt") + pp_nonempty=$(content_provider_pressure_event_count CONTENT_DISCOVERY_RECEIVED_OFFERS "$WORK/logs-client.txt") +} + content_provider_pressure_run() { PHASE=content-provider-pressure-setup pp_prefix=content-provider-pressure @@ -210,16 +237,7 @@ content_provider_pressure_run() { stop_privacy_observers || fail CONTENT_PROVIDER_PRESSURE_CAPTURE_NOT_DRAINED content_provider_stop_control_observer || fail CONTENT_PROVIDER_PRESSURE_CONTROL_NOT_DRAINED capture_product_logs - pp_queries=0 - pp_offers=0 - for pp_node in relay0 relay1 relay2 relay3 relay4 relay5; do - pp_queries=$((pp_queries + $(awk -v baseline="$pp_baseline_ms" \ - '$1 > baseline && $3 == "event=CONTENT_PROVIDER_QUERY_STARTED" {n++} END {print n+0}' "$WORK/logs-$pp_node.txt"))) - pp_offers=$((pp_offers + $(awk -v baseline="$pp_baseline_ms" \ - '$1 > baseline && $3 == "event=CONTENT_PROVIDER_OFFER_VERIFIED" {n++} END {print n+0}' "$WORK/logs-$pp_node.txt"))) - done - pp_discovered=$(awk -v baseline="$pp_baseline_ms" \ - '$1 > baseline && $3 == "event=CONTENT_DISCOVERY_COMPLETED" {n++} END {print n+0}' "$WORK/logs-client.txt") + content_provider_pressure_discovery_counts if [ -L "$pp_client/object.bin" ] \ || [ "$(stat -Lc '%a:%u:%g:%h' "$pp_client/object.bin")" != "600:$AGENT_UID:$AGENT_GID:1" ]; then fail CONTENT_PROVIDER_PRESSURE_OUTPUT_UNSAFE @@ -228,12 +246,16 @@ content_provider_pressure_run() { --argjson bytes "$(stat -Lc '%s' "$pp_client/object.bin")" \ --argjson baseline "$pp_baseline_ms" --argjson queries "$pp_queries" \ --argjson offers "$pp_offers" --argjson discovered "$pp_discovered" \ + --argjson exact_started "$pp_exact_started" --argjson exact_finished "$pp_exact_finished" \ + --argjson nonempty "$pp_nonempty" \ --argjson started_mono "$pp_started_mono" --argjson completed_mono "$pp_completed_mono" \ --argjson started_unix "$pp_started_unix" --argjson completed_unix "$pp_completed_unix" \ '{sha256:$sha256,bytes:$bytes,client_cache_initially_absent:true, client_mount_positive_control:true,client_mount_cannot_read_replica_stores:true, publisher_process_exited_before_fetch:true,event_baseline_unix_ms:$baseline, generic_dht_queries:$queries,authenticated_upstream_offers:$offers, + exact_lookup_started:$exact_started,exact_lookup_finished:$exact_finished, + client_nonempty_verified_responses:$nonempty, client_forwarded_discovery:$discovered,started_monotonic_ns:$started_mono, completed_monotonic_ns:$completed_mono,started_unix_ms:$started_unix,completed_unix_ms:$completed_unix}' \ >"$WORK/$pp_prefix-object.json" diff --git a/tests/integration/test-content-provider-pressure-smoke.py b/tests/integration/test-content-provider-pressure-smoke.py index efabc8885..b8eb47279 100644 --- a/tests/integration/test-content-provider-pressure-smoke.py +++ b/tests/integration/test-content-provider-pressure-smoke.py @@ -6,6 +6,7 @@ import os from pathlib import Path import runpy +import subprocess import tempfile import unittest from unittest.mock import patch @@ -36,12 +37,13 @@ def diagnostic(kind, record): selected = original["selected_route"] output = dict(original["output"], started_monotonic_ns=controller["started_ns"], completed_monotonic_ns=12_000_000_000, started_unix_ms=1_001_000, - completed_unix_ms=1_012_000) + completed_unix_ms=1_012_000, exact_lookup_started=0, exact_lookup_finished=0, + client_nonempty_verified_responses=1) providers = {node: dict(reset=dict(serving=False, publications=0), serve=dict(serving=True, publications=1, replication_enabled=False), status=dict(serving=True, publications=1, replication_enabled=False), stop=dict(serving=False, publications=0)) for node in CHECK["NODES"]} - return dict(schema_version=1, kind="content-provider-pressure-smoke", success=True, + return dict(schema_version=2, kind="content-provider-pressure-smoke", success=True, scope=dict(CHECK["SCOPE"]), publication=original["publication"], expected_peers=original["expected_peers"], layout=original["layout"], original_selected_route=copy.deepcopy(selected), selected_before=copy.deepcopy(selected), @@ -113,6 +115,84 @@ def test_complete_pressure_and_serial_useful_continuation(self): self.assertFalse(evidence["scope"]["pressure_recovery_claimed"]) self.assertFalse(evidence["scope"]["owner_goodput_claimed"]) + def test_fresh_exact_refresh_is_not_reported_as_generic_discovery(self): + evidence = fixture() + evidence["output"].update(generic_dht_queries=0, authenticated_upstream_offers=0, + exact_lookup_started=3, exact_lookup_finished=3, + client_nonempty_verified_responses=3, client_forwarded_discovery=3) + self.assertEqual(CHECK["validate"](evidence), evidence) + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + write_files(work, evidence) + self.assertEqual(CHECK["build_evidence"](work), evidence) + self.assertEqual(evidence["output"]["generic_dht_queries"], 0) + self.assertEqual(evidence["output"]["authenticated_upstream_offers"], 0) + + def test_exact_finish_without_three_nonempty_verified_responses_is_not_proof(self): + exact = dict(generic_dht_queries=0, authenticated_upstream_offers=0, + exact_lookup_started=3, exact_lookup_finished=3, + client_nonempty_verified_responses=3, client_forwarded_discovery=3) + for key, value in (("exact_lookup_started", 0), ("exact_lookup_finished", 2), + ("exact_lookup_finished", 4), ("client_nonempty_verified_responses", 0), + ("client_nonempty_verified_responses", 2), ("client_forwarded_discovery", 2), + ("authenticated_upstream_offers", 1)): + with self.subTest(key=key, value=value): + evidence = fixture() + evidence["output"].update(exact) + evidence["output"][key] = value + with self.assertRaises(ValueError): + CHECK["validate"](evidence) + + def test_missing_generic_or_empty_discovery_cannot_pass(self): + for key, value in (("generic_dht_queries", 0), ("authenticated_upstream_offers", 2), + ("client_nonempty_verified_responses", 0), ("client_forwarded_discovery", 0)): + self.rejected(lambda e: e["output"].__setitem__(key, value)) + for key in CHECK["DISCOVERY_COUNTS"]: + for value in (-1, True, 1.0, 2**32): + self.rejected(lambda e: e["output"].__setitem__(key, value)) + self.rejected(lambda e: e["output"].pop(key)) + self.rejected(lambda e: e["output"].__setitem__("unknown", "raw-log")) + self.rejected(lambda e: e.__setitem__("schema_version", 1)) + self.rejected(lambda e: e["output"].__setitem__("event_baseline_unix_ms", 1_001_001)) + + def test_real_shell_counters_exclude_other_phases_and_preserve_event_meanings(self): + def row(timestamp, event): + return f"{timestamp}\tlevel=1\tevent={event}\tsession=\tpath=-\n" + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + for node in ("client", "relay0", "relay1", "relay2", "relay3", "relay4", "relay5"): + (work/f"logs-{node}.txt").write_text("", encoding="ascii") + relay = work/"logs-relay1.txt" + client = work/"logs-client.txt" + command = '. "$1"\nWORK=$2\npp_baseline_ms=100\npp_started_unix=200\npp_completed_unix=400\n' + command += 'content_provider_pressure_discovery_counts\n' + command += 'printf "%s %s %s %s %s %s\\n" "$pp_queries" "$pp_offers" "$pp_discovered" ' + command += '"$pp_exact_started" "$pp_exact_finished" "$pp_nonempty"\n' + def count(): + result = subprocess.run(["sh", "-eu", "-c", command, "sh", + str(HERE/"content-provider-pressure-smoke.sh"), str(work)], + capture_output=True, check=True, timeout=5) + self.assertEqual(result.stderr, b"") + return [int(value) for value in result.stdout.split()] + # Earlier native/HTTPS events and work after this fetch do not count. + outside = "".join(row(ts, event) for ts in (99, 100, 101, 199, 401) + for event in ("CONTENT_PROVIDER_QUERY_STARTED", "CONTENT_PROVIDER_OFFER_VERIFIED", + "CONTENT_EXACT_LOOKUP_STARTED", "CONTENT_EXACT_LOOKUP_FINISHED")) + relay.write_text(outside + "".join(row(ts, "CONTENT_EXACT_LOOKUP_STARTED") + + row(ts+1, "CONTENT_EXACT_LOOKUP_FINISHED") for ts in (200, 250, 399)), encoding="ascii") + client.write_text("".join(row(ts, event) for ts in (199, 201, 251, 400, 401) + for event in ("CONTENT_DISCOVERY_RECEIVED_OFFERS", "CONTENT_DISCOVERY_COMPLETED")) + + row(300, "CONTENT_DISCOVERY_RECEIVED_EMPTY"), encoding="ascii") + self.assertEqual(count(), [0, 0, 3, 3, 3, 3]) + relay.write_text(outside + row(200, "CONTENT_PROVIDER_QUERY_STARTED") + + row(300, "CONTENT_PROVIDER_OFFER_VERIFIED") * 3, encoding="ascii") + client.write_text(row(400, "CONTENT_DISCOVERY_RECEIVED_OFFERS") + + row(400, "CONTENT_DISCOVERY_COMPLETED"), encoding="ascii") + self.assertEqual(count(), [1, 3, 1, 0, 0, 1]) + client.write_text(row(400, "CONTENT_DISCOVERY_RECEIVED_EMPTY") + + row(400, "CONTENT_DISCOVERY_COMPLETED"), encoding="ascii") + self.assertEqual(count(), [1, 3, 1, 0, 0, 0]) + def test_contraction_requires_three_actual_leases_and_available_resources(self): for key, value in (("initial_allowance", 2), ("peak_leases", 2), ("min_allowance", 2), ("min_memory_workers", 2), ("min_fd_workers", None), ("active_leases", 1), From c48fba8cbc78dd006dc7514b5a0e1fe9e014af8b Mon Sep 17 00:00:00 2001 From: Erik Le Blansch <58982453+erikleblansch@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:45:52 +0200 Subject: [PATCH 49/49] docs: record verified provider continuation under CPU pressure --- docs/IMPLEMENTATION_STATUS.md | 52 +++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 3be788473..552296a2f 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -29,7 +29,7 @@ The immediate milestones build on the retained candidates and evidence below: | Workstream and repositories | Next demonstrable milestone | Dependencies and remaining blockers | | --- | --- | --- | -| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Prove pressure retirement and continuation, then resume joined Weather delivery. | The corrected standalone baseline passes on `9441b451`. The first pressure run observes real contraction and serial continuation but fails its discovery-evidence check: the phase uses targeted refresh, not generic DHT discovery. Correct that distinction and rerun the complete sequence; retain the failed result. | +| Continue shared cache delivery — `volparossa`, `volparossa-weather` | Integrate the verified provider sequence, then resume joined Weather delivery. | The full sequence passes on `0f391c77`, including real CPU-pressure contraction and useful serial continuation. Independent raw-evidence reconstruction passes; the earlier failed result remains unchanged. Join the pressure fixture to PR227 and complete its integration checks. | | Continue weather field integration — `volparossa-weather`, `volparossa` | Deliver native precipitation and PM2.5/PM10 through the shared cache into Drizz.li, with correct source time windows, missing values, refresh and expiry. | The latest joined attempt on `9441b451` stops during the guest build because of development-host memory pressure, before any Weather request. Prepare execution away from the development host without uploading private inputs or bypassing source limits. Existing synthetic UI, temperature/wind and separate NOAA source results do not cover joined delivery. | | Extend the same provider to Overmorrow — `volparossa-weather`, `volparossa` | Deliver core-cache products into the actual client, then verify native packaging. | PR9–PR13 are merged with passing CI. Exact table extraction and a fresh full browser trial pass with synthetic products. A new immutable bundle with the corrected verifier passes capture, runtime and extracted-tree checks; the old bundle remains unchanged. Joined core delivery and native Android/iOS execution remain unverified. | @@ -39,10 +39,10 @@ corrected core stops during its build because of host memory pressure. Neither reaches Weather acquisition. Do not automatically retry source requests or route them through other peers to evade a source limit. The earlier ECMWF HTTP 429 and its bounded Retry-After diagnostics remain relevant to the next acquisition. -Pressure retirement and continuation remain separate evidence gaps, not implied -by successful parallel delivery. The separate pressure phase has reached real -transfer and cleanup, but its first complete report fails validation. It is not -part of either passing standalone result or either joined attempt. +Pressure retirement and useful serial continuation now pass in a separate phase +of the complete `0f391c77` sequence. This does not prove that pressure has fallen +before continuation, improved owner-task performance, or joined Weather delivery. +The first pressure run remains failed; neither joined attempt reaches this phase. The Drizz.li/Weather candidate and frozen input bundle remain intact; Overmorrow's core delivery and native execution are still required. Keep shared interface changes under one owner and integrate dependent consumers together. Any new VM @@ -183,9 +183,45 @@ client responses. Counters cover only the actual fetch interval; an exact lookup finish alone cannot satisfy the check. The earlier generic discovery proof and all cold-cache, three-provider, pressure, continuation, route, privacy and cleanup requirements remain mandatory. A targeted lookup is not a DHT query. -The correction passes 117 focused fixture checks and independent review, but -still requires a new full protected trial. The old raw result lacks these new -counters and remains rejected rather than being upgraded after the fact. +The correction passes 117 focused fixture checks and independent review. +The new full protected trial below supplies its runtime evidence. +The old raw result lacks these new counters and remains rejected rather than +being upgraded after the fact. + +### Verified provider continuation under CPU pressure + +[Run `38207713645`](https://github.com/VOLPAROSSA/volparossa/actions/runs/38207713645) +passes on exact source `0f391c77a2b9ee72ec2df26cb398a03041ed314b`. +Independent validation reconstructs the schema-four aggregate and schema-two +pressure component from the raw artifacts; they match the published reports. + +During the separate pressure phase, worker allowance falls from twelve to one +under measured CPU-only pressure, after a peak of three leases. Two useful +serial continuations return thirty chunks, 7,864,320 bytes. The complete +sixty-chunk object, 15,728,640 bytes, arrives from three providers in 57.462 +seconds without origin fallback. Targeted discovery records three starts, +three finishes and three verified nonempty responses; generic DHT queries +remain zero. Both pressure workers are reaped without forced kills, and no +transfer workers remain active. + +The complete sequence also verifies healthy native and HTTPS parallel delivery, +late complementary-provider discovery and requester cancellation. Healthy phases +each return the same sixty-chunk object with actual three-provider payload +overlaps of 4.647 and 4.300 seconds. HTTPS uses one authenticated TLS 1.3 HEAD +and no origin body request. Late recovery releases its gate after 5.371 seconds +and completes in 12.887 seconds. After requester cancellation, independent +cache-only delivery completes in 116 ms and control IPC reaches EOF in 10 ms. + +Privacy checks retain the required relay paths. Cleanup removes all owned +objects, and the disposable guest-root network state matches before and after. +This is not evidence of development-host firewall equality, improved owner-task +performance, recovery of CPU headroom, or joined Weather/browser delivery. + +Evidence is retained at +`.git/ci-evidence/content-pressure-hosted-fixed.t5ru8b/`. The aggregate SHA-256 is +`d25eb8460f7edc8f428c9d438db805b254f334608aa9842edc7f8b878c420872`; +the pressure component SHA-256 is +`5c2e2fc6044fc502f21c129022ab355f9a664878eac2388f9f24682d7e8e52c6`. ### Verified cache delivery and remaining integration