diff --git a/.github/workflows/real-binary.yml b/.github/workflows/real-binary.yml new file mode 100644 index 000000000..229f40159 --- /dev/null +++ b/.github/workflows/real-binary.yml @@ -0,0 +1,72 @@ +# Policy: weekly/workflow_dispatch plus pull requests that change this gate; not a required merge gate. +name: Real-binary chain + +on: + schedule: + - cron: "17 20 * * 0" + workflow_dispatch: + pull_request: + paths: + - .github/workflows/real-binary.yml + - scripts/ci/run-real-binary-gate.sh + - scripts/ci/check-real-binary-results.py + - macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift + - macos/EngramServiceCoreTests/CollectorCLIIntegrationTests.swift + +permissions: + contents: read + +concurrency: + group: real-binary-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +env: + XCODEGEN_SHA256: "090ec29491aad50aec10631bf6e62253fed733c50f3aab0f5ffc86bc170bdbef" + XCODEGEN_VERSION: "2.45.4" + +jobs: + real-binary-chain: + name: Real-binary chain + runs-on: macos-15 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install xcodegen + run: bash scripts/ci/install-xcodegen.sh "$XCODEGEN_VERSION" "$XCODEGEN_SHA256" + + - name: Cache SPM packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/Library/Caches/EngramSourcePackages + key: spm-${{ runner.os }}-${{ runner.arch }}-hosted-realbinary-${{ hashFiles('macos/project.yml', 'macos/Engram.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ runner.os }}-${{ runner.arch }}-hosted-realbinary- + + - name: Generate Xcode project + run: scripts/check-xcodeproj-drift.sh + + - name: Build binaries and run the real-binary chain + run: | + # Hosted macos-15 is several times slower, and varies run to run, than + # the Mac that produced the local receipt: the 114 MB large-history + # chain's HQ ingest took about 7 min in run 39093667839 (pass) and + # more than 12 min in run 39097383289 (deadline at 3x), while every + # other test passed each time. The scale lengthens deadlines only. + bash scripts/ci/run-real-binary-gate.sh \ + --out "$RUNNER_TEMP/engram-real-binary-out" \ + --derived-data "$RUNNER_TEMP/engram-real-binary-derived" \ + --cloned-source-packages "$HOME/Library/Caches/EngramSourcePackages" \ + --timeout-scale 6 + + - name: Publish receipt + if: always() + run: | + out="$RUNNER_TEMP/engram-real-binary-out" + if [ -f "$out/receipt.md" ]; then + cat "$out/receipt.md" >> "$GITHUB_STEP_SUMMARY" + else + echo "No real-binary receipt was written." >> "$GITHUB_STEP_SUMMARY" + fi + if [ -f "$out/receipt.json" ]; then + cat "$out/receipt.json" + fi diff --git a/docs/TODO.md b/docs/TODO.md index 241c33a36..402d1e90c 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -49,6 +49,25 @@ authorizes host changes. `CollectorBinaryShadowIntegrationTests.swift` skips unless three binary-path environment variables are set, and no CI workflow sets them. Done when CI sets them or a documented manual gate with a receipt exists. + Update 2026-10-18 (branch `ci/real-binary-gate-20261018`): shipped the + non-required `.github/workflows/real-binary.yml` (weekly Sunday 20:17 UTC, + `workflow_dispatch`, and pull requests that touch its own five gate paths; + not in `test.yml` or `ci-gate`) and `scripts/ci/run-real-binary-gate.sh`, + which builds the Debug `EngramCollector`/`EngramService`/`EngramRemoteServer` + tools, sets the three binary variables (plain and `TEST_RUNNER_` forms) with + an isolated `CFFIXED_USER_HOME`, and runs + `CollectorBinaryShadowIntegrationTests`, `CollectorLargeHistoryBinaryTests` + and `CollectorCLIIntegrationTests` (CLI needs only the collector binary). + `scripts/ci/check-real-binary-results.py` compares the exact test set parsed + from the Swift sources with the xcresult tree and fails on any missing, + extra, failed or skipped test except the opt-in browser demo. Local Debug + receipt on Xcode 26.4 at `b2e8ef2a` (dirty paths 0): + `REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0`; the + same classes run without the variables exit 0 with 50 skipped and the + verifier rejects them (`result=fail ... skipped=50`). Remains: the PR's own + hosted macos-15 "Real-binary chain" run must be green with that line (Debug + tool loading and the 25 s scope deadlines are unproven on hosted runners); + record the run URL here and close the item. - **App "open Web reader" entry (design section 1).** In `collector`/`replica` roles the App shows only the unavailable message; it has no action that opens the Web reader as the design states. diff --git a/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift b/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift index ddc15a019..0bd205995 100644 --- a/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift +++ b/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift @@ -114,6 +114,15 @@ private struct ShadowBinaries { } return Self(collector: paths[0], service: paths[1], remote: paths[2]) } + + /// Deadline multiplier for slower hosts (hosted CI runners), set by + /// scripts/ci/run-real-binary-gate.sh --timeout-scale. Clamped to 1...10; + /// anything unset or malformed means 1. It never changes an assertion. + static var timeoutScale: Double { + guard let raw = ProcessInfo.processInfo.environment["ENGRAM_REAL_BINARY_TIMEOUT_SCALE"], + let value = Double(raw), value.isFinite else { return 1 } + return min(max(value, 1), 10) + } } private struct ShadowRole { @@ -209,7 +218,7 @@ private final class BinaryShadowScope: @unchecked Sendable { cursorLegacy: Bool = false, seedStores: Bool = true, timeout: TimeInterval = 25, piLargeReply: Bool = false) throws { guard [.codex, .claudeCode, .minimax, .lobsterai, .qwen, .qoder, .iflow, .vscode, .cline, .commandcode, .copilot, .geminiCli, .opencode, .kimi, .cursor, .antigravity, .windsurf, .pi, .grok].contains(sourceKind) else { throw BinaryShadowFailure.fixture } guard !cursorLegacy || sourceKind == .cursor else { throw BinaryShadowFailure.fixture } - self.deadline = Date().addingTimeInterval(timeout) + self.deadline = Date().addingTimeInterval(timeout * ShadowBinaries.timeoutScale) self.binaries = binaries self.sourceKind = sourceKind self.piLargeReply = piLargeReply @@ -665,8 +674,8 @@ private final class BinaryShadowScope: @unchecked Sendable { try checkRunning() let configuration = URLSessionConfiguration.ephemeral configuration.connectionProxyDictionary = [:] - configuration.timeoutIntervalForRequest = 0.5 - configuration.timeoutIntervalForResource = 1.5 + configuration.timeoutIntervalForRequest = 0.5 * ShadowBinaries.timeoutScale + configuration.timeoutIntervalForResource = 1.5 * ShadowBinaries.timeoutScale let session = URLSession(configuration: configuration) defer { session.invalidateAndCancel() } var request = URLRequest(url: replica.baseURL.appendingPathComponent(path)) @@ -3741,7 +3750,7 @@ private extension BinaryShadowScope { XCTAssertEqual(manifest.wholeSourceSHA256, EngramCollectorCore.ArchiveV2Hash.sha256(expected)) let configuration = URLSessionConfiguration.ephemeral configuration.connectionProxyDictionary = [:] - configuration.timeoutIntervalForRequest = 10 + configuration.timeoutIntervalForRequest = 10 * ShadowBinaries.timeoutScale let session = URLSession(configuration: configuration) defer { session.invalidateAndCancel() } var offset = 0 diff --git a/scripts/ci/check-real-binary-results.py b/scripts/ci/check-real-binary-results.py new file mode 100755 index 000000000..03e6084e2 --- /dev/null +++ b/scripts/ci/check-real-binary-results.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Fail the real-binary chain gate unless the expected XCTest set actually ran. + +The real-binary suites skip themselves when the explicit binary-path +environment variables do not reach the test runner, and a fully skipped run +still exits 0. This verifier therefore compares the exact (class, test) set +declared in the Swift sources with the Test Case nodes exported by +`xcrun xcresulttool get test-results tests`, and accepts only Passed results +(plus Skipped for explicitly allowed opt-in tests). +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import re +import sys +from pathlib import Path + + +CLASS_PATTERN = re.compile(r"^(?:final )?class (\w+): XCTestCase\b") +EXTENSION_PATTERN = re.compile(r"^extension (\w+)\b") +OTHER_TOP_LEVEL_PATTERN = re.compile( + r"^(?:(?:private|fileprivate|public|internal|final|@\w+)\s+)*" + r"(?:class|struct|enum|extension|actor|protocol)\b" +) +TEST_PATTERN = re.compile(r"^ {4}func (test\w+)\(") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--tests-json", required=True, type=Path) + parser.add_argument("--source", required=True, type=Path, action="append") + parser.add_argument("--allow-skip", action="append", default=[]) + parser.add_argument("--xcodebuild-exit-code", required=True, type=int) + parser.add_argument("--binary", action="append", default=[]) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--xcode-version", required=True) + parser.add_argument("--dirty-paths", type=int, default=None) + parser.add_argument("--receipt", required=True, type=Path) + parser.add_argument("--summary", required=True, type=Path) + return parser.parse_args() + + +def expected_tests(sources: list[Path], reasons: list[str]) -> list[tuple[str, str]]: + found: list[tuple[str, str]] = [] + for source in sources: + try: + lines = source.read_text().splitlines() + except OSError as error: + reasons.append(f"cannot read test source {source}: {error}") + continue + test_classes: set[str] = set() + current: str | None = None + for line in lines: + class_match = CLASS_PATTERN.match(line) + if class_match is not None: + current = class_match.group(1) + test_classes.add(current) + continue + extension_match = EXTENSION_PATTERN.match(line) + if extension_match is not None: + name = extension_match.group(1) + current = name if name in test_classes else None + continue + if OTHER_TOP_LEVEL_PATTERN.match(line): + current = None + continue + test_match = TEST_PATTERN.match(line) + if test_match is not None and current is not None: + found.append((current, test_match.group(1))) + if not found: + reasons.append("no XCTest methods found in the test sources") + duplicates = sorted({item for item in found if found.count(item) > 1}) + for suite, name in duplicates: + reasons.append(f"duplicate expected test {suite}/{name}") + return sorted(set(found)) + + +def actual_tests(path: Path, reasons: list[str]) -> list[dict[str, object]]: + try: + payload = json.loads(path.read_text()) + except (OSError, ValueError) as error: + reasons.append(f"cannot parse xcresult tests JSON {path}: {error}") + return [] + if not isinstance(payload, dict) or not isinstance(payload.get("testNodes"), list): + reasons.append("xcresult tests JSON has no testNodes array") + return [] + + records: list[dict[str, object]] = [] + + def walk(node: object, suite: str | None) -> None: + if not isinstance(node, dict): + return + node_type = node.get("nodeType") + name = str(node.get("name", "")) + if node_type == "Test Suite": + suite = name + elif node_type == "Test Case": + if name.endswith("()"): + name = name[:-2] + records.append( + { + "suite": suite, + "name": name, + "result": node.get("result"), + "durationInSeconds": node.get("durationInSeconds"), + } + ) + return + for child in node.get("children") or []: + walk(child, suite) + + for node in payload["testNodes"]: + walk(node, None) + if not records: + reasons.append("xcresult contains zero Test Case nodes") + return records + + +def hash_binaries(specs: list[str], reasons: list[str]) -> dict[str, dict[str, object]]: + binaries: dict[str, dict[str, object]] = {} + for spec in specs: + label, separator, raw_path = spec.partition("=") + if not separator or not label or not raw_path: + reasons.append(f"malformed --binary {spec!r}; expected label=PATH") + continue + path = Path(raw_path) + digest: str | None = None + try: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + except OSError as error: + reasons.append(f"cannot hash binary {label} at {path}: {error}") + binaries[label] = {"path": str(path), "sha256": digest} + return binaries + + +def main() -> int: + args = parse_args() + reasons: list[str] = [] + if args.xcodebuild_exit_code != 0: + reasons.append(f"xcodebuild test exited with code {args.xcodebuild_exit_code}") + + expected = expected_tests(args.source, reasons) + records = actual_tests(args.tests_json, reasons) + allowed_skips = set(args.allow_skip) + expected_set = set(expected) + + seen: dict[tuple[str, str], int] = {} + for record in records: + key = (str(record["suite"]), str(record["name"])) + seen[key] = seen.get(key, 0) + 1 + for suite, name in sorted(key for key, count in seen.items() if count > 1): + reasons.append(f"duplicate result for {suite}/{name}") + for suite, name in sorted(expected_set - set(seen)): + reasons.append(f"expected test did not run: {suite}/{name}") + for suite, name in sorted(set(seen) - expected_set): + reasons.append(f"unexpected test ran: {suite}/{name}") + + passed = 0 + skipped: list[str] = [] + failed: list[str] = [] + for record in records: + label = f"{record['suite']}/{record['name']}" + result = record["result"] + if result == "Passed": + passed += 1 + elif result == "Skipped": + skipped.append(label) + if record["name"] not in allowed_skips: + reasons.append(f"test skipped outside the allow-skip list: {label}") + else: + failed.append(label) + reasons.append(f"test result {result!r}: {label}") + + binaries = hash_binaries(args.binary, reasons) + status = "fail" if reasons else "pass" + receipt = { + "schema": 1, + "gate": "real-binary-chain", + "result": status, + "git_sha": args.git_sha, + "dirty_paths": args.dirty_paths, + "xcode_version": args.xcode_version.strip(), + "generated_at": datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + "xcodebuild_exit_code": args.xcodebuild_exit_code, + "expected": len(expected), + "passed": passed, + "skipped": skipped, + "failed": failed, + "reasons": reasons, + "binaries": binaries, + "tests": records, + } + line = ( + f"REAL_BINARY_GATE result={status} expected={len(expected)} " + f"passed={passed} skipped={len(skipped)} failed={len(failed)}" + ) + args.receipt.write_text(json.dumps(receipt, indent=2) + "\n") + summary = [ + "## Real-binary chain", + "", + f"`{line}`", + "", + f"- git sha: `{args.git_sha}` (dirty paths: {args.dirty_paths})", + f"- Xcode: `{args.xcode_version.strip()}`", + f"- xcodebuild exit code: {args.xcodebuild_exit_code}", + ] + for label, info in sorted(binaries.items()): + summary.append(f"- {label}: `{info['sha256']}`") + if skipped: + summary.append(f"- skipped: {', '.join(skipped)}") + if reasons: + summary.extend(["", "### Failures", ""]) + summary.extend(f"- {reason}" for reason in reasons) + args.summary.write_text("\n".join(summary) + "\n") + + for reason in reasons: + print(reason, file=sys.stderr) + print(line) + return 1 if reasons else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/run-real-binary-gate.sh b/scripts/ci/run-real-binary-gate.sh new file mode 100755 index 000000000..4f1c102d6 --- /dev/null +++ b/scripts/ci/run-real-binary-gate.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Real-binary chain gate (plan W6 item 2): build the Debug EngramCollector, +# EngramService and EngramRemoteServer tools, run the real-binary XCTest +# classes against them, and fail unless the expected test set actually ran. +# Used by .github/workflows/real-binary.yml and for a local Mac receipt. +# +# Usage: run-real-binary-gate.sh --out DIR [--derived-data DIR] +# [--cloned-source-packages DIR] +# [--timeout-scale N] +# --timeout-scale multiplies the shadow scope deadlines inside the tests +# (ENGRAM_REAL_BINARY_TIMEOUT_SCALE, 1-10, default 1) for slower hosts such as +# hosted runners; it changes no assertion, only how long the chain may take. +set -euo pipefail + +usage() { + echo "usage: $0 --out DIR [--derived-data DIR] [--cloned-source-packages DIR] [--timeout-scale N]" >&2 + exit 2 +} + +if [ "$(uname -s)" != "Darwin" ]; then + echo "run-real-binary-gate.sh requires macOS" >&2 + exit 2 +fi + +OUT="" +DD="" +SPM="" +SCALE="1" +while [ "$#" -gt 0 ]; do + case "$1" in + --out) [ "$#" -ge 2 ] || usage; OUT="$2"; shift 2 ;; + --derived-data) [ "$#" -ge 2 ] || usage; DD="$2"; shift 2 ;; + --cloned-source-packages) [ "$#" -ge 2 ] || usage; SPM="$2"; shift 2 ;; + --timeout-scale) [ "$#" -ge 2 ] || usage; SCALE="$2"; shift 2 ;; + *) usage ;; + esac +done +[ -n "$OUT" ] || usage +case "$SCALE" in + [1-9]|10) ;; + *) echo "--timeout-scale must be an integer from 1 to 10, got '$SCALE'" >&2; exit 2 ;; +esac + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +if [ -e "$OUT" ] && [ -n "$(ls -A "$OUT")" ]; then + echo "--out $OUT must not exist or must be empty" >&2 + exit 2 +fi +mkdir -p "$OUT" +chmod 700 "$OUT" +OUT="$(cd "$OUT" && pwd)" +if [ -z "$DD" ]; then + DD="$(mktemp -d /private/tmp/engram-real-binary-dd.XXXXXX)" +fi + +# Record the checkout state before building so the receipt names what was built. +git_sha="$(git -C "$ROOT" rev-parse HEAD)" +dirty="$(git -C "$ROOT" status --porcelain | wc -l | tr -d ' ')" + +common=( + -project "$ROOT/macos/Engram.xcodeproj" + -configuration Debug + -destination 'platform=macOS' + -derivedDataPath "$DD" +) +if [ -n "$SPM" ]; then + common+=(-clonedSourcePackagesDirPath "$SPM") +fi +common+=(CODE_SIGNING_ALLOWED=NO) + +for scheme in EngramCollector EngramService EngramRemoteServer; do + xcodebuild build -scheme "$scheme" "${common[@]}" 2>&1 | tee -a "$OUT/build.log" +done +xcodebuild build-for-testing -scheme EngramServiceCore "${common[@]}" 2>&1 | tee -a "$OUT/build.log" + +P="$DD/Build/Products/Debug" +COLLECTOR="$P/EngramCollector" +SERVICE="$P/EngramService" +REMOTE="$P/EngramRemoteServer" +for binary in "$COLLECTOR" "$SERVICE" "$REMOTE"; do + if [ ! -x "$binary" ]; then + echo "missing built binary: $binary" >&2 + exit 1 + fi +done + +# Same CFFIXED_USER_HOME isolation as the swift-unit EngramServiceCore run: +# invariant 6 (tests avoid production Engram data). +TEST_HOME="$(mktemp -d "$OUT/test-home.XXXXXX")" +chmod 700 "$TEST_HOME" + +# ENGRAM_SHADOW_* is deliberately never set here: the browser demo stays an +# operator opt-in and is the only test the verifier allows to skip. +echo "REAL_BINARY_GATE timeout_scale=$SCALE" +set +e +env \ + ENGRAM_REAL_BINARY_TIMEOUT_SCALE="$SCALE" \ + TEST_RUNNER_ENGRAM_REAL_BINARY_TIMEOUT_SCALE="$SCALE" \ + ENGRAM_COLLECTOR_BINARY="$COLLECTOR" \ + TEST_RUNNER_ENGRAM_COLLECTOR_BINARY="$COLLECTOR" \ + ENGRAM_SERVICE_BINARY="$SERVICE" \ + TEST_RUNNER_ENGRAM_SERVICE_BINARY="$SERVICE" \ + ENGRAM_REMOTE_SERVER_BINARY="$REMOTE" \ + TEST_RUNNER_ENGRAM_REMOTE_SERVER_BINARY="$REMOTE" \ + CFFIXED_USER_HOME="$TEST_HOME" \ + TEST_RUNNER_CFFIXED_USER_HOME="$TEST_HOME" \ + xcodebuild test-without-building -scheme EngramServiceCore "${common[@]}" \ + -only-testing:EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests \ + -only-testing:EngramServiceCoreTests/CollectorLargeHistoryBinaryTests \ + -only-testing:EngramServiceCoreTests/CollectorCLIIntegrationTests \ + -resultBundlePath "$OUT/real-binary.xcresult" \ + 2>&1 | tee "$OUT/test.log" +rc=${PIPESTATUS[0]} + +if [ -d "$OUT/real-binary.xcresult" ]; then + xcrun xcresulttool get test-results tests --path "$OUT/real-binary.xcresult" >"$OUT/tests.json" +else + : >"$OUT/tests.json" +fi + +tests_dir="$ROOT/macos/EngramServiceCoreTests" +python3 "$ROOT/scripts/ci/check-real-binary-results.py" \ + --tests-json "$OUT/tests.json" \ + --source "$tests_dir/CollectorBinaryShadowIntegrationTests.swift" \ + --source "$tests_dir/CollectorCLIIntegrationTests.swift" \ + --allow-skip testRealBinaryHTTPSBrowserDemoWithBoundedHold \ + --xcodebuild-exit-code "$rc" \ + --binary collector="$COLLECTOR" \ + --binary service="$SERVICE" \ + --binary remote-server="$REMOTE" \ + --git-sha "$git_sha" \ + --xcode-version "$(xcodebuild -version | tr '\n' ' ')" \ + --dirty-paths "$dirty" \ + --receipt "$OUT/receipt.json" \ + --summary "$OUT/receipt.md" diff --git a/tests/scripts/ci-workflow.test.ts b/tests/scripts/ci-workflow.test.ts index ea459620c..8144da57c 100644 --- a/tests/scripts/ci-workflow.test.ts +++ b/tests/scripts/ci-workflow.test.ts @@ -21,6 +21,14 @@ const perfWorkflow = readFileSync( resolve(repoRoot, '.github/workflows/perf.yml'), 'utf8', ); +const realBinaryWorkflow = readFileSync( + resolve(repoRoot, '.github/workflows/real-binary.yml'), + 'utf8', +); +const realBinaryGateScript = readFileSync( + resolve(repoRoot, 'scripts/ci/run-real-binary-gate.sh'), + 'utf8', +); const xcodegenInstallerPath = resolve( repoRoot, 'scripts/ci/install-xcodegen.sh', @@ -70,12 +78,14 @@ const xcodegenWorkflows = [ releaseWorkflow, codeqlWorkflow, perfWorkflow, + realBinaryWorkflow, ]; const allWorkflows = [ testWorkflow, releaseWorkflow, codeqlWorkflow, perfWorkflow, + realBinaryWorkflow, dependencyReviewWorkflow, ]; const actionPins = { @@ -280,6 +290,8 @@ describe('CI workflow hardening', () => { it('keeps pull-request code off persistent self-hosted runners', () => { expect(testWorkflow).not.toContain('runs-on: [self-hosted'); expect(codeqlWorkflow).not.toContain('runs-on: [self-hosted'); + expect(realBinaryWorkflow).not.toContain('runs-on: [self-hosted'); + expect(realBinaryWorkflow).toContain('runs-on: macos-15'); const macosVitest = testWorkflow.slice( testWorkflow.indexOf(' macos-vitest:'), @@ -879,6 +891,77 @@ describe('Perf workflow', () => { }); }); +describe('Real-binary workflow', () => { + const parsed = parseDocument(realBinaryWorkflow).toJS() as { + on?: { + schedule?: Array<{ cron?: string }>; + pull_request?: { paths?: string[] }; + } & Record; + }; + + it('documents weekly/on-demand policy as non-gating', () => { + expect(realBinaryWorkflow).toMatch( + /^# Policy: weekly\/workflow_dispatch plus pull requests that change this gate; not a required merge gate\.\nname: Real-binary chain\n/, + ); + expect(parsed.on?.schedule).toEqual([{ cron: '17 20 * * 0' }]); + expect(parsed.on).toHaveProperty('workflow_dispatch'); + expect(parsed.on?.pull_request?.paths).toEqual([ + '.github/workflows/real-binary.yml', + 'scripts/ci/run-real-binary-gate.sh', + 'scripts/ci/check-real-binary-results.py', + 'macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift', + 'macos/EngramServiceCoreTests/CollectorCLIIntegrationTests.swift', + ]); + expect(realBinaryWorkflow).toContain('runs-on: macos-15'); + expect(realBinaryWorkflow).toContain('timeout-minutes: 60'); + expect(realBinaryWorkflow).toContain( + 'group: real-binary-$' + '{{ github.ref }}-$' + '{{ github.event_name }}', + ); + expect(realBinaryWorkflow).toContain('cancel-in-progress: true'); + expect(realBinaryWorkflow).toContain( + 'bash scripts/ci/run-real-binary-gate.sh', + ); + expect(realBinaryWorkflow).toContain('GITHUB_STEP_SUMMARY'); + // Not wired into the required Tests workflow or its ci-gate. + expect(testWorkflow).not.toContain('real-binary'); + }); + + it('runs only the binary classes with explicit binaries and never opts into the browser hold', () => { + for (const target of [ + 'CollectorBinaryShadowIntegrationTests', + 'CollectorLargeHistoryBinaryTests', + 'CollectorCLIIntegrationTests', + ]) { + expect(realBinaryGateScript).toContain( + `-only-testing:EngramServiceCoreTests/${target}`, + ); + } + for (const variable of [ + 'TEST_RUNNER_ENGRAM_COLLECTOR_BINARY', + 'TEST_RUNNER_ENGRAM_SERVICE_BINARY', + 'TEST_RUNNER_ENGRAM_REMOTE_SERVER_BINARY', + 'TEST_RUNNER_CFFIXED_USER_HOME', + 'TEST_RUNNER_ENGRAM_REAL_BINARY_TIMEOUT_SCALE', + ]) { + expect(realBinaryGateScript).toContain(variable); + } + // Hosted runners get longer scope deadlines, never different assertions. + expect(realBinaryWorkflow).toContain('--timeout-scale 6'); + expect(realBinaryGateScript).toContain('test-without-building'); + expect(realBinaryGateScript).toContain( + 'xcresulttool get test-results tests', + ); + expect(realBinaryGateScript).toContain('check-real-binary-results.py'); + expect(realBinaryGateScript).toContain( + '--allow-skip testRealBinaryHTTPSBrowserDemoWithBoundedHold', + ); + for (const text of [realBinaryWorkflow, realBinaryGateScript]) { + expect(text).not.toMatch(/^(?!\s*#).*ENGRAM_SHADOW_/m); + expect(text).not.toContain('upload-artifact'); + } + }); +}); + describe('Dependency Review workflow', () => { it('fail-closes pull requests that introduce moderate-or-higher vulnerabilities', () => { expect(dependencyReviewWorkflow).toContain('name: Dependency Review'); diff --git a/tests/scripts/real-binary-gate.test.ts b/tests/scripts/real-binary-gate.test.ts new file mode 100644 index 000000000..e491ff139 --- /dev/null +++ b/tests/scripts/real-binary-gate.test.ts @@ -0,0 +1,348 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const repoRoot = resolve(import.meta.dirname, '../..'); +const gate = resolve(repoRoot, 'scripts/ci/check-real-binary-results.py'); +const python = process.env.PYTHON3 ?? 'python3'; +const testsDir = resolve(repoRoot, 'macos/EngramServiceCoreTests'); +const realSources = [ + join(testsDir, 'CollectorBinaryShadowIntegrationTests.swift'), + join(testsDir, 'CollectorCLIIntegrationTests.swift'), +]; +const browserDemo = 'testRealBinaryHTTPSBrowserDemoWithBoundedHold'; + +type TestCase = { suite: string; name: string; result: string }; + +// Independent of the Python parser: XCTestCase classes and their extensions. +function realExpectedTests(): TestCase[] { + const tests: TestCase[] = []; + for (const source of realSources) { + const testClasses = new Set(); + let current: string | null = null; + for (const line of readFileSync(source, 'utf8').split('\n')) { + const cls = line.match(/^(?:final )?class (\w+): XCTestCase\b/); + const ext = line.match(/^extension (\w+)\b/); + if (cls) { + current = cls[1]; + testClasses.add(current); + } else if (ext) { + current = testClasses.has(ext[1]) ? ext[1] : null; + } else if ( + /^\S/.test(line) && + /\b(class|struct|enum|extension)\b/.test(line) + ) { + current = null; + } else { + const test = line.match(/^ {4}func (test\w+)\(/); + if (test && current) { + tests.push({ suite: current, name: test[1], result: 'Passed' }); + } + } + } + } + return tests; +} + +function xcresult(cases: TestCase[]) { + const suites = new Map(); + for (const testCase of cases) { + suites.set(testCase.suite, [ + ...(suites.get(testCase.suite) ?? []), + testCase, + ]); + } + return { + devices: [], + testNodes: [ + { + nodeType: 'Test Plan', + name: 'EngramServiceCore', + children: [ + { + nodeType: 'Unit test bundle', + name: 'EngramServiceCoreTests', + children: [...suites].map(([suite, members]) => ({ + nodeType: 'Test Suite', + name: suite, + children: members.map((member) => ({ + nodeType: 'Test Case', + name: `${member.name}()`, + nodeIdentifier: `${suite}/${member.name}()`, + result: member.result, + durationInSeconds: 1.5, + children: + member.result === 'Skipped' + ? [{ nodeType: 'Failure Message', name: 'skipped' }] + : [], + })), + })), + }, + ], + }, + ], + }; +} + +let workdir: string; + +function runGate( + json: unknown, + options: { exitCode?: string; sources?: string[] } = {}, +) { + const testsJson = join(workdir, 'tests.json'); + writeFileSync( + testsJson, + typeof json === 'string' ? json : JSON.stringify(json), + ); + const binary = join(workdir, 'EngramCollector'); + writeFileSync(binary, 'collector-bytes'); + const receipt = join(workdir, 'receipt.json'); + const summary = join(workdir, 'receipt.md'); + const args = [ + gate, + '--tests-json', + testsJson, + ...(options.sources ?? realSources).flatMap((source) => [ + '--source', + source, + ]), + '--allow-skip', + browserDemo, + '--xcodebuild-exit-code', + options.exitCode ?? '0', + '--binary', + `collector=${binary}`, + '--git-sha', + 'deadbeef', + '--xcode-version', + 'Xcode 26.4 Build version 17E192 ', + '--dirty-paths', + '0', + '--receipt', + receipt, + '--summary', + summary, + ]; + const result = spawnSync(python, args, { encoding: 'utf8' }); + return { result, receipt, summary }; +} + +function withResult( + cases: TestCase[], + name: string, + result: string, +): TestCase[] { + return cases.map((testCase) => + testCase.name === name ? { ...testCase, result } : testCase, + ); +} + +describe('real-binary chain result gate', () => { + beforeEach(() => { + workdir = mkdtempSync(join(tmpdir(), 'engram-real-binary-gate-')); + }); + + afterEach(() => { + rmSync(workdir, { recursive: true, force: true }); + }); + + it('parses the expected real-binary test set from the Swift sources', () => { + const expected = realExpectedTests(); + const bySuite = Object.fromEntries( + [ + 'CollectorBinaryShadowIntegrationTests', + 'CollectorLargeHistoryBinaryTests', + 'CollectorCLIIntegrationTests', + ].map((suite) => [ + suite, + expected.filter((test) => test.suite === suite).length, + ]), + ); + expect(bySuite).toEqual({ + CollectorBinaryShadowIntegrationTests: 32, + CollectorLargeHistoryBinaryTests: 1, + CollectorCLIIntegrationTests: 17, + }); + expect(expected.map((test) => test.name)).toContain(browserDemo); + }); + + it('accepts every test passed plus the opt-in browser skip and writes a receipt', () => { + const cases = withResult(realExpectedTests(), browserDemo, 'Skipped'); + const { result, receipt, summary } = runGate(xcresult(cases)); + expect(result.stderr).toBe(''); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0', + ); + const written = JSON.parse(readFileSync(receipt, 'utf8')); + expect(written).toMatchObject({ + schema: 1, + result: 'pass', + git_sha: 'deadbeef', + dirty_paths: 0, + xcode_version: 'Xcode 26.4 Build version 17E192', + expected: 50, + passed: 49, + skipped: [`CollectorBinaryShadowIntegrationTests/${browserDemo}`], + failed: [], + reasons: [], + }); + expect(written.binaries.collector.sha256).toBe( + createHash('sha256').update('collector-bytes').digest('hex'), + ); + expect(written.tests).toHaveLength(50); + expect(readFileSync(summary, 'utf8')).toContain( + 'REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0', + ); + }); + + it('accepts the browser demo when it passed', () => { + const { result } = runGate(xcresult(realExpectedTests())); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=pass expected=50 passed=50 skipped=0 failed=0', + ); + }); + + // Repro for docs/TODO.md W6.2 (real-binary chain in a recorded gate): when + // the binary-path variables never reach the test runner every real-binary + // test reports Skipped and xcodebuild still exits 0, which looked green. + it('rejects a run where every real-binary test skipped (repro)', () => { + const cases = realExpectedTests().map((testCase) => ({ + ...testCase, + result: 'Skipped', + })); + const { result, receipt } = runGate(xcresult(cases)); + expect(result.status).toBe(1); + expect(result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=fail expected=50 passed=0 skipped=50 failed=0', + ); + expect(result.stderr).toContain( + 'test skipped outside the allow-skip list: CollectorCLIIntegrationTests/', + ); + expect(JSON.parse(readFileSync(receipt, 'utf8')).result).toBe('fail'); + }); + + it('rejects a skipped test outside the allow-skip list', () => { + const cases = withResult( + realExpectedTests(), + 'testLargeHistoryIsByteExactOnBothReplicasAndCompleteAcrossWebPages', + 'Skipped', + ); + const { result } = runGate(xcresult(cases)); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + 'test skipped outside the allow-skip list: CollectorLargeHistoryBinaryTests/testLargeHistoryIsByteExactOnBothReplicasAndCompleteAcrossWebPages', + ); + }); + + it('rejects a failed test and a nonzero xcodebuild exit code', () => { + const failed = withResult( + realExpectedTests(), + 'testRealPiNineMiBMessageReachesBothReplicasAndHQWebIPC', + 'Failed', + ); + const failedRun = runGate(xcresult(failed), { exitCode: '65' }); + expect(failedRun.result.status).toBe(1); + expect(failedRun.result.stdout).toContain('result=fail'); + expect(failedRun.result.stdout).toContain('failed=1'); + expect(failedRun.result.stderr).toContain( + "test result 'Failed': CollectorBinaryShadowIntegrationTests/testRealPiNineMiBMessageReachesBothReplicasAndHQWebIPC", + ); + + const exitOnly = runGate(xcresult(realExpectedTests()), { + exitCode: '65', + }); + expect(exitOnly.result.status).toBe(1); + expect(exitOnly.result.stderr).toContain( + 'xcodebuild test exited with code 65', + ); + }); + + it('rejects missing or unexpected tests', () => { + const expected = realExpectedTests(); + const missing = runGate( + xcresult( + expected.filter( + (test) => test.suite !== 'CollectorCLIIntegrationTests', + ), + ), + ); + expect(missing.result.status).toBe(1); + expect(missing.result.stderr).toContain( + 'expected test did not run: CollectorCLIIntegrationTests/', + ); + + const extra = runGate( + xcresult([ + ...expected, + { + suite: 'CollectorBinaryPerformanceTests', + name: 'testSomethingElse', + result: 'Passed', + }, + ]), + ); + expect(extra.result.status).toBe(1); + expect(extra.result.stderr).toContain( + 'unexpected test ran: CollectorBinaryPerformanceTests/testSomethingElse', + ); + + const source = join(workdir, 'Inline.swift'); + writeFileSync( + source, + [ + 'final class InlineTests: XCTestCase {', + ' func testRenamed() async throws {}', + '}', + 'private struct Helper {', + ' func testHelperIsNotATest() {}', + '}', + 'extension InlineTests {', + ' func testInExtension() {}', + '}', + '', + ].join('\n'), + ); + const renamed = runGate( + xcresult([ + { suite: 'InlineTests', name: 'testOldName', result: 'Passed' }, + { suite: 'InlineTests', name: 'testInExtension', result: 'Passed' }, + ]), + { sources: [source] }, + ); + expect(renamed.result.status).toBe(1); + expect(renamed.result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=fail expected=2 passed=2 skipped=0 failed=0', + ); + expect(renamed.result.stderr).toContain( + 'expected test did not run: InlineTests/testRenamed', + ); + expect(renamed.result.stderr).toContain( + 'unexpected test ran: InlineTests/testOldName', + ); + expect(renamed.result.stderr).not.toContain('testHelperIsNotATest'); + }); + + it('rejects an empty or malformed result JSON', () => { + const empty = runGate(''); + expect(empty.result.status).toBe(1); + expect(empty.result.stderr).toContain('cannot parse xcresult tests JSON'); + + const noNodes = runGate({ testNodes: [] }); + expect(noNodes.result.status).toBe(1); + expect(noNodes.result.stderr).toContain( + 'xcresult contains zero Test Case nodes', + ); + + const wrongShape = runGate({ devices: [] }); + expect(wrongShape.result.status).toBe(1); + expect(wrongShape.result.stderr).toContain( + 'xcresult tests JSON has no testNodes array', + ); + }); +});