From b2e8ef2af8587d0744aec7e1ef01a644216963aa Mon Sep 17 00:00:00 2001 From: reviewer Date: Sun, 18 Oct 2026 07:52:28 +0800 Subject: [PATCH 1/6] ci: add non-required real-binary chain gate (W6.2) Add .github/workflows/real-binary.yml (weekly, workflow_dispatch, and pull requests that touch the gate's own files) plus scripts/ci/run-real-binary-gate.sh, which builds the Debug EngramCollector, EngramService and EngramRemoteServer tools, runs CollectorBinaryShadowIntegrationTests, CollectorLargeHistoryBinaryTests and CollectorCLIIntegrationTests with the explicit binary-path variables and an isolated CFFIXED_USER_HOME, and exports the xcresult test tree. scripts/ci/check-real-binary-results.py fails unless the exact expected (class, test) set parsed from the Swift sources ran and passed; only the opt-in browser demo may skip, so an all-skipped run is red. The workflow is not part of test.yml or the ci-gate contract. Claude-Session: https://claude.ai/code/session_01RE2f7JXQBa8xzoFrxuHS6v --- .github/workflows/real-binary.yml | 66 +++++++ scripts/ci/check-real-binary-results.py | 230 ++++++++++++++++++++++++ scripts/ci/run-real-binary-gate.sh | 122 +++++++++++++ 3 files changed, 418 insertions(+) create mode 100644 .github/workflows/real-binary.yml create mode 100755 scripts/ci/check-real-binary-results.py create mode 100755 scripts/ci/run-real-binary-gate.sh diff --git a/.github/workflows/real-binary.yml b/.github/workflows/real-binary.yml new file mode 100644 index 000000000..ebe64d5eb --- /dev/null +++ b/.github/workflows/real-binary.yml @@ -0,0 +1,66 @@ +# Policy: weekly/workflow_dispatch plus pull requests that change this gate; not a required merge gate. +name: Real-binary chain + +on: + schedule: + - cron: "17 20 * * 0" + workflow_dispatch: + pull_request: + paths: + - .github/workflows/real-binary.yml + - scripts/ci/run-real-binary-gate.sh + - scripts/ci/check-real-binary-results.py + - macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift + - macos/EngramServiceCoreTests/CollectorCLIIntegrationTests.swift + +permissions: + contents: read + +concurrency: + group: real-binary-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +env: + XCODEGEN_SHA256: "090ec29491aad50aec10631bf6e62253fed733c50f3aab0f5ffc86bc170bdbef" + XCODEGEN_VERSION: "2.45.4" + +jobs: + real-binary-chain: + name: Real-binary chain + runs-on: macos-15 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install xcodegen + run: bash scripts/ci/install-xcodegen.sh "$XCODEGEN_VERSION" "$XCODEGEN_SHA256" + + - name: Cache SPM packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/Library/Caches/EngramSourcePackages + key: spm-${{ runner.os }}-${{ runner.arch }}-hosted-realbinary-${{ hashFiles('macos/project.yml', 'macos/Engram.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} + restore-keys: spm-${{ runner.os }}-${{ runner.arch }}-hosted-realbinary- + + - name: Generate Xcode project + run: scripts/check-xcodeproj-drift.sh + + - name: Build binaries and run the real-binary chain + run: | + bash scripts/ci/run-real-binary-gate.sh \ + --out "$RUNNER_TEMP/engram-real-binary-out" \ + --derived-data "$RUNNER_TEMP/engram-real-binary-derived" \ + --cloned-source-packages "$HOME/Library/Caches/EngramSourcePackages" + + - name: Publish receipt + if: always() + run: | + out="$RUNNER_TEMP/engram-real-binary-out" + if [ -f "$out/receipt.md" ]; then + cat "$out/receipt.md" >> "$GITHUB_STEP_SUMMARY" + else + echo "No real-binary receipt was written." >> "$GITHUB_STEP_SUMMARY" + fi + if [ -f "$out/receipt.json" ]; then + cat "$out/receipt.json" + fi diff --git a/scripts/ci/check-real-binary-results.py b/scripts/ci/check-real-binary-results.py new file mode 100755 index 000000000..03e6084e2 --- /dev/null +++ b/scripts/ci/check-real-binary-results.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Fail the real-binary chain gate unless the expected XCTest set actually ran. + +The real-binary suites skip themselves when the explicit binary-path +environment variables do not reach the test runner, and a fully skipped run +still exits 0. This verifier therefore compares the exact (class, test) set +declared in the Swift sources with the Test Case nodes exported by +`xcrun xcresulttool get test-results tests`, and accepts only Passed results +(plus Skipped for explicitly allowed opt-in tests). +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import re +import sys +from pathlib import Path + + +CLASS_PATTERN = re.compile(r"^(?:final )?class (\w+): XCTestCase\b") +EXTENSION_PATTERN = re.compile(r"^extension (\w+)\b") +OTHER_TOP_LEVEL_PATTERN = re.compile( + r"^(?:(?:private|fileprivate|public|internal|final|@\w+)\s+)*" + r"(?:class|struct|enum|extension|actor|protocol)\b" +) +TEST_PATTERN = re.compile(r"^ {4}func (test\w+)\(") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--tests-json", required=True, type=Path) + parser.add_argument("--source", required=True, type=Path, action="append") + parser.add_argument("--allow-skip", action="append", default=[]) + parser.add_argument("--xcodebuild-exit-code", required=True, type=int) + parser.add_argument("--binary", action="append", default=[]) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--xcode-version", required=True) + parser.add_argument("--dirty-paths", type=int, default=None) + parser.add_argument("--receipt", required=True, type=Path) + parser.add_argument("--summary", required=True, type=Path) + return parser.parse_args() + + +def expected_tests(sources: list[Path], reasons: list[str]) -> list[tuple[str, str]]: + found: list[tuple[str, str]] = [] + for source in sources: + try: + lines = source.read_text().splitlines() + except OSError as error: + reasons.append(f"cannot read test source {source}: {error}") + continue + test_classes: set[str] = set() + current: str | None = None + for line in lines: + class_match = CLASS_PATTERN.match(line) + if class_match is not None: + current = class_match.group(1) + test_classes.add(current) + continue + extension_match = EXTENSION_PATTERN.match(line) + if extension_match is not None: + name = extension_match.group(1) + current = name if name in test_classes else None + continue + if OTHER_TOP_LEVEL_PATTERN.match(line): + current = None + continue + test_match = TEST_PATTERN.match(line) + if test_match is not None and current is not None: + found.append((current, test_match.group(1))) + if not found: + reasons.append("no XCTest methods found in the test sources") + duplicates = sorted({item for item in found if found.count(item) > 1}) + for suite, name in duplicates: + reasons.append(f"duplicate expected test {suite}/{name}") + return sorted(set(found)) + + +def actual_tests(path: Path, reasons: list[str]) -> list[dict[str, object]]: + try: + payload = json.loads(path.read_text()) + except (OSError, ValueError) as error: + reasons.append(f"cannot parse xcresult tests JSON {path}: {error}") + return [] + if not isinstance(payload, dict) or not isinstance(payload.get("testNodes"), list): + reasons.append("xcresult tests JSON has no testNodes array") + return [] + + records: list[dict[str, object]] = [] + + def walk(node: object, suite: str | None) -> None: + if not isinstance(node, dict): + return + node_type = node.get("nodeType") + name = str(node.get("name", "")) + if node_type == "Test Suite": + suite = name + elif node_type == "Test Case": + if name.endswith("()"): + name = name[:-2] + records.append( + { + "suite": suite, + "name": name, + "result": node.get("result"), + "durationInSeconds": node.get("durationInSeconds"), + } + ) + return + for child in node.get("children") or []: + walk(child, suite) + + for node in payload["testNodes"]: + walk(node, None) + if not records: + reasons.append("xcresult contains zero Test Case nodes") + return records + + +def hash_binaries(specs: list[str], reasons: list[str]) -> dict[str, dict[str, object]]: + binaries: dict[str, dict[str, object]] = {} + for spec in specs: + label, separator, raw_path = spec.partition("=") + if not separator or not label or not raw_path: + reasons.append(f"malformed --binary {spec!r}; expected label=PATH") + continue + path = Path(raw_path) + digest: str | None = None + try: + digest = hashlib.sha256(path.read_bytes()).hexdigest() + except OSError as error: + reasons.append(f"cannot hash binary {label} at {path}: {error}") + binaries[label] = {"path": str(path), "sha256": digest} + return binaries + + +def main() -> int: + args = parse_args() + reasons: list[str] = [] + if args.xcodebuild_exit_code != 0: + reasons.append(f"xcodebuild test exited with code {args.xcodebuild_exit_code}") + + expected = expected_tests(args.source, reasons) + records = actual_tests(args.tests_json, reasons) + allowed_skips = set(args.allow_skip) + expected_set = set(expected) + + seen: dict[tuple[str, str], int] = {} + for record in records: + key = (str(record["suite"]), str(record["name"])) + seen[key] = seen.get(key, 0) + 1 + for suite, name in sorted(key for key, count in seen.items() if count > 1): + reasons.append(f"duplicate result for {suite}/{name}") + for suite, name in sorted(expected_set - set(seen)): + reasons.append(f"expected test did not run: {suite}/{name}") + for suite, name in sorted(set(seen) - expected_set): + reasons.append(f"unexpected test ran: {suite}/{name}") + + passed = 0 + skipped: list[str] = [] + failed: list[str] = [] + for record in records: + label = f"{record['suite']}/{record['name']}" + result = record["result"] + if result == "Passed": + passed += 1 + elif result == "Skipped": + skipped.append(label) + if record["name"] not in allowed_skips: + reasons.append(f"test skipped outside the allow-skip list: {label}") + else: + failed.append(label) + reasons.append(f"test result {result!r}: {label}") + + binaries = hash_binaries(args.binary, reasons) + status = "fail" if reasons else "pass" + receipt = { + "schema": 1, + "gate": "real-binary-chain", + "result": status, + "git_sha": args.git_sha, + "dirty_paths": args.dirty_paths, + "xcode_version": args.xcode_version.strip(), + "generated_at": datetime.datetime.now(datetime.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + "xcodebuild_exit_code": args.xcodebuild_exit_code, + "expected": len(expected), + "passed": passed, + "skipped": skipped, + "failed": failed, + "reasons": reasons, + "binaries": binaries, + "tests": records, + } + line = ( + f"REAL_BINARY_GATE result={status} expected={len(expected)} " + f"passed={passed} skipped={len(skipped)} failed={len(failed)}" + ) + args.receipt.write_text(json.dumps(receipt, indent=2) + "\n") + summary = [ + "## Real-binary chain", + "", + f"`{line}`", + "", + f"- git sha: `{args.git_sha}` (dirty paths: {args.dirty_paths})", + f"- Xcode: `{args.xcode_version.strip()}`", + f"- xcodebuild exit code: {args.xcodebuild_exit_code}", + ] + for label, info in sorted(binaries.items()): + summary.append(f"- {label}: `{info['sha256']}`") + if skipped: + summary.append(f"- skipped: {', '.join(skipped)}") + if reasons: + summary.extend(["", "### Failures", ""]) + summary.extend(f"- {reason}" for reason in reasons) + args.summary.write_text("\n".join(summary) + "\n") + + for reason in reasons: + print(reason, file=sys.stderr) + print(line) + return 1 if reasons else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/run-real-binary-gate.sh b/scripts/ci/run-real-binary-gate.sh new file mode 100755 index 000000000..8a8d4edf5 --- /dev/null +++ b/scripts/ci/run-real-binary-gate.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Real-binary chain gate (plan W6 item 2): build the Debug EngramCollector, +# EngramService and EngramRemoteServer tools, run the real-binary XCTest +# classes against them, and fail unless the expected test set actually ran. +# Used by .github/workflows/real-binary.yml and for a local Mac receipt. +# +# Usage: run-real-binary-gate.sh --out DIR [--derived-data DIR] +# [--cloned-source-packages DIR] +set -euo pipefail + +usage() { + echo "usage: $0 --out DIR [--derived-data DIR] [--cloned-source-packages DIR]" >&2 + exit 2 +} + +if [ "$(uname -s)" != "Darwin" ]; then + echo "run-real-binary-gate.sh requires macOS" >&2 + exit 2 +fi + +OUT="" +DD="" +SPM="" +while [ "$#" -gt 0 ]; do + case "$1" in + --out) [ "$#" -ge 2 ] || usage; OUT="$2"; shift 2 ;; + --derived-data) [ "$#" -ge 2 ] || usage; DD="$2"; shift 2 ;; + --cloned-source-packages) [ "$#" -ge 2 ] || usage; SPM="$2"; shift 2 ;; + *) usage ;; + esac +done +[ -n "$OUT" ] || usage + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +if [ -e "$OUT" ] && [ -n "$(ls -A "$OUT")" ]; then + echo "--out $OUT must not exist or must be empty" >&2 + exit 2 +fi +mkdir -p "$OUT" +chmod 700 "$OUT" +OUT="$(cd "$OUT" && pwd)" +if [ -z "$DD" ]; then + DD="$(mktemp -d /private/tmp/engram-real-binary-dd.XXXXXX)" +fi + +# Record the checkout state before building so the receipt names what was built. +git_sha="$(git -C "$ROOT" rev-parse HEAD)" +dirty="$(git -C "$ROOT" status --porcelain | wc -l | tr -d ' ')" + +common=( + -project "$ROOT/macos/Engram.xcodeproj" + -configuration Debug + -destination 'platform=macOS' + -derivedDataPath "$DD" +) +if [ -n "$SPM" ]; then + common+=(-clonedSourcePackagesDirPath "$SPM") +fi +common+=(CODE_SIGNING_ALLOWED=NO) + +for scheme in EngramCollector EngramService EngramRemoteServer; do + xcodebuild build -scheme "$scheme" "${common[@]}" 2>&1 | tee -a "$OUT/build.log" +done +xcodebuild build-for-testing -scheme EngramServiceCore "${common[@]}" 2>&1 | tee -a "$OUT/build.log" + +P="$DD/Build/Products/Debug" +COLLECTOR="$P/EngramCollector" +SERVICE="$P/EngramService" +REMOTE="$P/EngramRemoteServer" +for binary in "$COLLECTOR" "$SERVICE" "$REMOTE"; do + if [ ! -x "$binary" ]; then + echo "missing built binary: $binary" >&2 + exit 1 + fi +done + +# Same CFFIXED_USER_HOME isolation as the swift-unit EngramServiceCore run: +# invariant 6 (tests avoid production Engram data). +TEST_HOME="$(mktemp -d "$OUT/test-home.XXXXXX")" +chmod 700 "$TEST_HOME" + +# ENGRAM_SHADOW_* is deliberately never set here: the browser demo stays an +# operator opt-in and is the only test the verifier allows to skip. +set +e +env \ + ENGRAM_COLLECTOR_BINARY="$COLLECTOR" \ + TEST_RUNNER_ENGRAM_COLLECTOR_BINARY="$COLLECTOR" \ + ENGRAM_SERVICE_BINARY="$SERVICE" \ + TEST_RUNNER_ENGRAM_SERVICE_BINARY="$SERVICE" \ + ENGRAM_REMOTE_SERVER_BINARY="$REMOTE" \ + TEST_RUNNER_ENGRAM_REMOTE_SERVER_BINARY="$REMOTE" \ + CFFIXED_USER_HOME="$TEST_HOME" \ + TEST_RUNNER_CFFIXED_USER_HOME="$TEST_HOME" \ + xcodebuild test-without-building -scheme EngramServiceCore "${common[@]}" \ + -only-testing:EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests \ + -only-testing:EngramServiceCoreTests/CollectorLargeHistoryBinaryTests \ + -only-testing:EngramServiceCoreTests/CollectorCLIIntegrationTests \ + -resultBundlePath "$OUT/real-binary.xcresult" \ + 2>&1 | tee "$OUT/test.log" +rc=${PIPESTATUS[0]} + +if [ -d "$OUT/real-binary.xcresult" ]; then + xcrun xcresulttool get test-results tests --path "$OUT/real-binary.xcresult" >"$OUT/tests.json" +else + : >"$OUT/tests.json" +fi + +tests_dir="$ROOT/macos/EngramServiceCoreTests" +python3 "$ROOT/scripts/ci/check-real-binary-results.py" \ + --tests-json "$OUT/tests.json" \ + --source "$tests_dir/CollectorBinaryShadowIntegrationTests.swift" \ + --source "$tests_dir/CollectorCLIIntegrationTests.swift" \ + --allow-skip testRealBinaryHTTPSBrowserDemoWithBoundedHold \ + --xcodebuild-exit-code "$rc" \ + --binary collector="$COLLECTOR" \ + --binary service="$SERVICE" \ + --binary remote-server="$REMOTE" \ + --git-sha "$git_sha" \ + --xcode-version "$(xcodebuild -version | tr '\n' ' ')" \ + --dirty-paths "$dirty" \ + --receipt "$OUT/receipt.json" \ + --summary "$OUT/receipt.md" From 9b6b7829c49fecb0a5752bf9823f85c19391c5b5 Mon Sep 17 00:00:00 2001 From: reviewer Date: Sun, 18 Oct 2026 07:54:59 +0800 Subject: [PATCH 2/6] test(ci): pin the real-binary gate verifier and workflow contract tests/scripts/real-binary-gate.test.ts drives check-real-binary-results.py with synthetic xcresult trees: it accepts 49 passed plus the opt-in browser skip (and 50 passed), and rejects an all-skipped run (repro for TODO W6.2), a skip outside the allow list, a failed test, a nonzero xcodebuild exit, missing/renamed/extra tests and empty or malformed JSON. ci-workflow.test.ts adds real-binary.yml to the YAML/SHA-pin/xcodegen contracts and the self-hosted ban, and pins its policy header, triggers, runner, timeout, concurrency and the runner script's -only-testing, TEST_RUNNER_ binary variables and verifier call. Claude-Session: https://claude.ai/code/session_01RE2f7JXQBa8xzoFrxuHS6v --- tests/scripts/ci-workflow.test.ts | 80 ++++++ tests/scripts/real-binary-gate.test.ts | 348 +++++++++++++++++++++++++ 2 files changed, 428 insertions(+) create mode 100644 tests/scripts/real-binary-gate.test.ts diff --git a/tests/scripts/ci-workflow.test.ts b/tests/scripts/ci-workflow.test.ts index ea459620c..4e060e233 100644 --- a/tests/scripts/ci-workflow.test.ts +++ b/tests/scripts/ci-workflow.test.ts @@ -21,6 +21,14 @@ const perfWorkflow = readFileSync( resolve(repoRoot, '.github/workflows/perf.yml'), 'utf8', ); +const realBinaryWorkflow = readFileSync( + resolve(repoRoot, '.github/workflows/real-binary.yml'), + 'utf8', +); +const realBinaryGateScript = readFileSync( + resolve(repoRoot, 'scripts/ci/run-real-binary-gate.sh'), + 'utf8', +); const xcodegenInstallerPath = resolve( repoRoot, 'scripts/ci/install-xcodegen.sh', @@ -70,12 +78,14 @@ const xcodegenWorkflows = [ releaseWorkflow, codeqlWorkflow, perfWorkflow, + realBinaryWorkflow, ]; const allWorkflows = [ testWorkflow, releaseWorkflow, codeqlWorkflow, perfWorkflow, + realBinaryWorkflow, dependencyReviewWorkflow, ]; const actionPins = { @@ -280,6 +290,8 @@ describe('CI workflow hardening', () => { it('keeps pull-request code off persistent self-hosted runners', () => { expect(testWorkflow).not.toContain('runs-on: [self-hosted'); expect(codeqlWorkflow).not.toContain('runs-on: [self-hosted'); + expect(realBinaryWorkflow).not.toContain('runs-on: [self-hosted'); + expect(realBinaryWorkflow).toContain('runs-on: macos-15'); const macosVitest = testWorkflow.slice( testWorkflow.indexOf(' macos-vitest:'), @@ -879,6 +891,74 @@ describe('Perf workflow', () => { }); }); +describe('Real-binary workflow', () => { + const parsed = parseDocument(realBinaryWorkflow).toJS() as { + on?: { + schedule?: Array<{ cron?: string }>; + pull_request?: { paths?: string[] }; + } & Record; + }; + + it('documents weekly/on-demand policy as non-gating', () => { + expect(realBinaryWorkflow).toMatch( + /^# Policy: weekly\/workflow_dispatch plus pull requests that change this gate; not a required merge gate\.\nname: Real-binary chain\n/, + ); + expect(parsed.on?.schedule).toEqual([{ cron: '17 20 * * 0' }]); + expect(parsed.on).toHaveProperty('workflow_dispatch'); + expect(parsed.on?.pull_request?.paths).toEqual([ + '.github/workflows/real-binary.yml', + 'scripts/ci/run-real-binary-gate.sh', + 'scripts/ci/check-real-binary-results.py', + 'macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift', + 'macos/EngramServiceCoreTests/CollectorCLIIntegrationTests.swift', + ]); + expect(realBinaryWorkflow).toContain('runs-on: macos-15'); + expect(realBinaryWorkflow).toContain('timeout-minutes: 45'); + expect(realBinaryWorkflow).toContain( + 'group: real-binary-$' + '{{ github.ref }}-$' + '{{ github.event_name }}', + ); + expect(realBinaryWorkflow).toContain('cancel-in-progress: true'); + expect(realBinaryWorkflow).toContain( + 'bash scripts/ci/run-real-binary-gate.sh', + ); + expect(realBinaryWorkflow).toContain('GITHUB_STEP_SUMMARY'); + // Not wired into the required Tests workflow or its ci-gate. + expect(testWorkflow).not.toContain('real-binary'); + }); + + it('runs only the binary classes with explicit binaries and never opts into the browser hold', () => { + for (const target of [ + 'CollectorBinaryShadowIntegrationTests', + 'CollectorLargeHistoryBinaryTests', + 'CollectorCLIIntegrationTests', + ]) { + expect(realBinaryGateScript).toContain( + `-only-testing:EngramServiceCoreTests/${target}`, + ); + } + for (const variable of [ + 'TEST_RUNNER_ENGRAM_COLLECTOR_BINARY', + 'TEST_RUNNER_ENGRAM_SERVICE_BINARY', + 'TEST_RUNNER_ENGRAM_REMOTE_SERVER_BINARY', + 'TEST_RUNNER_CFFIXED_USER_HOME', + ]) { + expect(realBinaryGateScript).toContain(variable); + } + expect(realBinaryGateScript).toContain('test-without-building'); + expect(realBinaryGateScript).toContain( + 'xcresulttool get test-results tests', + ); + expect(realBinaryGateScript).toContain('check-real-binary-results.py'); + expect(realBinaryGateScript).toContain( + '--allow-skip testRealBinaryHTTPSBrowserDemoWithBoundedHold', + ); + for (const text of [realBinaryWorkflow, realBinaryGateScript]) { + expect(text).not.toMatch(/^(?!\s*#).*ENGRAM_SHADOW_/m); + expect(text).not.toContain('upload-artifact'); + } + }); +}); + describe('Dependency Review workflow', () => { it('fail-closes pull requests that introduce moderate-or-higher vulnerabilities', () => { expect(dependencyReviewWorkflow).toContain('name: Dependency Review'); diff --git a/tests/scripts/real-binary-gate.test.ts b/tests/scripts/real-binary-gate.test.ts new file mode 100644 index 000000000..e491ff139 --- /dev/null +++ b/tests/scripts/real-binary-gate.test.ts @@ -0,0 +1,348 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const repoRoot = resolve(import.meta.dirname, '../..'); +const gate = resolve(repoRoot, 'scripts/ci/check-real-binary-results.py'); +const python = process.env.PYTHON3 ?? 'python3'; +const testsDir = resolve(repoRoot, 'macos/EngramServiceCoreTests'); +const realSources = [ + join(testsDir, 'CollectorBinaryShadowIntegrationTests.swift'), + join(testsDir, 'CollectorCLIIntegrationTests.swift'), +]; +const browserDemo = 'testRealBinaryHTTPSBrowserDemoWithBoundedHold'; + +type TestCase = { suite: string; name: string; result: string }; + +// Independent of the Python parser: XCTestCase classes and their extensions. +function realExpectedTests(): TestCase[] { + const tests: TestCase[] = []; + for (const source of realSources) { + const testClasses = new Set(); + let current: string | null = null; + for (const line of readFileSync(source, 'utf8').split('\n')) { + const cls = line.match(/^(?:final )?class (\w+): XCTestCase\b/); + const ext = line.match(/^extension (\w+)\b/); + if (cls) { + current = cls[1]; + testClasses.add(current); + } else if (ext) { + current = testClasses.has(ext[1]) ? ext[1] : null; + } else if ( + /^\S/.test(line) && + /\b(class|struct|enum|extension)\b/.test(line) + ) { + current = null; + } else { + const test = line.match(/^ {4}func (test\w+)\(/); + if (test && current) { + tests.push({ suite: current, name: test[1], result: 'Passed' }); + } + } + } + } + return tests; +} + +function xcresult(cases: TestCase[]) { + const suites = new Map(); + for (const testCase of cases) { + suites.set(testCase.suite, [ + ...(suites.get(testCase.suite) ?? []), + testCase, + ]); + } + return { + devices: [], + testNodes: [ + { + nodeType: 'Test Plan', + name: 'EngramServiceCore', + children: [ + { + nodeType: 'Unit test bundle', + name: 'EngramServiceCoreTests', + children: [...suites].map(([suite, members]) => ({ + nodeType: 'Test Suite', + name: suite, + children: members.map((member) => ({ + nodeType: 'Test Case', + name: `${member.name}()`, + nodeIdentifier: `${suite}/${member.name}()`, + result: member.result, + durationInSeconds: 1.5, + children: + member.result === 'Skipped' + ? [{ nodeType: 'Failure Message', name: 'skipped' }] + : [], + })), + })), + }, + ], + }, + ], + }; +} + +let workdir: string; + +function runGate( + json: unknown, + options: { exitCode?: string; sources?: string[] } = {}, +) { + const testsJson = join(workdir, 'tests.json'); + writeFileSync( + testsJson, + typeof json === 'string' ? json : JSON.stringify(json), + ); + const binary = join(workdir, 'EngramCollector'); + writeFileSync(binary, 'collector-bytes'); + const receipt = join(workdir, 'receipt.json'); + const summary = join(workdir, 'receipt.md'); + const args = [ + gate, + '--tests-json', + testsJson, + ...(options.sources ?? realSources).flatMap((source) => [ + '--source', + source, + ]), + '--allow-skip', + browserDemo, + '--xcodebuild-exit-code', + options.exitCode ?? '0', + '--binary', + `collector=${binary}`, + '--git-sha', + 'deadbeef', + '--xcode-version', + 'Xcode 26.4 Build version 17E192 ', + '--dirty-paths', + '0', + '--receipt', + receipt, + '--summary', + summary, + ]; + const result = spawnSync(python, args, { encoding: 'utf8' }); + return { result, receipt, summary }; +} + +function withResult( + cases: TestCase[], + name: string, + result: string, +): TestCase[] { + return cases.map((testCase) => + testCase.name === name ? { ...testCase, result } : testCase, + ); +} + +describe('real-binary chain result gate', () => { + beforeEach(() => { + workdir = mkdtempSync(join(tmpdir(), 'engram-real-binary-gate-')); + }); + + afterEach(() => { + rmSync(workdir, { recursive: true, force: true }); + }); + + it('parses the expected real-binary test set from the Swift sources', () => { + const expected = realExpectedTests(); + const bySuite = Object.fromEntries( + [ + 'CollectorBinaryShadowIntegrationTests', + 'CollectorLargeHistoryBinaryTests', + 'CollectorCLIIntegrationTests', + ].map((suite) => [ + suite, + expected.filter((test) => test.suite === suite).length, + ]), + ); + expect(bySuite).toEqual({ + CollectorBinaryShadowIntegrationTests: 32, + CollectorLargeHistoryBinaryTests: 1, + CollectorCLIIntegrationTests: 17, + }); + expect(expected.map((test) => test.name)).toContain(browserDemo); + }); + + it('accepts every test passed plus the opt-in browser skip and writes a receipt', () => { + const cases = withResult(realExpectedTests(), browserDemo, 'Skipped'); + const { result, receipt, summary } = runGate(xcresult(cases)); + expect(result.stderr).toBe(''); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0', + ); + const written = JSON.parse(readFileSync(receipt, 'utf8')); + expect(written).toMatchObject({ + schema: 1, + result: 'pass', + git_sha: 'deadbeef', + dirty_paths: 0, + xcode_version: 'Xcode 26.4 Build version 17E192', + expected: 50, + passed: 49, + skipped: [`CollectorBinaryShadowIntegrationTests/${browserDemo}`], + failed: [], + reasons: [], + }); + expect(written.binaries.collector.sha256).toBe( + createHash('sha256').update('collector-bytes').digest('hex'), + ); + expect(written.tests).toHaveLength(50); + expect(readFileSync(summary, 'utf8')).toContain( + 'REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0', + ); + }); + + it('accepts the browser demo when it passed', () => { + const { result } = runGate(xcresult(realExpectedTests())); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=pass expected=50 passed=50 skipped=0 failed=0', + ); + }); + + // Repro for docs/TODO.md W6.2 (real-binary chain in a recorded gate): when + // the binary-path variables never reach the test runner every real-binary + // test reports Skipped and xcodebuild still exits 0, which looked green. + it('rejects a run where every real-binary test skipped (repro)', () => { + const cases = realExpectedTests().map((testCase) => ({ + ...testCase, + result: 'Skipped', + })); + const { result, receipt } = runGate(xcresult(cases)); + expect(result.status).toBe(1); + expect(result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=fail expected=50 passed=0 skipped=50 failed=0', + ); + expect(result.stderr).toContain( + 'test skipped outside the allow-skip list: CollectorCLIIntegrationTests/', + ); + expect(JSON.parse(readFileSync(receipt, 'utf8')).result).toBe('fail'); + }); + + it('rejects a skipped test outside the allow-skip list', () => { + const cases = withResult( + realExpectedTests(), + 'testLargeHistoryIsByteExactOnBothReplicasAndCompleteAcrossWebPages', + 'Skipped', + ); + const { result } = runGate(xcresult(cases)); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + 'test skipped outside the allow-skip list: CollectorLargeHistoryBinaryTests/testLargeHistoryIsByteExactOnBothReplicasAndCompleteAcrossWebPages', + ); + }); + + it('rejects a failed test and a nonzero xcodebuild exit code', () => { + const failed = withResult( + realExpectedTests(), + 'testRealPiNineMiBMessageReachesBothReplicasAndHQWebIPC', + 'Failed', + ); + const failedRun = runGate(xcresult(failed), { exitCode: '65' }); + expect(failedRun.result.status).toBe(1); + expect(failedRun.result.stdout).toContain('result=fail'); + expect(failedRun.result.stdout).toContain('failed=1'); + expect(failedRun.result.stderr).toContain( + "test result 'Failed': CollectorBinaryShadowIntegrationTests/testRealPiNineMiBMessageReachesBothReplicasAndHQWebIPC", + ); + + const exitOnly = runGate(xcresult(realExpectedTests()), { + exitCode: '65', + }); + expect(exitOnly.result.status).toBe(1); + expect(exitOnly.result.stderr).toContain( + 'xcodebuild test exited with code 65', + ); + }); + + it('rejects missing or unexpected tests', () => { + const expected = realExpectedTests(); + const missing = runGate( + xcresult( + expected.filter( + (test) => test.suite !== 'CollectorCLIIntegrationTests', + ), + ), + ); + expect(missing.result.status).toBe(1); + expect(missing.result.stderr).toContain( + 'expected test did not run: CollectorCLIIntegrationTests/', + ); + + const extra = runGate( + xcresult([ + ...expected, + { + suite: 'CollectorBinaryPerformanceTests', + name: 'testSomethingElse', + result: 'Passed', + }, + ]), + ); + expect(extra.result.status).toBe(1); + expect(extra.result.stderr).toContain( + 'unexpected test ran: CollectorBinaryPerformanceTests/testSomethingElse', + ); + + const source = join(workdir, 'Inline.swift'); + writeFileSync( + source, + [ + 'final class InlineTests: XCTestCase {', + ' func testRenamed() async throws {}', + '}', + 'private struct Helper {', + ' func testHelperIsNotATest() {}', + '}', + 'extension InlineTests {', + ' func testInExtension() {}', + '}', + '', + ].join('\n'), + ); + const renamed = runGate( + xcresult([ + { suite: 'InlineTests', name: 'testOldName', result: 'Passed' }, + { suite: 'InlineTests', name: 'testInExtension', result: 'Passed' }, + ]), + { sources: [source] }, + ); + expect(renamed.result.status).toBe(1); + expect(renamed.result.stdout.trim()).toBe( + 'REAL_BINARY_GATE result=fail expected=2 passed=2 skipped=0 failed=0', + ); + expect(renamed.result.stderr).toContain( + 'expected test did not run: InlineTests/testRenamed', + ); + expect(renamed.result.stderr).toContain( + 'unexpected test ran: InlineTests/testOldName', + ); + expect(renamed.result.stderr).not.toContain('testHelperIsNotATest'); + }); + + it('rejects an empty or malformed result JSON', () => { + const empty = runGate(''); + expect(empty.result.status).toBe(1); + expect(empty.result.stderr).toContain('cannot parse xcresult tests JSON'); + + const noNodes = runGate({ testNodes: [] }); + expect(noNodes.result.status).toBe(1); + expect(noNodes.result.stderr).toContain( + 'xcresult contains zero Test Case nodes', + ); + + const wrongShape = runGate({ devices: [] }); + expect(wrongShape.result.status).toBe(1); + expect(wrongShape.result.stderr).toContain( + 'xcresult tests JSON has no testNodes array', + ); + }); +}); From efb43b10fc040a84024c118cdd64ab2766eef222 Mon Sep 17 00:00:00 2001 From: reviewer Date: Sun, 18 Oct 2026 08:01:19 +0800 Subject: [PATCH 3/6] docs: record the W6.2 real-binary gate and its local receipt Update the TODO W6.2 bullet with what shipped (real-binary.yml, the runner script and the exact-set verifier), the local Debug receipt (REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0 at b2e8ef2a, plus the all-skipped negative run rejected by the verifier) and what remains: the PR's own hosted macos-15 run. Claude-Session: https://claude.ai/code/session_01RE2f7JXQBa8xzoFrxuHS6v --- docs/TODO.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/TODO.md b/docs/TODO.md index c8ba493d6..4b041b5d1 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -39,6 +39,25 @@ authorizes host changes. `CollectorBinaryShadowIntegrationTests.swift` skips unless three binary-path environment variables are set, and no CI workflow sets them. Done when CI sets them or a documented manual gate with a receipt exists. + Update 2026-10-18 (branch `ci/real-binary-gate-20261018`): shipped the + non-required `.github/workflows/real-binary.yml` (weekly Sunday 20:17 UTC, + `workflow_dispatch`, and pull requests that touch its own five gate paths; + not in `test.yml` or `ci-gate`) and `scripts/ci/run-real-binary-gate.sh`, + which builds the Debug `EngramCollector`/`EngramService`/`EngramRemoteServer` + tools, sets the three binary variables (plain and `TEST_RUNNER_` forms) with + an isolated `CFFIXED_USER_HOME`, and runs + `CollectorBinaryShadowIntegrationTests`, `CollectorLargeHistoryBinaryTests` + and `CollectorCLIIntegrationTests` (CLI needs only the collector binary). + `scripts/ci/check-real-binary-results.py` compares the exact test set parsed + from the Swift sources with the xcresult tree and fails on any missing, + extra, failed or skipped test except the opt-in browser demo. Local Debug + receipt on Xcode 26.4 at `b2e8ef2a` (dirty paths 0): + `REAL_BINARY_GATE result=pass expected=50 passed=49 skipped=1 failed=0`; the + same classes run without the variables exit 0 with 50 skipped and the + verifier rejects them (`result=fail ... skipped=50`). Remains: the PR's own + hosted macos-15 "Real-binary chain" run must be green with that line (Debug + tool loading and the 25 s scope deadlines are unproven on hosted runners); + record the run URL here and close the item. - **App "open Web reader" entry (design section 1).** In `collector`/`replica` roles the App shows only the unavailable message; it has no action that opens the Web reader as the design states. From d27b37e15d53290b0089ffdcbc1c96d4c95c1ab6 Mon Sep 17 00:00:00 2001 From: reviewer Date: Sun, 18 Oct 2026 09:55:54 +0800 Subject: [PATCH 4/6] ci(real-binary): scale shadow deadlines on slower hosts The first hosted run of the gate (PR #475, run 39084776142) executed all 50 tests: 48 passed, the browser demo skipped by design, and CollectorLargeHistoryBinaryTests failed with BinaryShadowFailure.deadline after both replicas had already stored the 114 MB history. macos-15 hosted runners are several times slower than the Mac that produced the local receipt, so the fixed 240 s scope budget is a host-speed limit, not a correctness signal. Add ENGRAM_REAL_BINARY_TIMEOUT_SCALE (1-10, default 1) read by BinaryShadowScope, a --timeout-scale option on run-real-binary-gate.sh that sets it for the test runner, and --timeout-scale 3 in real-binary.yml. The scale lengthens deadlines only; assertions and the expected test set are unchanged, and the ENGRAM_SHADOW_ opt-in stays unset. Claude-Session: https://claude.ai/code/session_01RE2f7JXQBa8xzoFrxuHS6v --- .github/workflows/real-binary.yml | 7 ++++++- .../CollectorBinaryShadowIntegrationTests.swift | 11 ++++++++++- scripts/ci/run-real-binary-gate.sh | 15 ++++++++++++++- tests/scripts/ci-workflow.test.ts | 3 +++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/real-binary.yml b/.github/workflows/real-binary.yml index ebe64d5eb..0f4fc9838 100644 --- a/.github/workflows/real-binary.yml +++ b/.github/workflows/real-binary.yml @@ -47,10 +47,15 @@ jobs: - name: Build binaries and run the real-binary chain run: | + # Hosted macos-15 is several times slower than the Mac that produced + # the local receipt: the 114 MB large-history chain exceeded its + # 240 s scope deadline there (PR #475 run 39084776142) while every + # other test passed. The scale lengthens deadlines only. bash scripts/ci/run-real-binary-gate.sh \ --out "$RUNNER_TEMP/engram-real-binary-out" \ --derived-data "$RUNNER_TEMP/engram-real-binary-derived" \ - --cloned-source-packages "$HOME/Library/Caches/EngramSourcePackages" + --cloned-source-packages "$HOME/Library/Caches/EngramSourcePackages" \ + --timeout-scale 3 - name: Publish receipt if: always() diff --git a/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift b/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift index ddc15a019..02060f874 100644 --- a/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift +++ b/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift @@ -114,6 +114,15 @@ private struct ShadowBinaries { } return Self(collector: paths[0], service: paths[1], remote: paths[2]) } + + /// Deadline multiplier for slower hosts (hosted CI runners), set by + /// scripts/ci/run-real-binary-gate.sh --timeout-scale. Clamped to 1...10; + /// anything unset or malformed means 1. It never changes an assertion. + static var timeoutScale: Double { + guard let raw = ProcessInfo.processInfo.environment["ENGRAM_REAL_BINARY_TIMEOUT_SCALE"], + let value = Double(raw), value.isFinite else { return 1 } + return min(max(value, 1), 10) + } } private struct ShadowRole { @@ -209,7 +218,7 @@ private final class BinaryShadowScope: @unchecked Sendable { cursorLegacy: Bool = false, seedStores: Bool = true, timeout: TimeInterval = 25, piLargeReply: Bool = false) throws { guard [.codex, .claudeCode, .minimax, .lobsterai, .qwen, .qoder, .iflow, .vscode, .cline, .commandcode, .copilot, .geminiCli, .opencode, .kimi, .cursor, .antigravity, .windsurf, .pi, .grok].contains(sourceKind) else { throw BinaryShadowFailure.fixture } guard !cursorLegacy || sourceKind == .cursor else { throw BinaryShadowFailure.fixture } - self.deadline = Date().addingTimeInterval(timeout) + self.deadline = Date().addingTimeInterval(timeout * ShadowBinaries.timeoutScale) self.binaries = binaries self.sourceKind = sourceKind self.piLargeReply = piLargeReply diff --git a/scripts/ci/run-real-binary-gate.sh b/scripts/ci/run-real-binary-gate.sh index 8a8d4edf5..4f1c102d6 100755 --- a/scripts/ci/run-real-binary-gate.sh +++ b/scripts/ci/run-real-binary-gate.sh @@ -6,10 +6,14 @@ # # Usage: run-real-binary-gate.sh --out DIR [--derived-data DIR] # [--cloned-source-packages DIR] +# [--timeout-scale N] +# --timeout-scale multiplies the shadow scope deadlines inside the tests +# (ENGRAM_REAL_BINARY_TIMEOUT_SCALE, 1-10, default 1) for slower hosts such as +# hosted runners; it changes no assertion, only how long the chain may take. set -euo pipefail usage() { - echo "usage: $0 --out DIR [--derived-data DIR] [--cloned-source-packages DIR]" >&2 + echo "usage: $0 --out DIR [--derived-data DIR] [--cloned-source-packages DIR] [--timeout-scale N]" >&2 exit 2 } @@ -21,15 +25,21 @@ fi OUT="" DD="" SPM="" +SCALE="1" while [ "$#" -gt 0 ]; do case "$1" in --out) [ "$#" -ge 2 ] || usage; OUT="$2"; shift 2 ;; --derived-data) [ "$#" -ge 2 ] || usage; DD="$2"; shift 2 ;; --cloned-source-packages) [ "$#" -ge 2 ] || usage; SPM="$2"; shift 2 ;; + --timeout-scale) [ "$#" -ge 2 ] || usage; SCALE="$2"; shift 2 ;; *) usage ;; esac done [ -n "$OUT" ] || usage +case "$SCALE" in + [1-9]|10) ;; + *) echo "--timeout-scale must be an integer from 1 to 10, got '$SCALE'" >&2; exit 2 ;; +esac ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" if [ -e "$OUT" ] && [ -n "$(ls -A "$OUT")" ]; then @@ -81,8 +91,11 @@ chmod 700 "$TEST_HOME" # ENGRAM_SHADOW_* is deliberately never set here: the browser demo stays an # operator opt-in and is the only test the verifier allows to skip. +echo "REAL_BINARY_GATE timeout_scale=$SCALE" set +e env \ + ENGRAM_REAL_BINARY_TIMEOUT_SCALE="$SCALE" \ + TEST_RUNNER_ENGRAM_REAL_BINARY_TIMEOUT_SCALE="$SCALE" \ ENGRAM_COLLECTOR_BINARY="$COLLECTOR" \ TEST_RUNNER_ENGRAM_COLLECTOR_BINARY="$COLLECTOR" \ ENGRAM_SERVICE_BINARY="$SERVICE" \ diff --git a/tests/scripts/ci-workflow.test.ts b/tests/scripts/ci-workflow.test.ts index 4e060e233..8aec3380e 100644 --- a/tests/scripts/ci-workflow.test.ts +++ b/tests/scripts/ci-workflow.test.ts @@ -941,9 +941,12 @@ describe('Real-binary workflow', () => { 'TEST_RUNNER_ENGRAM_SERVICE_BINARY', 'TEST_RUNNER_ENGRAM_REMOTE_SERVER_BINARY', 'TEST_RUNNER_CFFIXED_USER_HOME', + 'TEST_RUNNER_ENGRAM_REAL_BINARY_TIMEOUT_SCALE', ]) { expect(realBinaryGateScript).toContain(variable); } + // Hosted runners get longer scope deadlines, never different assertions. + expect(realBinaryWorkflow).toContain('--timeout-scale 3'); expect(realBinaryGateScript).toContain('test-without-building'); expect(realBinaryGateScript).toContain( 'xcresulttool get test-results tests', From 63c3277bd8e3e0068cd13f813c71146a6206d879 Mon Sep 17 00:00:00 2001 From: reviewer Date: Sun, 18 Oct 2026 10:49:53 +0800 Subject: [PATCH 5/6] ci(real-binary): scale the fixture's replica HTTP timeouts too Second hosted run (39089748788, timeout_scale=3): the large-history test got past the hq replica's 114 MB byte check, then the m1 manifest fetch hit the fixture's fixed 0.5 s request timeout (NSURLError -1001) while the runner was still busy. Apply ENGRAM_REAL_BINARY_TIMEOUT_SCALE to the fixture's replica fetch (0.5 s / 1.5 s) and chunk download (10 s) URLSession timeouts as well. The Web read client's IPC timeout stays at its 2 s maximum; those calls already retry until the scope deadline. Claude-Session: https://claude.ai/code/session_01RE2f7JXQBa8xzoFrxuHS6v --- .../CollectorBinaryShadowIntegrationTests.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift b/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift index 02060f874..0bd205995 100644 --- a/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift +++ b/macos/EngramServiceCoreTests/CollectorBinaryShadowIntegrationTests.swift @@ -674,8 +674,8 @@ private final class BinaryShadowScope: @unchecked Sendable { try checkRunning() let configuration = URLSessionConfiguration.ephemeral configuration.connectionProxyDictionary = [:] - configuration.timeoutIntervalForRequest = 0.5 - configuration.timeoutIntervalForResource = 1.5 + configuration.timeoutIntervalForRequest = 0.5 * ShadowBinaries.timeoutScale + configuration.timeoutIntervalForResource = 1.5 * ShadowBinaries.timeoutScale let session = URLSession(configuration: configuration) defer { session.invalidateAndCancel() } var request = URLRequest(url: replica.baseURL.appendingPathComponent(path)) @@ -3750,7 +3750,7 @@ private extension BinaryShadowScope { XCTAssertEqual(manifest.wholeSourceSHA256, EngramCollectorCore.ArchiveV2Hash.sha256(expected)) let configuration = URLSessionConfiguration.ephemeral configuration.connectionProxyDictionary = [:] - configuration.timeoutIntervalForRequest = 10 + configuration.timeoutIntervalForRequest = 10 * ShadowBinaries.timeoutScale let session = URLSession(configuration: configuration) defer { session.invalidateAndCancel() } var offset = 0 From 10f5ab21f876b73b768a4a940165d41cc0dfc3bd Mon Sep 17 00:00:00 2001 From: reviewer Date: Sun, 18 Oct 2026 12:56:10 +0800 Subject: [PATCH 6/6] ci(real-binary): give hosted runners 6x deadlines and a 60 min job budget Run 39093667839 passed with the large-history test at 429.6 s; run 39097383289 on the same code hit the scaled 720 s deadline while HQ was still ingesting the 114 MB history (both replicas had the bytes after 12 s in both runs). The hosted runner's speed varies about 2x between runs, so use --timeout-scale 6 and raise the job timeout to 60 minutes. Assertions and the expected test set are unchanged. Claude-Session: https://claude.ai/code/session_01RE2f7JXQBa8xzoFrxuHS6v --- .github/workflows/real-binary.yml | 13 +++++++------ tests/scripts/ci-workflow.test.ts | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/real-binary.yml b/.github/workflows/real-binary.yml index 0f4fc9838..229f40159 100644 --- a/.github/workflows/real-binary.yml +++ b/.github/workflows/real-binary.yml @@ -28,7 +28,7 @@ jobs: real-binary-chain: name: Real-binary chain runs-on: macos-15 - timeout-minutes: 45 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -47,15 +47,16 @@ jobs: - name: Build binaries and run the real-binary chain run: | - # Hosted macos-15 is several times slower than the Mac that produced - # the local receipt: the 114 MB large-history chain exceeded its - # 240 s scope deadline there (PR #475 run 39084776142) while every - # other test passed. The scale lengthens deadlines only. + # Hosted macos-15 is several times slower, and varies run to run, than + # the Mac that produced the local receipt: the 114 MB large-history + # chain's HQ ingest took about 7 min in run 39093667839 (pass) and + # more than 12 min in run 39097383289 (deadline at 3x), while every + # other test passed each time. The scale lengthens deadlines only. bash scripts/ci/run-real-binary-gate.sh \ --out "$RUNNER_TEMP/engram-real-binary-out" \ --derived-data "$RUNNER_TEMP/engram-real-binary-derived" \ --cloned-source-packages "$HOME/Library/Caches/EngramSourcePackages" \ - --timeout-scale 3 + --timeout-scale 6 - name: Publish receipt if: always() diff --git a/tests/scripts/ci-workflow.test.ts b/tests/scripts/ci-workflow.test.ts index 8aec3380e..8144da57c 100644 --- a/tests/scripts/ci-workflow.test.ts +++ b/tests/scripts/ci-workflow.test.ts @@ -913,7 +913,7 @@ describe('Real-binary workflow', () => { 'macos/EngramServiceCoreTests/CollectorCLIIntegrationTests.swift', ]); expect(realBinaryWorkflow).toContain('runs-on: macos-15'); - expect(realBinaryWorkflow).toContain('timeout-minutes: 45'); + expect(realBinaryWorkflow).toContain('timeout-minutes: 60'); expect(realBinaryWorkflow).toContain( 'group: real-binary-$' + '{{ github.ref }}-$' + '{{ github.event_name }}', ); @@ -946,7 +946,7 @@ describe('Real-binary workflow', () => { expect(realBinaryGateScript).toContain(variable); } // Hosted runners get longer scope deadlines, never different assertions. - expect(realBinaryWorkflow).toContain('--timeout-scale 3'); + expect(realBinaryWorkflow).toContain('--timeout-scale 6'); expect(realBinaryGateScript).toContain('test-without-building'); expect(realBinaryGateScript).toContain( 'xcresulttool get test-results tests',