diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 555017223ea9..d92892a425e8 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -37,6 +37,24 @@ In PowerShell, use `$env:ORCA_DEV_MAIN_SOURCEMAPS='0'` before `pnpm dev`.
Remove the variable when debugging the main process with source maps. Renderer and
preload settings are unaffected; production builds always keep their hidden main maps.
+Local installs use pnpm's global virtual store: each package version is one slot
+under `~/pnpm/store` that every worktree links to, instead of a full `node_modules` per worktree.
+An existing worktree converts on its next `pnpm install`, and an older branch without the setting
+converts back the same way. CI always uses the normal layout. On Windows, pnpm links with
+junctions when you lack the symlink privilege, so no administrator rights or Developer Mode are
+needed. Things to know:
+
+- Keep `~/node_modules` (and any other `node_modules` above the store) out of your home
+ directory. Node resolves upward from the store slot, so a stray one there supplies
+ dependencies the package never declared and hides missing-dependency bugs; `pnpm install` warns
+ when it finds one.
+- Native addons are never rebuilt in place in a shared slot: each CPU gets its own build
+ beside it (see [`native-build-entry.cjs`](../config/scripts/native-build-entry.cjs)), so
+ cross-arch packaging is safe for other worktrees.
+- `ORCA_PNPM_GLOBAL_VIRTUAL_STORE=0` opts out. Keep it set
+ in your shell: `pnpm run` re-checks the install and relinks a worktree whose layout no longer
+ matches.
+
Ordinary installs include native optional dependencies for the current OS and CPU only.
Before a cross-architecture build (including `pnpm build:mac`, which produces both x64 and
arm64 artifacts by default), run `pnpm install:release` to add the other CPU's variants.
diff --git a/.github/actions/install-node-dependencies/action.yml b/.github/actions/install-node-dependencies/action.yml
index b81dd6aed515..1332106a4db6 100644
--- a/.github/actions/install-node-dependencies/action.yml
+++ b/.github/actions/install-node-dependencies/action.yml
@@ -86,7 +86,7 @@ runs:
case "$LOOKUP_REQUEST" in
[aA][uU][tT][oO])
# Hosted runners have Node for this manifest-only check before toolchain setup.
- lookup_only="$(node -p 'const p = require("./package.json"); p.engines?.node === "24" && typeof p.packageManager === "string" && p.packageManager.split("+")[0] === "pnpm@12.8.1"')"
+ lookup_only="$(node -p 'const p = require("./package.json"); p.engines?.node === "24" && typeof p.packageManager === "string" && p.packageManager.split("+")[0] === "pnpm@12.14.0"')"
;;
esac
printf 'lookup-only=%s\n' "$lookup_only" >> "$GITHUB_OUTPUT"
diff --git a/.github/actions/prepare-headless-compiler/action.yml b/.github/actions/prepare-headless-compiler/action.yml
index 9e71e6df1b07..b6e60d989986 100644
--- a/.github/actions/prepare-headless-compiler/action.yml
+++ b/.github/actions/prepare-headless-compiler/action.yml
@@ -14,7 +14,7 @@ runs:
- id: identity
shell: bash
env:
- COMPILER_POLICY_HASH: ${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', 'config/patches/**', '.github/actions/install-node-dependencies/**', '.github/actions/restore-pnpm-verification/**', 'config/scripts/headless-detector-compiler-cache.mjs', '.github/actions/prepare-headless-compiler/action.yml') }}
+ COMPILER_POLICY_HASH: ${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', 'config/pnpmfile.cjs', 'config/patches/**', '.github/actions/install-node-dependencies/**', '.github/actions/restore-pnpm-verification/**', 'config/scripts/headless-detector-compiler-cache.mjs', '.github/actions/prepare-headless-compiler/action.yml') }}
run: node config/scripts/headless-detector-compiler-cache.mjs identity
- id: cache
uses: actions/cache/restore@v5
diff --git a/.github/actions/prepare-linux-package-fixture/action.yml b/.github/actions/prepare-linux-package-fixture/action.yml
index e3f297f4b104..146d4c47b25d 100644
--- a/.github/actions/prepare-linux-package-fixture/action.yml
+++ b/.github/actions/prepare-linux-package-fixture/action.yml
@@ -7,10 +7,16 @@ inputs:
save-cache:
description: Build and save a missing fixture on the default branch.
default: 'false'
+ load:
+ description: Load the image here; 'false' only restores, so the caller can load it in a background run step.
+ default: 'true'
outputs:
image:
description: Loaded image to use as a Docker build cache, or empty on a cache miss.
value: ${{ steps.image.outputs.image }}
+ cache-hit:
+ description: Whether the fixture archive was restored under the exact key.
+ value: ${{ steps.cache.outputs.cache-hit }}
runs:
using: composite
steps:
@@ -20,39 +26,17 @@ runs:
continue-on-error: true
with:
path: ${{ runner.temp }}/orca-package-fixtures/${{ inputs.fixture }}.tar
- key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml') }}
+ key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml', '.github/actions/prepare-linux-package-fixture/load-fixture.sh') }}
- name: Load or seed fixture build cache
id: image
+ if: inputs.load != 'false'
shell: bash
env:
FIXTURE: ${{ inputs.fixture }}
SAVE_CACHE: ${{ inputs.save-cache }}
CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
- run: |
- set -euo pipefail
- case "$FIXTURE" in
- headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;;
- *) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;;
- esac
- archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar"
- image="orca-package-fixture-$FIXTURE:cache"
- if [[ "$CACHE_HIT" == true ]] && docker load --input "$archive"; then
- if docker image inspect "$image" > /dev/null; then
- echo "image=$image" >> "$GITHUB_OUTPUT"
- exit 0
- fi
- fi
- if [[ "$SAVE_CACHE" != true ]]; then
- echo 'No usable fixture cache; the package runner will build normally.'
- exit 0
- fi
- docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \
- --tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE"
- mkdir -p "$(dirname "$archive")"
- docker save --output "$archive" "$image"
- echo "image=$image" >> "$GITHUB_OUTPUT"
- echo 'built=true' >> "$GITHUB_OUTPUT"
+ run: bash "$GITHUB_ACTION_PATH/load-fixture.sh"
- name: Save fixture image
if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && steps.image.outputs.built == 'true'
diff --git a/.github/actions/prepare-linux-package-fixture/load-fixture.sh b/.github/actions/prepare-linux-package-fixture/load-fixture.sh
new file mode 100755
index 000000000000..9d76ac92806f
--- /dev/null
+++ b/.github/actions/prepare-linux-package-fixture/load-fixture.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Loads a restored fixture image; only the default-branch warmer builds and exports a missing one.
+set -euo pipefail
+case "$FIXTURE" in
+ headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;;
+ *) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;;
+esac
+archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar"
+image="orca-package-fixture-$FIXTURE:cache"
+if [[ "${CACHE_HIT:-}" == true ]] && docker load --input "$archive"; then
+ if docker image inspect "$image" > /dev/null; then
+ echo "image=$image" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+fi
+if [[ "${SAVE_CACHE:-}" != true ]]; then
+ echo 'No usable fixture cache; the package runner will build normally.'
+ exit 0
+fi
+docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \
+ --tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE"
+mkdir -p "$(dirname "$archive")"
+docker save --output "$archive" "$image"
+echo "image=$image" >> "$GITHUB_OUTPUT"
+echo 'built=true' >> "$GITHUB_OUTPUT"
diff --git a/.github/actions/prepare-native-runtime/action.yml b/.github/actions/prepare-native-runtime/action.yml
index 76f9954b7e2c..61c289c052b8 100644
--- a/.github/actions/prepare-native-runtime/action.yml
+++ b/.github/actions/prepare-native-runtime/action.yml
@@ -54,7 +54,7 @@ runs:
env:
NATIVE_RUNTIME: ${{ inputs.native-runtime }}
NODE_VERSION: ${{ inputs.node-version }}
- NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/initialize-windows-native-toolchain.mjs', 'config/scripts/capture-windows-native-environment.cmd', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }}
+ NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', 'config/pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/initialize-windows-native-toolchain.mjs', 'config/scripts/capture-windows-native-environment.cmd', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }}
run: |
scope="${ImageOS:-$RUNNER_OS}"
if [ -r /etc/os-release ]; then
diff --git a/.github/actions/prepare-wayland-clipboard/action.yml b/.github/actions/prepare-wayland-clipboard/action.yml
index 78c622855bd5..c35382714207 100644
--- a/.github/actions/prepare-wayland-clipboard/action.yml
+++ b/.github/actions/prepare-wayland-clipboard/action.yml
@@ -5,6 +5,14 @@ inputs:
save-cache:
description: Publish a validated helper from the default branch.
default: 'false'
+ build:
+ description: Build here; 'false' only restores, so the caller can build in a background run step.
+ default: 'true'
+
+outputs:
+ cache-hit:
+ description: Whether the helper was restored under the exact key.
+ value: ${{ steps.cache.outputs.cache-hit }}
runs:
using: composite
@@ -30,22 +38,14 @@ runs:
key: ${{ steps.identity.outputs.key }}
- name: Build or validate Wayland clipboard helper
+ if: inputs.build != 'false'
shell: bash
env:
CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
- run: |
- if node config/scripts/build-wayland-clipboard.mjs; then
- exit 0
- fi
- if [ "$CACHE_HIT" != true ]; then
- exit 1
- fi
- echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
- rm -rf -- native/wayland-clipboard/.build
- node config/scripts/build-wayland-clipboard.mjs
+ run: bash "$GITHUB_ACTION_PATH/build-or-validate.sh"
- name: Save validated Wayland clipboard helper from main
- if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+ if: inputs.build != 'false' && inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
uses: actions/cache/save@v5
continue-on-error: true
with:
diff --git a/.github/actions/prepare-wayland-clipboard/build-or-validate.sh b/.github/actions/prepare-wayland-clipboard/build-or-validate.sh
new file mode 100755
index 000000000000..d71b802edc11
--- /dev/null
+++ b/.github/actions/prepare-wayland-clipboard/build-or-validate.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Validates a restored helper and rebuilds once from source when the restored copy is unusable.
+set -e
+if node config/scripts/build-wayland-clipboard.mjs; then
+ exit 0
+fi
+if [ "${CACHE_HIT:-}" != true ]; then
+ exit 1
+fi
+echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.'
+rm -rf -- native/wayland-clipboard/.build
+node config/scripts/build-wayland-clipboard.mjs
diff --git a/.github/workflows/adhoc-mac-build.yml b/.github/workflows/adhoc-mac-build.yml
index b270cb727250..3772f4e4d061 100644
--- a/.github/workflows/adhoc-mac-build.yml
+++ b/.github/workflows/adhoc-mac-build.yml
@@ -128,7 +128,7 @@ jobs:
echo "Resolved $REQUESTED_REF -> $sha"
echo "sha=$sha" >>"$GITHUB_OUTPUT"
- # Why: a branch cut before the orcad template landed has none to build or ship.
+ # Resolve shared build identity while detecting whether this ref ships a template.
orcad-template-support:
needs: resolve-ref
runs-on: ubuntu-latest
@@ -137,13 +137,17 @@ jobs:
contents: read
outputs:
ships: ${{ steps.detect.outputs.ships }}
+ version: ${{ steps.adhoc.outputs.version }}
+ commit: ${{ steps.adhoc.outputs.commit }}
+ name: ${{ steps.adhoc.outputs.name }}
steps:
- - name: Checkout the template packager only
+ - name: Checkout template detection and version scripts
uses: actions/checkout@v6
with:
ref: ${{ needs.resolve-ref.outputs.sha }}
sparse-checkout: |
- /config/scripts/packaged-orcad-template.cjs
+ /config/scripts/
+ /package.json
/config/tsconfig.relay.json
sparse-checkout-cone-mode: false
persist-credentials: false
@@ -166,6 +170,47 @@ jobs:
echo "::notice::This ref predates the orcad template; the build ships without it."
fi
+ - name: Setup Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version-file: package.json
+ package-manager-cache: false
+
+ - name: Compute adhoc version
+ id: adhoc
+ shell: bash
+ env:
+ REF: ${{ inputs.ref || github.ref_name }}
+ LABEL: ${{ inputs.label }}
+ MAIN_REPO_TOKEN: ${{ github.token }}
+ run: |
+ set -euo pipefail
+ # Why this check: the version scripts are read from the branch being built,
+ # not from main, so a branch cut before the adhoc channel landed has no
+ # copy of them. Say that plainly instead of failing with a module-not-found.
+ for script in adhoc-build-version dev-channel-base-version; do
+ if [[ ! -f "config/scripts/$script.mjs" ]]; then
+ echo "::error::$REF has no config/scripts/$script.mjs; rebase it onto a main that has the adhoc channel."
+ exit 1
+ fi
+ done
+ # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
+ # GitHub release and leaves the tag, which still owns that number.
+ # Releases-only let adhoc sit on a number already taken, so the updater
+ # would not install it. Empty on failure — the script then falls back
+ # to package.json.
+ published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
+ "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
+ --jq '.[].ref | sub("^refs/tags/"; "")' || true)"
+ ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \
+ node config/scripts/adhoc-build-version.mjs \
+ >"$RUNNER_TEMP/adhoc-identity.txt"
+ if ! grep -q '^name=' "$RUNNER_TEMP/adhoc-identity.txt"; then
+ echo "::error::adhoc-build-version.mjs emitted no release name; $REF's copy of the script is out of sync with this workflow."
+ exit 1
+ fi
+ cat "$RUNNER_TEMP/adhoc-identity.txt" >>"$GITHUB_OUTPUT"
+
# Design D2, as release-cut does: every desktop build ships the orcad template (server JS plus
# every target's addons), merged from the node-server lanes that qualify each slot at this ref.
# Without it an adhoc build cannot deploy managed orcad to an SSH host. No secrets, and the mac
@@ -199,8 +244,8 @@ jobs:
environment: adhoc-mac-build
outputs:
tag: ${{ steps.release.outputs.tag }}
- version: ${{ steps.adhoc.outputs.version }}
- head_sha: ${{ steps.adhoc.outputs.head_sha }}
+ version: ${{ needs.orcad-template-support.outputs.version }}
+ head_sha: ${{ needs.resolve-ref.outputs.sha }}
published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }}
ships_orcad_template: ${{ needs.orcad-template-support.outputs.ships }}
runs-on: blacksmith-6vcpu-macos-15
@@ -343,42 +388,6 @@ jobs:
APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
- - name: Compute adhoc version
- id: adhoc
- shell: bash
- env:
- REF: ${{ inputs.ref || github.ref_name }}
- LABEL: ${{ inputs.label }}
- MAIN_REPO_TOKEN: ${{ github.token }}
- run: |
- set -euo pipefail
- # Why this check: the version scripts are read from the branch being built,
- # not from main, so a branch cut before the adhoc channel landed has no
- # copy of them. Say that plainly instead of failing with a module-not-found.
- for script in adhoc-build-version dev-channel-base-version; do
- if [[ ! -f "config/scripts/$script.mjs" ]]; then
- echo "::error::$REF has no config/scripts/$script.mjs; rebase it onto a main that has the adhoc channel."
- exit 1
- fi
- done
- echo "head_sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT"
- # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the
- # GitHub release and leaves the tag, which still owns that number.
- # Releases-only let adhoc sit on a number already taken, so the updater
- # would not install it. Empty on failure — the script then falls back
- # to package.json.
- published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \
- "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \
- --jq '.[].ref | sub("^refs/tags/"; "")' || true)"
- ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \
- node config/scripts/adhoc-build-version.mjs \
- >"$RUNNER_TEMP/adhoc-identity.txt"
- if ! grep -q '^name=' "$RUNNER_TEMP/adhoc-identity.txt"; then
- echo "::error::adhoc-build-version.mjs emitted no release name; $REF's copy of the script is out of sync with this workflow."
- exit 1
- fi
- cat "$RUNNER_TEMP/adhoc-identity.txt" >>"$GITHUB_OUTPUT"
-
- name: Build app
run: pnpm build:release
env:
@@ -414,9 +423,9 @@ jobs:
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
- TAG: v${{ steps.adhoc.outputs.version }}
- NAME: ${{ steps.adhoc.outputs.name }}
- SHA: ${{ steps.adhoc.outputs.head_sha }}
+ TAG: v${{ needs.orcad-template-support.outputs.version }}
+ NAME: ${{ needs.orcad-template-support.outputs.name }}
+ SHA: ${{ needs.resolve-ref.outputs.sha }}
REF: ${{ inputs.ref || github.ref_name }}
# Via env, not inline `${{ }}`: both land inside a shell string, and an
# expression expanded there is substituted before bash parses the line
@@ -463,13 +472,13 @@ jobs:
timeout_minutes: 45
max_attempts: 2
retry_wait_seconds: 30
- command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
+ command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_ADHOC=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi
env:
# Why: electron-builder's github publisher targets the repo named in the
# config; the token must therefore carry write access to orca-adhoc.
GH_TOKEN: ${{ steps.app_token.outputs.token }}
- ORCA_ADHOC_BUILD_VERSION: ${{ steps.adhoc.outputs.version }}
- ORCA_BUILD_COMMIT: ${{ steps.adhoc.outputs.commit }}
+ ORCA_ADHOC_BUILD_VERSION: ${{ needs.orcad-template-support.outputs.version }}
+ ORCA_BUILD_COMMIT: ${{ needs.orcad-template-support.outputs.commit }}
# beforePack and afterPack fail the package when the template is absent.
ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ needs.orcad-template-support.outputs.ships == 'true' && '1' || '' }}
CSC_LINK: ${{ secrets.MAC_CERTS }}
@@ -516,7 +525,7 @@ jobs:
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
TAG: ${{ steps.release.outputs.tag }}
- NAME: ${{ steps.adhoc.outputs.name }}
+ NAME: ${{ needs.orcad-template-support.outputs.name }}
run: |
set -euo pipefail
# --title again: electron-builder resolves this draft by tag and may
@@ -597,27 +606,37 @@ jobs:
echo "::warning::Could not prune $tag"
done <<<"$stale"
- # Why this runs after the mac leg rather than beside it: the tag and version
- # are computed inside that job, so until it has run nothing else can name the
- # release to upload into. The cost is small enough not to matter — the Windows
- # leg measures ~7.5 min (install 2m45, build 35s, NSIS package 3m) against a
- # ~9.5 min mac run, which keeps a adhoc run far inside its interval.
- #
- # Why `./` rather than a pinned `@main`: `uses:` resolves against the ref this
- # file itself came from, which for an ordinary dispatch is main. Someone who
- # deliberately points the Actions "Use workflow from" picker at a branch
- # already gets that branch's copy of this entire file, so this follows the same
- # rule instead of inventing a second one.
+ # Build unsigned Windows artifacts alongside Mac; publication waits below.
build-adhoc-win:
- needs: build-adhoc-mac
- # Only once the mac release is actually live: there is no release to upload
- # into otherwise, and the Windows workflow refuses to create one.
- if: needs.build-adhoc-mac.outputs.published == 'true'
+ needs: [resolve-ref, orcad-template-support, orcad-template]
+ if: >-
+ !cancelled() && github.repository == 'stablyai/orca' &&
+ needs.orcad-template-support.result == 'success' &&
+ (needs.orcad-template.result == 'success' ||
+ (needs.orcad-template.result == 'skipped' &&
+ needs.orcad-template-support.outputs.ships == 'false'))
uses: ./.github/workflows/dev-channel-win-build.yml
secrets: inherit
+ with:
+ channel: adhoc
+ tag: v${{ needs.orcad-template-support.outputs.version }}
+ ref: ${{ needs.resolve-ref.outputs.sha }}
+ version: ${{ needs.orcad-template-support.outputs.version }}
+ orcad_template: ${{ needs.orcad-template-support.outputs.ships == 'true' }}
+ stage_only: true
+
+ # Upload only after Mac succeeds; an unsigned build may overlap its notarization.
+ publish-adhoc-win:
+ needs: [build-adhoc-mac, build-adhoc-win]
+ if: >-
+ !cancelled() && needs.build-adhoc-mac.result == 'success' &&
+ needs.build-adhoc-win.result == 'success' &&
+ needs.build-adhoc-mac.outputs.published == 'true'
+ permissions:
+ contents: read
+ actions: read
+ uses: ./.github/workflows/dev-channel-win-publish.yml
+ secrets: inherit
with:
channel: adhoc
tag: ${{ needs.build-adhoc-mac.outputs.tag }}
- ref: ${{ needs.build-adhoc-mac.outputs.head_sha }}
- version: ${{ needs.build-adhoc-mac.outputs.version }}
- orcad_template: ${{ needs.build-adhoc-mac.outputs.ships_orcad_template == 'true' }}
diff --git a/.github/workflows/ci-cache-warmup-platforms.yml b/.github/workflows/ci-cache-warmup-platforms.yml
index 06ba3eaeb3b0..35a667ef6aae 100644
--- a/.github/workflows/ci-cache-warmup-platforms.yml
+++ b/.github/workflows/ci-cache-warmup-platforms.yml
@@ -20,7 +20,7 @@ on:
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.npmrc'
- - '.pnpmfile.cjs'
+ - 'config/pnpmfile.cjs'
- 'config/scripts/ensure-native-runtime.mjs'
- 'config/scripts/initialize-windows-native-toolchain.mjs'
- 'config/scripts/capture-windows-native-environment.cmd'
diff --git a/.github/workflows/ci-cache-warmup.yml b/.github/workflows/ci-cache-warmup.yml
index e360f5b02e8a..f1d2f4423af1 100644
--- a/.github/workflows/ci-cache-warmup.yml
+++ b/.github/workflows/ci-cache-warmup.yml
@@ -29,7 +29,7 @@ on:
- 'pnpm-lock.yaml'
- 'pnpm-workspace.yaml'
- '.npmrc'
- - '.pnpmfile.cjs'
+ - 'config/pnpmfile.cjs'
- 'config/scripts/ensure-native-runtime.mjs'
- 'config/scripts/initialize-windows-native-toolchain.mjs'
- 'config/scripts/capture-windows-native-environment.cmd'
diff --git a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml
index 728e8eabb658..cd41c5281bc8 100644
--- a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml
+++ b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml
@@ -160,9 +160,10 @@ jobs:
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
diff --git a/.github/workflows/cloud-deploy-relay-production-capacity.yml b/.github/workflows/cloud-deploy-relay-production-capacity.yml
index 66a2f734ffb0..7f75b7e8d3e0 100644
--- a/.github/workflows/cloud-deploy-relay-production-capacity.yml
+++ b/.github/workflows/cloud-deploy-relay-production-capacity.yml
@@ -129,9 +129,10 @@ jobs:
- uses: actions/checkout@v4
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
diff --git a/.github/workflows/cloud-deploy-relay-production-multi-target.yml b/.github/workflows/cloud-deploy-relay-production-multi-target.yml
index 9c850a31d67d..d780c402a960 100644
--- a/.github/workflows/cloud-deploy-relay-production-multi-target.yml
+++ b/.github/workflows/cloud-deploy-relay-production-multi-target.yml
@@ -186,9 +186,10 @@ jobs:
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
index 8d659ce33499..c0ae3451dc72 100644
--- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
@@ -91,8 +91,8 @@ jobs:
- uses: actions/checkout@v4
- - uses: pnpm/action-setup@v4
- with: { package_json_file: cloud/package.json }
+ - uses: pnpm/setup@v2
+ with: { package-json-file: cloud/package.json, install: false }
- uses: actions/setup-node@v4
with:
@@ -441,9 +441,15 @@ jobs:
if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end
] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \
<<< "${CURRENT_RUNTIME}"
- CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
+ if ! CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \
"${DIRECTOR_ORIGIN}/v1/admin/cell-status" \
- "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"
+ "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"; then
+ if jq -e '.error == "cell_in_reserve_mode"' \
+ "${RUNNER_TEMP}/current-cell-status.json" >/dev/null 2>&1; then
+ echo "${TARGET_CELL_ID} is in reserve mode: flip it to admitMode=db with the cell-flags workflow and wait for it to record db before rolling it, then flip it back to reserve" >&2
+ fi
+ exit 1
+ fi
SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \
<<< "${CURRENT_DIRECTOR_STATUS}")"
if test "${ROLLBACK_RESUME}" = true && ! jq -e \
diff --git a/.github/workflows/cloud-deploy-relay-production.yml b/.github/workflows/cloud-deploy-relay-production.yml
index 257e7aeb06aa..9aa7af1f5599 100644
--- a/.github/workflows/cloud-deploy-relay-production.yml
+++ b/.github/workflows/cloud-deploy-relay-production.yml
@@ -83,9 +83,10 @@ jobs:
github-token: ${{ github.token }}
run-id: ${{ inputs.monitor-run-id }}
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
diff --git a/.github/workflows/cloud-monitor-relay-production-job.yml b/.github/workflows/cloud-monitor-relay-production-job.yml
index 663b40961ac7..1c10b9059345 100644
--- a/.github/workflows/cloud-monitor-relay-production-job.yml
+++ b/.github/workflows/cloud-monitor-relay-production-job.yml
@@ -57,9 +57,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
diff --git a/.github/workflows/cloud-operate-relay-asia-admission.yml b/.github/workflows/cloud-operate-relay-asia-admission.yml
index 0a0e764d9ffb..79752e43db66 100644
--- a/.github/workflows/cloud-operate-relay-asia-admission.yml
+++ b/.github/workflows/cloud-operate-relay-asia-admission.yml
@@ -194,9 +194,10 @@ jobs:
with:
node-version: 24
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
if: ${{ steps.inputs.outputs.canary == 'true' }}
- name: Install exact canary dependencies
diff --git a/.github/workflows/cloud-operate-relay-production-cell-flags.yml b/.github/workflows/cloud-operate-relay-production-cell-flags.yml
index c11fd5db52ca..d3e0e2d9fd4f 100644
--- a/.github/workflows/cloud-operate-relay-production-cell-flags.yml
+++ b/.github/workflows/cloud-operate-relay-production-cell-flags.yml
@@ -32,7 +32,8 @@ on:
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
- reregisterInFlight
+ reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip
+ must name phonePath)
required: true
type: string
expected-generation:
@@ -62,7 +63,7 @@ defaults:
jobs:
operate:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
- runs-on: ubuntu-latest
+ runs-on: blacksmith-2vcpu-ubuntu-2204
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
timeout-minutes: 60
environment: production
diff --git a/.github/workflows/cloud-operate-relay-production-rehome-job.yml b/.github/workflows/cloud-operate-relay-production-rehome-job.yml
index 587cd9558696..cbc3efd38d0b 100644
--- a/.github/workflows/cloud-operate-relay-production-rehome-job.yml
+++ b/.github/workflows/cloud-operate-relay-production-rehome-job.yml
@@ -134,9 +134,9 @@ jobs:
--drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \
| tee "${RUNNER_TEMP}/relay-rehome-control.json"
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
if: ${{ inputs.mode == 'enable' }}
- with: { package_json_file: cloud/package.json }
+ with: { package-json-file: cloud/package.json, install: false }
- run: pnpm install --frozen-lockfile
if: ${{ inputs.mode == 'enable' }}
diff --git a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml
index e045d10928e7..2711cd062107 100644
--- a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml
+++ b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml
@@ -25,15 +25,17 @@ on:
type: string
flipped-at:
description: >-
- When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The 503
- background is the median of the 10 minutes before it
+ When the reserve (or phonePath=memory) write applied, ISO with a zone, e.g.
+ 2026-10-14T12:00:00Z. The non-drain 503 background and the phone rules' background are
+ the 10 minutes before it
required: false
default: ''
type: string
background-503-per-minute:
description: >-
- Director 503s per minute before the flip; overrides the one measured before flipped-at.
- One of the two is required
+ Director non-drain 503s per minute before the flip (all 503s less drain-return deferrals
+ and a host's own early or duplicate re-dials, from the director's runtime metrics);
+ overrides the one measured before flipped-at. One of the two is required
required: false
default: ''
type: string
@@ -71,13 +73,15 @@ env:
jobs:
watch:
if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }}
- runs-on: ubuntu-latest
+ runs-on: blacksmith-2vcpu-ubuntu-2204
# Six 55-minute segments cover the longest watch; under the 360-minute job limit.
timeout-minutes: 350
environment: production
outputs:
state: ${{ steps.outcome.outputs.state }}
verdict: ${{ steps.outcome.outputs.verdict }}
+ # phonePath when a step-6 phone rule tripped; anything else flips the cell back whole.
+ flip: ${{ steps.outcome.outputs.flip }}
env:
DURATION_MINUTES: ${{ inputs.duration-minutes }}
FLIPPED_AT: ${{ inputs.flipped-at }}
@@ -88,6 +92,10 @@ jobs:
set -euo pipefail
[[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]]
node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)"
+ # A phone-rule trip writes phonePath=db mid-watch, through the flag tool.
+ ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)"
+ echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}"
+ export ORCA_RELAY_CONTROL_ACCESS_TOKEN
node dev/scripts/watch-relay-reserve-cell.mjs watch \
--mode "${MODE}" --cell-id "${CELL_ID}" \
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
@@ -247,7 +255,7 @@ jobs:
# Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an
# operator too) flips back like a trip. In dry-run this only plans the write.
if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }}
- runs-on: ubuntu-latest
+ runs-on: blacksmith-2vcpu-ubuntu-2204
# A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls.
timeout-minutes: 60
environment: production
@@ -257,6 +265,7 @@ jobs:
# lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve.
env:
VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }}
+ FLIP: ${{ needs.watch.outputs.flip || 'admitMode' }}
steps:
- uses: actions/checkout@v4
if: ${{ always() }}
@@ -296,7 +305,7 @@ jobs:
--mode "${MODE}" --cell-id "${CELL_ID}" \
--cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \
--project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \
- --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" \
+ --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" --flip "${FLIP}" \
| tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"
# The flip-back line is logged only once the write (or its dry-run plan) finished; without
@@ -307,12 +316,15 @@ jobs:
if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then
exit 0
fi
- {
- echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode"
- echo "Flip it to db by hand with Operate Relay Production Cell Flags."
- } >> "${GITHUB_STEP_SUMMARY}"
+ if [[ "${FLIP}" == phonePath ]]; then
+ echo "## FLIP BACK FAILED: ${CELL_ID} may still have phonePath=memory" >> "${GITHUB_STEP_SUMMARY}"
+ echo "Set phonePath=default by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}"
+ else
+ echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" >> "${GITHUB_STEP_SUMMARY}"
+ echo "Flip it to db by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}"
+ fi
node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \
- --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}"
+ --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" --flip "${FLIP}"
- name: Audit summary
if: ${{ always() }}
diff --git a/.github/workflows/cloud-operate-relay-staging-cell-flags.yml b/.github/workflows/cloud-operate-relay-staging-cell-flags.yml
index 12781fd16a99..501d00faebc1 100644
--- a/.github/workflows/cloud-operate-relay-staging-cell-flags.yml
+++ b/.github/workflows/cloud-operate-relay-staging-cell-flags.yml
@@ -33,7 +33,8 @@ on:
Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a
key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck,
admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs,
- reregisterInFlight
+ reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip
+ must name phonePath)
required: true
type: string
expected-generation:
diff --git a/.github/workflows/cloud-prove-relay-asia-staging.yml b/.github/workflows/cloud-prove-relay-asia-staging.yml
index ad94ee002791..e214a8a330d5 100644
--- a/.github/workflows/cloud-prove-relay-asia-staging.yml
+++ b/.github/workflows/cloud-prove-relay-asia-staging.yml
@@ -124,9 +124,10 @@ jobs:
with:
node-version: 24
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- name: Require the exact staging director image before promotion
env:
@@ -433,9 +434,10 @@ jobs:
with:
node-version: 24
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- name: Install exact load-harness dependencies
run: pnpm install --frozen-lockfile
diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml
index c39edcb11a38..4d0f74949a2f 100644
--- a/.github/workflows/cloud-verify.yml
+++ b/.github/workflows/cloud-verify.yml
@@ -86,9 +86,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
@@ -125,9 +126,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- - uses: pnpm/action-setup@v4
+ - uses: pnpm/setup@v2
with:
- package_json_file: cloud/package.json
+ package-json-file: cloud/package.json
+ install: false
- uses: actions/setup-node@v4
with:
diff --git a/.github/workflows/daily-mac-build.yml b/.github/workflows/daily-mac-build.yml
index 28e95ba96d64..ce0f2fc7a045 100644
--- a/.github/workflows/daily-mac-build.yml
+++ b/.github/workflows/daily-mac-build.yml
@@ -388,7 +388,7 @@ jobs:
timeout_minutes: 45
max_attempts: 2
retry_wait_seconds: 30
- command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
+ command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_DAILY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi
env:
# Why: electron-builder's github publisher targets the repo named in the
# config; the token must therefore carry write access to orca-daily.
diff --git a/.github/workflows/dev-channel-win-build.yml b/.github/workflows/dev-channel-win-build.yml
index 194659964863..adebe5da5e75 100644
--- a/.github/workflows/dev-channel-win-build.yml
+++ b/.github/workflows/dev-channel-win-build.yml
@@ -5,11 +5,8 @@ name: Dev Channel Windows Build
# Windows artifact can be rebuilt for an existing tag without paying for the mac
# leg's packaging and notarization again.
#
-# Each mac workflow calls this as a `needs:`-gated job once its release is live,
-# passing the tag it created. Both legs land in that one release, so a tag
-# carries every platform it managed to build. Measured cost of the Windows leg is
-# ~7.5 min (install 2m45, build 35s, NSIS package 3m) against a mac run of ~9.5
-# min, so running it after the mac job keeps an hourly well inside its cron.
+# Adhoc can stage Windows alongside macOS, then publish only after macOS succeeds.
+# Hourly, daily, and manual dispatches keep packaging and publishing together.
#
# Why unsigned: Windows release installers are signed by SignPath *after*
# packaging, and release-cut budgets 1h + 4h for those approval waits. That does
@@ -27,9 +24,8 @@ name: Dev Channel Windows Build
# installer run. Every way out — to another dev build, or back to Stable — works
# through the in-app updater. The picker offers a download for exactly that jump.
#
-# Called as a job by each mac workflow once its release is live, and separately
-# dispatchable by hand to rebuild a Windows artifact for an existing tag without
-# re-running the mac leg's twenty minutes of packaging and notarization:
+# Mac workflows call this directly or stage artifacts for a later publish job.
+# It is also dispatchable to rebuild Windows for an existing tag:
#
# gh workflow run dev-channel-win-build.yml --ref main \
# -f channel=adhoc -f tag=v1.4.178-adhoc.20260819010203 \
@@ -47,7 +43,7 @@ on:
required: true
type: string
tag:
- description: Existing release tag in the channel repo
+ description: Release tag for staged or published artifacts
required: true
type: string
ref:
@@ -58,6 +54,11 @@ on:
description: Version to package, without the leading v
required: true
type: string
+ stage_only:
+ description: Package unsigned artifacts for a later gated publish job
+ required: false
+ type: boolean
+ default: false
orcad_template:
description: Ship the orcad-template artifact the calling run built
required: false
@@ -276,6 +277,7 @@ jobs:
# Why the token is minted here and not at the top: installation tokens live
# one hour and nothing before this point writes anything.
- name: Mint dev channel repo token
+ if: ${{ !inputs.stage_only }}
id: app_token
uses: actions/create-github-app-token@v2
with:
@@ -289,6 +291,7 @@ jobs:
# while this was building, that would mint a fresh, untitled, Windows-only
# release. Check first and fail instead.
- name: Confirm the target release still exists
+ if: ${{ !inputs.stage_only }}
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
@@ -300,6 +303,7 @@ jobs:
fi
- name: Publish Windows artifacts
+ if: ${{ !inputs.stage_only }}
uses: nick-fields/retry@v4
with:
# 30 is the pack + upload budget; there is no notary queue on this leg.
@@ -323,6 +327,7 @@ jobs:
# release carrying the installer but not the manifest is one the picker
# offers and the update 404s on, so assert both.
- name: Verify Windows update manifest published
+ if: ${{ !inputs.stage_only }}
shell: bash
env:
GH_TOKEN: ${{ steps.app_token.outputs.token }}
@@ -338,3 +343,41 @@ jobs:
fi
done
echo "Windows artifacts verified on $TAG."
+
+ - name: Package Windows artifacts without publishing
+ if: inputs.stage_only
+ uses: nick-fields/retry@v4
+ with:
+ timeout_minutes: 30
+ max_attempts: 2
+ retry_wait_seconds: 30
+ command: node config/scripts/ensure-native-runtime.mjs --runtime=electron; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never
+ env:
+ ORCA_BUILD_COMMIT: ${{ inputs.ref }}
+ ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ inputs.orcad_template && '1' || '' }}
+
+ - name: Validate staged Windows artifacts
+ if: inputs.stage_only
+ shell: bash
+ run: |
+ set -euo pipefail
+ for required in orca-windows-setup.exe orca-windows-setup.exe.blockmap latest.yml; do
+ if [[ ! -s "dist/$required" ]]; then
+ echo "::error::Missing or empty staged Windows artifact: $required"
+ exit 1
+ fi
+ done
+
+ - name: Stage Windows artifacts for gated publishing
+ if: inputs.stage_only
+ uses: actions/upload-artifact@v7
+ with:
+ name: dev-channel-windows-${{ inputs.tag }}
+ path: |
+ dist/orca-windows-setup.exe
+ dist/orca-windows-setup.exe.blockmap
+ dist/latest.yml
+ if-no-files-found: error
+ retention-days: 3
+ compression-level: 0
+ overwrite: true
diff --git a/.github/workflows/dev-channel-win-publish.yml b/.github/workflows/dev-channel-win-publish.yml
new file mode 100644
index 000000000000..6864d1277d00
--- /dev/null
+++ b/.github/workflows/dev-channel-win-publish.yml
@@ -0,0 +1,107 @@
+name: Dev Channel Windows Publish
+
+on:
+ workflow_call:
+ inputs:
+ channel:
+ description: Dev channel whose live release receives the staged artifacts
+ required: true
+ type: string
+ tag:
+ description: Release tag used when staging the Windows artifacts
+ required: true
+ type: string
+
+permissions:
+ contents: read
+ actions: read
+
+concurrency:
+ group: dev-channel-win-publish-${{ inputs.tag }}
+ cancel-in-progress: false
+
+jobs:
+ publish-win:
+ if: github.repository == 'stablyai/orca'
+ environment: adhoc-mac-build
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ env:
+ CHANNEL: ${{ inputs.channel }}
+ TAG: ${{ inputs.tag }}
+ steps:
+ - name: Vet the requested inputs
+ shell: bash
+ run: |
+ set -euo pipefail
+ case "$CHANNEL" in
+ hourly|daily|adhoc) ;;
+ *) echo "::error::Unknown dev channel '$CHANNEL'."; exit 1 ;;
+ esac
+ if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-${CHANNEL}\.[0-9]+$ ]]; then
+ echo "::error::Tag '$TAG' is not a $CHANNEL release tag."
+ exit 1
+ fi
+
+ # No run-id: artifacts must belong to this run, never another dispatch.
+ - name: Download the staged Windows artifacts
+ uses: actions/download-artifact@v8
+ with:
+ name: dev-channel-windows-${{ inputs.tag }}
+ path: staged-windows
+
+ - name: Validate staged Windows artifacts
+ shell: bash
+ run: |
+ set -euo pipefail
+ for required in orca-windows-setup.exe orca-windows-setup.exe.blockmap latest.yml; do
+ if [[ ! -s "staged-windows/$required" ]]; then
+ echo "::error::Missing or empty staged Windows artifact: $required"
+ exit 1
+ fi
+ done
+
+ - name: Mint dev channel repo token
+ id: app_token
+ uses: actions/create-github-app-token@v2
+ with:
+ app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }}
+ private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }}
+ owner: stablyai
+ repositories: orca-${{ inputs.channel }}
+
+ - name: Confirm the target release is live
+ shell: bash
+ env:
+ GH_TOKEN: ${{ steps.app_token.outputs.token }}
+ run: |
+ set -euo pipefail
+ draft="$(gh release view "$TAG" --repo "stablyai/orca-$CHANNEL" --json isDraft --jq '.isDraft')"
+ if [[ "$draft" != false ]]; then
+ echo "::error::Release $TAG is not live; refusing to publish Windows artifacts."
+ exit 1
+ fi
+
+ - name: Publish Windows artifacts
+ uses: nick-fields/retry@v4
+ with:
+ timeout_minutes: 5
+ max_attempts: 2
+ retry_wait_seconds: 10
+ command: gh release upload "$TAG" staged-windows/orca-windows-setup.exe staged-windows/orca-windows-setup.exe.blockmap staged-windows/latest.yml --clobber --repo "stablyai/orca-$CHANNEL"
+ env:
+ GH_TOKEN: ${{ steps.app_token.outputs.token }}
+
+ - name: Verify Windows update manifest published
+ shell: bash
+ env:
+ GH_TOKEN: ${{ steps.app_token.outputs.token }}
+ run: |
+ set -euo pipefail
+ assets="$(gh release view "$TAG" --repo "stablyai/orca-$CHANNEL" --json assets --jq '.assets[].name')"
+ for required in latest.yml orca-windows-setup.exe orca-windows-setup.exe.blockmap; do
+ if ! grep -qx "$required" <<<"$assets"; then
+ echo "::error::$TAG is missing $required; Windows could not install this build."
+ exit 1
+ fi
+ done
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index a06601679553..dd78c63edd40 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -47,35 +47,32 @@ jobs:
persist-credentials: false
- name: Setup pnpm
- # v5 avoids the v6 bootstrap/shim regression when pinning pnpm 10.
- uses: pnpm/action-setup@v5
+ uses: pnpm/setup@v2
with:
- version: 10.34.6
- package_json_file: docs/site/package.json
- run_install: false
+ package-json-file: docs/site/package.json
+ install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
- node-version: 22
- # The repository root pins pnpm 12 while this isolated app pins 10;
- # setup-node's cache probe runs at the root and would auto-switch.
+ node-version-file: docs/site/package.json
+ # setup-node probes the store from the repository root, not this app.
package-manager-cache: false
- name: Install site dependencies
- run: pnpm --ignore-workspace install --frozen-lockfile
+ run: pnpm install --frozen-lockfile
- name: Run package tests
- run: pnpm --ignore-workspace test
+ run: pnpm test
- name: Lint site
- run: pnpm --ignore-workspace lint
+ run: pnpm lint
- name: Typecheck site
- run: pnpm --ignore-workspace exec tsc --noEmit --incremental false
+ run: pnpm exec tsc --noEmit --incremental false
- name: Build site
- run: pnpm --ignore-workspace build
+ run: pnpm build
release_gate:
name: Authorize release
@@ -222,21 +219,19 @@ jobs:
persist-credentials: false
- name: Setup pnpm
- # v5 avoids the v6 bootstrap/shim regression when pinning pnpm 10.
- uses: pnpm/action-setup@v5
+ uses: pnpm/setup@v2
with:
- version: 10.34.6
- package_json_file: docs/site/package.json
- run_install: false
+ package-json-file: docs/site/package.json
+ install: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
- node-version: 22
+ node-version-file: docs/site/package.json
package-manager-cache: false
- name: Install site dependencies
- run: pnpm --ignore-workspace install --frozen-lockfile
+ run: pnpm install --frozen-lockfile
- name: Verify Vercel credentials
env:
@@ -254,23 +249,23 @@ jobs:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
- run: pnpm --ignore-workspace exec vercel pull --yes --non-interactive --environment=production
+ run: pnpm exec vercel pull --yes --non-interactive --environment=production
- name: Run package tests
- run: pnpm --ignore-workspace test
+ run: pnpm test
- name: Lint site
- run: pnpm --ignore-workspace lint
+ run: pnpm lint
- name: Typecheck site
- run: pnpm --ignore-workspace exec tsc --noEmit --incremental false
+ run: pnpm exec tsc --noEmit --incremental false
- name: Build production site
- run: pnpm --ignore-workspace exec vercel build --prod --non-interactive
+ run: pnpm exec vercel build --prod --non-interactive
- name: Deploy production site
env:
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
- run: pnpm --ignore-workspace exec vercel deploy --prebuilt --prod --yes --non-interactive
+ run: pnpm exec vercel deploy --prebuilt --prod --yes --non-interactive
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 4dfc477850c7..ef402098804a 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -27,12 +27,23 @@ on:
required: false
type: boolean
default: true
+ managed_ssh_shards:
+ description: JSON shard assignments for the selected managed SSH specs
+ required: false
+ type: string
+ default: '["1/2","2/2"]'
# Why: artifact names are unique per workflow run, and pr.yml calls this workflow twice.
artifact_suffix:
description: Appended to the build and trace artifact names so two calls in one run do not collide
required: false
type: string
default: ''
+ # Why: the app, orcad and its test template (--dark-test-template) build with it; releases refuse it.
+ layout_owner:
+ description: Set to main to build with ORCA_LAYOUT_OWNER=main and record save-divergence reports
+ required: false
+ type: string
+ default: ''
workflow_dispatch:
inputs:
ref:
@@ -43,6 +54,12 @@ on:
description: JSON array of specs to run; empty runs the full suite
required: false
type: string
+ managed_ssh_shards:
+ description: Managed SSH shard assignments (use one for small selections)
+ required: false
+ type: choice
+ default: '["1/2","2/2"]'
+ options: ['["1/2","2/2"]', '["1/1"]']
schedule:
# One complete daily reference run; targeted PR coverage remains unchanged.
- cron: '0 17 * * *'
@@ -107,7 +124,12 @@ jobs:
env:
VITE_EXPOSE_STORE: 'true'
ORCA_CI_MAIN_SOURCEMAPS: '0'
- run: pnpm run build:electron-vite:parallel --mode e2e
+ LAYOUT_OWNER: ${{ inputs.layout_owner }}
+ run: |
+ if [ -n "$LAYOUT_OWNER" ]; then
+ export ORCA_LAYOUT_OWNER="$LAYOUT_OWNER"
+ fi
+ pnpm run build:electron-vite:parallel --mode e2e
# The CLI emits into out/main too; start only after Electron finishes clearing that tree.
- name: Build shared E2E CLI
@@ -125,6 +147,11 @@ jobs:
- wait: e2e-cli
+ # The switch-on specs skip, rather than fail, when the build left no marker.
+ - name: Check layout-owner build marker
+ if: inputs.layout_owner == 'main'
+ run: test -f out/main/ORCA_LAYOUT_OWNER_MAIN
+
- name: Check shared native runtime action
id: native-runtime-action
run: |
@@ -138,10 +165,8 @@ jobs:
- uses: ./.github/actions/build-mobile-web-bundle
if: hashFiles('.github/actions/build-mobile-web-bundle/action.yml') != ''
- # Only the completed output upload overlaps the change from Node to Electron ABI.
+ # Foreground: a background JavaScript action reads the event payload while later steps rewrite it.
- name: Upload E2E build output
- id: e2e-build-upload
- background: true
uses: actions/upload-artifact@v7
with:
name: e2e-build-out${{ inputs.artifact_suffix }}
@@ -164,8 +189,6 @@ jobs:
with:
native-runtime: electron
- - wait: e2e-build-upload
-
e2e:
name: e2e ${{ matrix.shard_name }}
needs: [build]
@@ -325,6 +348,7 @@ jobs:
- name: Run changed E2E specs
env:
TEST_FILES_JSON: ${{ inputs.test_files }}
+ LAYOUT_OWNER: ${{ inputs.layout_owner }}
run: |
if [ -f config/scripts/ci-e2e-job-selection.mjs ]; then
printf '%s\n' "$TEST_FILES_JSON" | node config/scripts/ci-e2e-job-selection.mjs > "$RUNNER_TEMP/general-e2e-specs"
@@ -347,12 +371,43 @@ jobs:
fi
E2E_ENV=(SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1)
E2E_PROJECT_ARGS=()
+ if [ -n "$LAYOUT_OWNER" ]; then
+ test -f out/main/ORCA_LAYOUT_OWNER_MAIN
+ mkdir -p layout-divergence
+ E2E_ENV+=(ORCA_LAYOUT_SAVE_DIVERGENCE_LOG="$PWD/layout-divergence/window-specs.jsonl")
+ E2E_ENV+=(PLAYWRIGHT_JSON_OUTPUT_FILE="$PWD/layout-divergence/window-specs-results.json")
+ E2E_PROJECT_ARGS+=('--reporter=list,json')
+ fi
if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then
E2E_PROJECT_ARGS+=(--project=electron-headful)
fi
xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \
pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}"
+ - name: Summarize layout save divergence
+ if: always() && inputs.layout_owner == 'main'
+ continue-on-error: true
+ run: &summarize_layout_divergence |
+ # Either file names a run: a log with no report is unattributed, a report with no log is zero.
+ shopt -s nullglob
+ runs=(layout-divergence/*.jsonl layout-divergence/*-results.json)
+ for run in $(printf '%s\n' "${runs[@]%.jsonl}" | sed 's/-results\.json$//' | sort -u); do
+ node config/scripts/layout-save-divergence-summary.mjs "$run.jsonl" "$run-summary.json" "$run-results.json"
+ if [ -f "$run-results.json" ]; then
+ node config/scripts/ci-e2e-failure-summary.mjs "$run-results.json"
+ fi
+ done
+
+ - name: Upload layout save divergence report
+ if: always() && inputs.layout_owner == 'main'
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: layout-divergence-changed${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }}
+ path: layout-divergence/
+ retention-days: 14
+ if-no-files-found: warn
+
- name: Upload Playwright traces
if: failure() || cancelled()
uses: actions/upload-artifact@v7
@@ -410,12 +465,24 @@ jobs:
# The packaged orcad slot the T6-11 launcher runs: its own node-pty under the pinned Node.
# The template is what `orca serve`'s default selection materializes that slot from.
- name: Build this runner's orcad slot and template
+ env:
+ LAYOUT_OWNER: ${{ inputs.layout_owner }}
run: |
slot="$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)"
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots "$slot"
- pnpm build:orcad
- node config/scripts/build-orcad-template.mjs --targets "$slot"
+ # The headless oracle's orcad host must hold the same layout owner as the Electron host,
+ # and so must the template `orca serve` materializes its server from.
+ if [ -n "$LAYOUT_OWNER" ]; then
+ export ORCA_LAYOUT_OWNER="$LAYOUT_OWNER"
+ pnpm build:orcad
+ node config/scripts/build-orcad-template.mjs --targets "$slot" --dark-test-template
+ test -f out/orcad/ORCA_LAYOUT_OWNER_MAIN
+ test -f out/orcad-template/ORCA_LAYOUT_OWNER_MAIN
+ else
+ pnpm build:orcad
+ node config/scripts/build-orcad-template.mjs --targets "$slot"
+ fi
- name: Switch serve hosts on one profile
if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts')
env:
@@ -431,7 +498,30 @@ jobs:
env:
SKIP_BUILD: '1'
ORCA_E2E_FORWARD_APP_LOGS: '1'
- run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/workspace-layout-oracle-headless.spec.ts --project=electron-headless --workers=1
+ LAYOUT_OWNER: ${{ inputs.layout_owner }}
+ run: |
+ REPORT_ARGS=()
+ if [ -n "$LAYOUT_OWNER" ]; then
+ test -f out/main/ORCA_LAYOUT_OWNER_MAIN
+ mkdir -p layout-divergence
+ export ORCA_LAYOUT_SAVE_DIVERGENCE_LOG="$PWD/layout-divergence/headless-oracle.jsonl"
+ export PLAYWRIGHT_JSON_OUTPUT_FILE="$PWD/layout-divergence/headless-oracle-results.json"
+ REPORT_ARGS=('--reporter=list,json')
+ fi
+ xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/workspace-layout-oracle-headless.spec.ts --project=electron-headless --workers=1 "${REPORT_ARGS[@]}"
+ - name: Summarize layout save divergence
+ if: always() && inputs.layout_owner == 'main'
+ continue-on-error: true
+ run: *summarize_layout_divergence
+ - name: Upload layout save divergence report
+ if: always() && inputs.layout_owner == 'main'
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: layout-divergence-headless${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }}
+ path: layout-divergence/
+ retention-days: 14
+ if-no-files-found: warn
- uses: actions/upload-artifact@v7
if: failure()
with:
@@ -440,7 +530,7 @@ jobs:
retention-days: 7
if-no-files-found: ignore
- # Every managed-orcad SSH spec on a Docker host, including #24979's relay-era conversion. Needs the
+ # Selected managed-orcad SSH specs on a Docker host; daily runs keep the full suite. Needs the
# orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds.
orcad-ssh-docker:
name: ssh on managed orcad (Docker ${{ matrix.shard }})
@@ -451,7 +541,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- shard: ['1/2', '2/2']
+ shard: ${{ fromJSON(inputs.managed_ssh_shards || '["1/2","2/2"]') }}
env:
ORCA_BACKGROUND_LAUNCH: '1'
steps:
@@ -472,23 +562,68 @@ jobs:
name: e2e-build-out${{ inputs.artifact_suffix }}
path: out/
# Built to a path the app does not look at by default, so the relay-era phase has no template.
+ # The switch-on leg deploys a switch-on server, so both sides of the host run one layout owner.
- name: Build the linux-x64-glibc orcad template
+ env:
+ LAYOUT_OWNER: ${{ inputs.layout_owner }}
run: |
pnpm build:orcad-prebuilds
pnpm build:orcad-prebuilds --require-slots linux-x64-glibc
- node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc
+ if [ -n "$LAYOUT_OWNER" ]; then
+ ORCA_LAYOUT_OWNER="$LAYOUT_OWNER" node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc --dark-test-template
+ test -f out/orcad-template/ORCA_LAYOUT_OWNER_MAIN
+ else
+ node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc
+ fi
mv out/orcad-template "$RUNNER_TEMP/orcad-convert-template"
# The Windows CLI matrix collects here too and skips: it needs a Windows host descriptor.
- name: Run managed SSH specs on a Docker host
env:
+ TEST_FILES_JSON: ${{ inputs.test_files }}
SKIP_BUILD: '1'
ORCA_E2E_SSH_DOCKER: '1'
ORCA_E2E_ORCAD_CONVERT_HOST: docker
ORCA_E2E_FORWARD_APP_LOGS: '1'
SHARD: ${{ matrix.shard }}
+ LAYOUT_OWNER: ${{ inputs.layout_owner }}
+ SHARD_INDEX: ${{ strategy.job-index }}
run: |
+ node --input-type=module <<'NODE' > "$RUNNER_TEMP/managed-ssh-specs"
+ import { existsSync } from 'node:fs'
+ const modulePath = './config/scripts/ci-e2e-job-selection.mjs'
+ const selection = existsSync(modulePath) ? await import(modulePath) : {}
+ // Older checkout refs retain their full-suite behavior.
+ const specs = selection.selectManagedSshE2eSpecs
+ ? selection.selectManagedSshE2eSpecs(process.env.TEST_FILES_JSON)
+ : ['tests/e2e/ssh-orcad-']
+ for (const spec of specs) {
+ console.log(spec.endsWith('.spec.ts') ? spec.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '$' : spec)
+ }
+ NODE
+ mapfile -t TEST_FILES < "$RUNNER_TEMP/managed-ssh-specs"
export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template"
- xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad- --project=electron-headless --workers=1 --shard="$SHARD"
+ REPORT_ARGS=()
+ if [ -n "$LAYOUT_OWNER" ]; then
+ test -f out/main/ORCA_LAYOUT_OWNER_MAIN
+ mkdir -p layout-divergence
+ export ORCA_LAYOUT_SAVE_DIVERGENCE_LOG="$PWD/layout-divergence/managed-ssh-$SHARD_INDEX.jsonl"
+ export PLAYWRIGHT_JSON_OUTPUT_FILE="$PWD/layout-divergence/managed-ssh-$SHARD_INDEX-results.json"
+ REPORT_ARGS=('--reporter=list,json')
+ fi
+ xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts "${TEST_FILES[@]}" --project=electron-headless --workers=1 --shard="$SHARD" "${REPORT_ARGS[@]}"
+ - name: Summarize layout save divergence
+ if: always() && inputs.layout_owner == 'main'
+ continue-on-error: true
+ run: *summarize_layout_divergence
+ - name: Upload layout save divergence report
+ if: always() && inputs.layout_owner == 'main'
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: layout-divergence-managed-ssh-${{ strategy.job-index }}${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }}
+ path: layout-divergence/
+ retention-days: 14
+ if-no-files-found: warn
- uses: actions/upload-artifact@v7
if: failure()
with:
diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml
index f78cc9c47e1b..4235713a9d5a 100644
--- a/.github/workflows/hourly-mac-build.yml
+++ b/.github/workflows/hourly-mac-build.yml
@@ -361,7 +361,7 @@ jobs:
timeout_minutes: 45
max_attempts: 2
retry_wait_seconds: 30
- command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always
+ command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi
env:
# Why: electron-builder's github publisher targets the repo named in the
# config; the token must therefore carry write access to orca-hourly.
diff --git a/.github/workflows/layout-owner-e2e.yml b/.github/workflows/layout-owner-e2e.yml
new file mode 100644
index 000000000000..4bafb59ac827
--- /dev/null
+++ b/.github/workflows/layout-owner-e2e.yml
@@ -0,0 +1,32 @@
+name: Layout owner E2E (manual)
+run-name: Layout owner E2E ${{ inputs.ref || github.ref }}
+
+on:
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: Commit or ref to test (defaults to the workflow commit)
+ required: false
+ type: string
+
+permissions:
+ contents: read
+
+# This experiment costs about an hour of runner time; keep one suite active repository-wide.
+concurrency:
+ group: layout-owner-e2e
+ cancel-in-progress: false
+
+jobs:
+ layout_owner_switch_on:
+ name: layout owner (switch on)
+ permissions:
+ contents: read
+ uses: ./.github/workflows/e2e.yml
+ with:
+ ref: ${{ inputs.ref || github.sha }}
+ test_files: '["tests/e2e/workspace-layout-oracle-headless.spec.ts","tests/e2e/workspace-layout-oracle.spec.ts","tests/e2e/cli-file-open-position.spec.ts","tests/e2e/relaunch-first-terminal.spec.ts","tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts","tests/e2e/terminal-cold-activation-deferral.spec.ts","tests/e2e/cli-first-worktree-single-terminal.spec.ts","tests/e2e/unopened-workspace-agent-single-resume.spec.ts","tests/e2e/terminal-pane-close-layout-consistency.spec.ts","tests/e2e/golden-file-open-edit-save.spec.ts","tests/e2e/golden-source-control-open-diff.spec.ts","tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts","tests/e2e/floating-workspace-pane-height.spec.ts","tests/e2e/editor-model-host-isolation.spec.ts","tests/e2e/editor-tab-selection-restore.spec.ts","tests/e2e/finished-agent-ghost-resume.spec.ts","tests/e2e/live-background-terminal-mount-authority.spec.ts","tests/e2e/workspace-back-forward-navigation.spec.ts","tests/e2e/worktree-lineage-toggle-shortcut.spec.ts","tests/e2e/local-cli-terminal-retention.spec.ts","tests/e2e/default-tabs-mark-crash-relaunch.spec.ts","tests/e2e/terminal-cli-split-closed-before-mount.spec.ts","tests/e2e/layout-stream-loss-reopen.spec.ts","tests/e2e/terminal-tab-close-restart-persistence.spec.ts","tests/e2e/artificial-opencode-terminal-load.spec.ts","tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts","tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts","tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts","tests/e2e/headless-paired-remote-terminal-viewer-size.spec.ts","tests/e2e/headless-serve-browser-tab-restart.spec.ts","tests/e2e/headless-serve-cli-terminal-retention-parity.spec.ts","tests/e2e/headless-serve-desktop-activation.spec.ts","tests/e2e/headless-serve-focused-terminal-create.spec.ts","tests/e2e/headless-serve-mobile-chat-tab-order.spec.ts","tests/e2e/headless-serve-mobile-native-chat-identity.spec.ts","tests/e2e/headless-serve-session-history-naming-requests.spec.ts","tests/e2e/layout-compat-old-phone-on-new-server.spec.ts","tests/e2e/layout-compat-serve-restart-keeps-at-at-terminals.spec.ts","tests/e2e/paired-mobile-serve-restart-terminal-tabs.spec.ts","tests/e2e/paired-mobile-desktop-tab-composition.spec.ts","tests/e2e/paired-remote-terminal-serve-restart-binding.spec.ts","tests/e2e/ssh-orcad-"]'
+ run_changed_e2e: true
+ needs_build: true
+ artifact_suffix: -layout-owner-switch-on
+ layout_owner: main
diff --git a/.github/workflows/macos-updater-tests.yml b/.github/workflows/macos-updater-tests.yml
index cf7d149ece86..fca034a15610 100644
--- a/.github/workflows/macos-updater-tests.yml
+++ b/.github/workflows/macos-updater-tests.yml
@@ -8,6 +8,9 @@ on:
- 'src/main/macos-update-running-instances*'
- 'src/main/updater*'
- 'src/main/updater/**'
+ - 'native/macos-shipit-scheduling/**'
+ - 'config/scripts/macos-shipit-scheduling*'
+ - 'config/electron-builder.config.cjs'
- 'src/main/startup/main-process-quit*'
- 'src/main/window/main-window-state-lifecycle*'
- 'src/main/window/dashboard-popout-window*'
@@ -35,11 +38,15 @@ jobs:
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
+ cache-electron-package: 'true'
+ - name: Install the pinned Electron binary for native updater probes
+ run: node config/scripts/install-electron-package-binary.mjs
- name: Exercise native application registry and update shutdown
env:
ORCA_BACKGROUND_LAUNCH: '1'
run: >-
pnpm exec vitest run --config config/vitest.config.ts
+ config/scripts/macos-shipit-scheduling.test.mjs
src/main/macos-update-running-instances.test.ts
src/main/macos-update-running-instances.integration.test.ts
src/main/updater.mac-install.test.ts
diff --git a/.github/workflows/node-server-tests.yml b/.github/workflows/node-server-tests.yml
index 0c0aba24d6e9..452ea7446f9a 100644
--- a/.github/workflows/node-server-tests.yml
+++ b/.github/workflows/node-server-tests.yml
@@ -14,7 +14,7 @@ on:
- 'pnpm-workspace.yaml'
- 'tsconfig.json'
- '.npmrc'
- - '.pnpmfile.cjs'
+ - 'config/pnpmfile.cjs'
- '.github/actions/install-node-dependencies/**'
- '.github/actions/restore-pnpm-verification/**'
- '.github/actions/prepare-headless-compiler/**'
@@ -128,7 +128,7 @@ jobs:
fi
persistence:
- needs: changes
+ needs: [changes, mobile_web_bundle]
concurrency:
group: node-server-persistence-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
@@ -136,6 +136,7 @@ jobs:
# Drafts re-trigger at readiness; inherited caller events must not depend on PR fields.
if: >-
${{ !cancelled() && needs.changes.outputs.should_run != 'false' &&
+ (needs.changes.outputs.qualification == 'false' || needs.mobile_web_bundle.result == 'success') &&
(github.event_name != 'pull_request' || github.event.pull_request.draft != true) }}
strategy:
fail-fast: false
@@ -154,7 +155,8 @@ jobs:
id: dependencies
with:
native-runtime: ${{ runner.os == 'Windows' && 'node' || 'none' }}
- cache-pnpm-store: ${{ runner.os != 'Windows' }}
+ # Template qualification must not wait for large download-cache uploads.
+ cache-pnpm-store: ${{ !inputs.build_template && runner.os != 'Windows' }}
cache-pnpm-store-lookup-only: 'true'
# Design D7 upgrade and rollback: the last Bun orcad, built from a main commit that shipped
# it, using the current Bun pin beside this checkout's Node slot; hand-over skips once PROTOCOL_VERSION
@@ -191,7 +193,14 @@ jobs:
if: runner.os == 'Windows'
with:
arches: ${{ (inputs.build_template && matrix.os == 'windows-2022') && 'x64 arm64' || (runner.arch == 'ARM64' && 'arm64' || 'x64') }}
+ - uses: actions/download-artifact@v8
+ if: needs.changes.outputs.qualification != 'false'
+ with:
+ name: orcad-mobile-web-bundle
+ path: out/mobile-web
+ # Narrow test-only runs skip the shared producer but still need a page.
- uses: ./.github/actions/build-mobile-web-bundle
+ if: needs.changes.outputs.qualification == 'false'
- run: pnpm build:orcad
env:
ORCA_REUSE_PREPARED_WINDOWS_SECURITY: ${{ github.event_name == 'pull_request' && runner.os == 'Windows' && steps.dependencies.outcome == 'success' }}
@@ -298,8 +307,7 @@ jobs:
# A rerun attempt re-uploads under the same name, which v4 otherwise refuses.
overwrite: true
- # The container lanes take the page from here: it is platform-independent, and building it needs
- # mobile/node_modules, which those images never install.
+ # Every qualification lane tests the same platform-independent page.
mobile_web_bundle:
needs: changes
if: >-
@@ -324,13 +332,13 @@ jobs:
overwrite: true
linux_glibc_floor:
- needs: [changes, persistence, mobile_web_bundle]
+ needs: [changes, mobile_web_bundle]
concurrency:
group: node-server-linux_glibc_floor-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
- # A failed smoke already blocks qualification; missing scope still selects every platform.
+ # Each Linux lane owns its inputs; the template still requires every lane to pass.
if: >-
- ${{ !cancelled() && needs.persistence.result == 'success' &&
+ ${{ !cancelled() &&
needs.mobile_web_bundle.result == 'success' &&
needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' }}
@@ -397,15 +405,15 @@ jobs:
overwrite: true
linux_glibc217_compat:
- needs: [changes, persistence]
+ needs: changes
concurrency:
group: node-server-linux_glibc217_compat-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
- # A failed smoke already blocks qualification; missing scope still selects every platform.
if: >-
- ${{ !cancelled() && needs.persistence.result == 'success' &&
+ ${{ !cancelled() &&
needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' &&
+ (github.event_name != 'pull_request' || github.event.pull_request.draft != true) &&
(github.event_name != 'pull_request' || inputs.build_template || inputs.ref != '' ||
needs.changes.outputs.native_compatibility != 'false') }}
runs-on: ubuntu-22.04
@@ -454,13 +462,12 @@ jobs:
overwrite: true
linux_musl:
- needs: [changes, persistence, mobile_web_bundle]
+ needs: [changes, mobile_web_bundle]
concurrency:
group: node-server-linux_musl-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }}
- # A failed smoke already blocks qualification; missing scope still selects every platform.
if: >-
- ${{ !cancelled() && needs.persistence.result == 'success' &&
+ ${{ !cancelled() &&
needs.mobile_web_bundle.result == 'success' &&
needs.changes.outputs.should_run != 'false' &&
needs.changes.outputs.qualification != 'false' }}
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 61a3e218a4d1..722d88082cf0 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -65,6 +65,7 @@ jobs:
/config/.bun-version
/config/scripts/check-readme-local-links.mjs
/config/scripts/pr-code-change-scope.mjs
+ /.oxfmtrc.json
/config/scripts/pr-package-test-paths.mjs
/config/scripts/pr-layout-oracle-route.mjs
/config/scripts/pr-e2e-source-routing.mjs
@@ -231,6 +232,7 @@ jobs:
PREFLIGHT_PHASE_SELECTED: ${{ needs.code_paths.outputs.typecheck == 'true' }}
PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }}
ORCA_UNIT_SELECTION_MODE: ${{ vars.ORCA_UNIT_SELECTION_MODE || 'shadow' }}
+ ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }}
# Five shards use fewer runner minutes; the repository variable can still request ten.
ORCA_UNIT_FULL_SHARD_COUNT: ${{ vars.ORCA_UNIT_FULL_SHARD_COUNT || '5' }}
run: |
diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml
index b2edb687dd07..45b90b59995d 100644
--- a/.github/workflows/release-cut.yml
+++ b/.github/workflows/release-cut.yml
@@ -1139,6 +1139,7 @@ jobs:
- orcad-template
- release-preflight
- release-javascript
+ - build-mac
# Fork: mac-only. Windows/Linux builds are disabled (no SignPath on this fork).
# macOS is built + signed by build-mac -> release-mac-build.yml.
if: false
@@ -1212,7 +1213,7 @@ jobs:
run: |
set -euo pipefail
git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA"
- git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs
+ git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs config/scripts/run-release-mac-build-workflow.mjs
- name: Restore composite actions from the workflow ref
shell: bash
@@ -1582,9 +1583,26 @@ jobs:
path: signing-stage/**
if-no-files-found: error
+ # Unsigned packaging overlaps macOS; only successful macOS builds may consume signing quota.
+ - name: Require macOS success before Windows signing
+ id: mac-signing-gate
+ if: matrix.platform == 'win' && github.run_attempt == 1
+ shell: bash
+ run: node config/scripts/run-release-mac-build-workflow.mjs wait
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ RELEASE_MAC_BUILD_REF: ${{ github.ref_name }}
+ RELEASE_MAC_BUILD_RELEASE_RUN_ID: ${{ github.run_id }}
+ RELEASE_MAC_BUILD_RUN_ID: ${{ needs.build-mac.outputs.run_id }}
+ RELEASE_MAC_BUILD_WORKFLOW_SHA: ${{ needs.build-mac.outputs.workflow_sha }}
+ RELEASE_MAC_BUILD_RUN_ATTEMPT: ${{ needs.build-mac.outputs.run_attempt }}
+ RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: ${{ needs.build-mac.outputs.wait_deadline_ms }}
+ RELEASE_MAC_BUILD_TAG: ${{ needs.cut.outputs.tag }}
+ RELEASE_MAC_BUILD_WORKFLOW: release-mac-build.yml
+
- name: Submit inner binaries signing request
id: submit-inner-signing
- if: matrix.platform == 'win' && github.run_attempt == 1 && steps.upload-unsigned-inner.outcome == 'success'
+ if: matrix.platform == 'win' && github.run_attempt == 1 && steps.mac-signing-gate.outcome == 'success' && steps.upload-unsigned-inner.outcome == 'success'
uses: signpath/github-action-submit-signing-request@v2
with:
api-token: ${{ secrets.SIGNPATH_API_TOKEN }}
@@ -1820,7 +1838,7 @@ jobs:
# so the release job waits while the signing request is approved in UI.
- name: Submit Windows installer signing request
id: submit-signing-request
- if: matrix.platform == 'win' && github.run_attempt == 1
+ if: matrix.platform == 'win' && github.run_attempt == 1 && steps.mac-signing-gate.outcome == 'success' && steps.upload-unsigned-windows-installer.outcome == 'success'
uses: signpath/github-action-submit-signing-request@v2
with:
api-token: ${{ secrets.SIGNPATH_API_TOKEN }}
diff --git a/.github/workflows/release-mac-build.yml b/.github/workflows/release-mac-build.yml
index 55c03b8cf818..f82c9ac66f71 100644
--- a/.github/workflows/release-mac-build.yml
+++ b/.github/workflows/release-mac-build.yml
@@ -13,6 +13,11 @@ on:
description: release-cut workflow run that requested this build
required: true
type: string
+ qualification_only:
+ description: Sign and notarize both architectures without publishing a release
+ required: false
+ default: false
+ type: boolean
javascript_source_sha:
description: Commit of the shared JavaScript artifact; empty keeps the legacy build
required: false
@@ -25,12 +30,23 @@ permissions:
contents: write
concurrency:
- group: release-mac-build-${{ inputs.tag }}
+ # Why: a qualification run that reuses a live tag must never queue behind or replace its release build.
+ group: release-mac-build-${{ inputs.tag }}${{ inputs.qualification_only && '-qualification' || '' }}
cancel-in-progress: false
jobs:
+ qualification-template:
+ if: inputs.qualification_only
+ permissions:
+ contents: read
+ uses: ./.github/workflows/node-server-tests.yml
+ with:
+ ref: ${{ github.sha }}
+ build_template: true
+
build-mac:
- if: github.repository == 'zpyoung/orca'
+ needs: qualification-template
+ if: ${{ !cancelled() && (!inputs.qualification_only || needs.qualification-template.result == 'success') && github.repository == 'zpyoung/orca' }}
# Fork: GitHub-hosted macOS runner (no Blacksmith on this fork). Signs +
# notarizes with the fork's own Apple Developer ID secrets.
runs-on: macos-15
@@ -45,12 +61,13 @@ jobs:
actions: read
contents: write
env:
+ ORCA_BACKGROUND_LAUNCH: '1'
NODE_OPTIONS: --max-old-space-size=4096
steps:
- name: Checkout
uses: actions/checkout@v6
with:
- ref: refs/tags/${{ inputs.tag }}
+ ref: ${{ inputs.qualification_only && github.sha || format('refs/tags/{0}', inputs.tag) }}
# Why: a relay-era ref (before #27215) needs the Windows relay addon this build no longer compiles.
- name: Refuse a ref that still ships the SSH relay
@@ -198,7 +215,7 @@ jobs:
with:
name: orcad-template
path: out/orcad-template
- run-id: ${{ inputs.release_run_id }}
+ run-id: ${{ inputs.qualification_only && github.run_id || inputs.release_run_id }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Gate runtime file-watcher process isolation
@@ -209,6 +226,7 @@ jobs:
ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs
- name: Abort if the parent release-cut run was cancelled
+ if: ${{ !inputs.qualification_only }}
# Why: this workflow is dispatched separately, so cancelling release-cut
# does not stop mac `--publish always`. A cancelled parent left v1.4.206
# public with only a partial mac upload.
@@ -234,10 +252,12 @@ jobs:
timeout_minutes: 70
max_attempts: 2
retry_wait_seconds: 30
- command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always -c.publish.releaseType=draft
+ # Fork: no notarization-overlap builder; it signs x64 and arm64 together and this channel ships arm64 only.
+ command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish "$ORCA_MAC_PUBLISH" -c.publish.releaseType=draft
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ORCA_REQUIRE_ORCAD_TEMPLATE: '1'
+ ORCA_MAC_PUBLISH: ${{ inputs.qualification_only && 'never' || 'always' }}
CSC_LINK: ${{ secrets.MAC_CERTS }}
CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }}
APPLE_ID: ${{ secrets.APPLE_ID }}
@@ -245,6 +265,7 @@ jobs:
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
- name: Verify release remains draft after artifact upload
+ if: ${{ !inputs.qualification_only }}
# Why: re-draft immediately if electron-builder flipped the GitHub
# release public, then fail. Checking without restoring leaves
# /releases/latest serving a missing Windows exe.
@@ -259,3 +280,16 @@ jobs:
# point between pack and upload without splitting those steps.
- name: Verify telemetry constants present in app.asar
run: node config/scripts/verify-telemetry-constants.mjs
+
+ - name: Upload qualification artifacts
+ if: inputs.qualification_only
+ uses: actions/upload-artifact@v7
+ with:
+ name: mac-notarization-qualification
+ path: |
+ dist/*.dmg
+ dist/*.zip
+ dist/*.blockmap
+ dist/*.yml
+ if-no-files-found: error
+ retention-days: 7
diff --git a/.github/workflows/ssh-hostile-hosts.yml b/.github/workflows/ssh-hostile-hosts.yml
index 8b1f7f7688c8..d1a3a15eee65 100644
--- a/.github/workflows/ssh-hostile-hosts.yml
+++ b/.github/workflows/ssh-hostile-hosts.yml
@@ -1,9 +1,9 @@
name: SSH hostile hosts
# Hostile-host matrix: the real client-side managed orcad deploy against container SSH targets
-# (old glibc, musl, no libstdc++, noexec home, no egress) and against each macOS runner's own
+# (old glibc, musl, no libstdc++, noexec home, no egress on either side) and against each macOS runner's own
# loopback sshd, asserting the runtime each host resolves and whether orcad activates there.
-# Heavy (three slot builds plus seven images), so it runs only when the deploy, the runtime store
+# Heavy (three slot builds plus eight images), so it runs only when the deploy, the runtime store
# or the slot build changes, and on demand.
on:
pull_request:
@@ -27,7 +27,10 @@ on:
- 'config/scripts/verify-packaged-orcad-template.cjs'
- 'config/patches/node-pty*'
# The deploy runs over this SSH transport.
- - 'src/main/ssh/ssh-connection*'
+ - 'src/main/ssh/connection/**'
+ - 'src/main/ssh/ssh-reconnect-error-classification.ts'
+ - 'src/main/ssh/ssh-posix-command-wrapper.ts'
+ - 'src/shared/shell-argument-quoting.ts'
- '!src/**/*.test.ts'
- 'src/main/ssh/hostile-host-test-support/ssh-hostile-hosts.docker.test.ts'
- '.github/workflows/ssh-hostile-hosts.yml'
diff --git a/.github/workflows/ssh-windows-hosts.yml b/.github/workflows/ssh-windows-hosts.yml
index 87c9f8378031..82fcb984f553 100644
--- a/.github/workflows/ssh-windows-hosts.yml
+++ b/.github/workflows/ssh-windows-hosts.yml
@@ -43,7 +43,10 @@ on:
- 'src/shared/windows-breakaway-launch*.ts'
- 'src/main/ipc/ssh/ssh-host-server-*.ts'
# The deploy runs over this SSH transport.
- - 'src/main/ssh/ssh-connection*'
+ - 'src/main/ssh/connection/**'
+ - 'src/main/ssh/ssh-reconnect-error-classification.ts'
+ - 'src/main/ssh/ssh-posix-command-wrapper.ts'
+ - 'src/shared/shell-argument-quoting.ts'
- '!src/**/*.test.ts'
- 'src/main/managed-server/host-state/orcad-windows-host-lane.test.ts'
- 'tests/e2e/ssh-orcad-auto-convert.spec.ts'
diff --git a/.github/workflows/terminal-perf.yml b/.github/workflows/terminal-perf.yml
index 58d93671d31a..c95854d49ee8 100644
--- a/.github/workflows/terminal-perf.yml
+++ b/.github/workflows/terminal-perf.yml
@@ -88,7 +88,7 @@ jobs:
const shared = process.env.RUNNER_KIND === 'github-hosted' && !process.env.JOB_CONTAINER &&
process.env.RUNNER_OS === 'Linux' && process.env.RUNNER_ARCH === 'X64' &&
manifest.engines?.node === '24' && typeof manifest.packageManager === 'string' &&
- manifest.packageManager.split('+')[0] === 'pnpm@12.8.1' &&
+ manifest.packageManager.split('+')[0] === 'pnpm@12.14.0' &&
supportedPostinstalls.includes(manifest.scripts?.postinstall) &&
/^ native-runtime:/m.test(inputs) && /^ cache-pnpm-store-lookup-only:/m.test(inputs) &&
fs.existsSync('.github/actions/prepare-native-runtime/action.yml') &&
diff --git a/.github/workflows/unit-plan.yml b/.github/workflows/unit-plan.yml
index b427908b26d1..acc6a8fc500a 100644
--- a/.github/workflows/unit-plan.yml
+++ b/.github/workflows/unit-plan.yml
@@ -39,6 +39,7 @@ jobs:
id: plan
env:
ORCA_UNIT_SELECTION_MODE: ${{ inputs.selection_mode }}
+ ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }}
run: node config/scripts/ci-unit-plan.mjs
- uses: actions/upload-artifact@v7
continue-on-error: true
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 180bd6c6e156..6f705b57f8dd 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -260,8 +260,8 @@ jobs:
if: ${{ !cancelled() && steps.root_dependencies.outcome == 'success' }}
working-directory: cloud
run: |
- npx --yes pnpm@10.34.6 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
- npx --yes pnpm@10.34.6 --filter '@orca-cloud/relay^...' build
+ pnpm --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts
+ pnpm --filter '@orca-cloud/relay^...' build
- name: Test relay integration contracts
if: ${{ !cancelled() && steps.relay_dependencies.outcome == 'success' }}
diff --git a/.github/workflows/wsl-orcad-e2e.yml b/.github/workflows/wsl-orcad-e2e.yml
index e33250d06e4c..5d06bc260252 100644
--- a/.github/workflows/wsl-orcad-e2e.yml
+++ b/.github/workflows/wsl-orcad-e2e.yml
@@ -3,24 +3,38 @@ name: WSL orcad E2E
# WSL via orcad (step C10) on real WSL2 distros: a GitHub-hosted windows-2025 runner imports a
# pinned Ubuntu 24.04 rootfs twice (systemd off and on) and runs
# src/main/wsl/wsl-orcad-real-distro-lane.test.ts against both: deploy and launch, the wsl.exe
-# stdio bridge, runtime RPC, liveness recovery, holder survival and teardown. Not a required check.
+# stdio bridge, runtime RPC, liveness recovery, holder survival, structured chat and teardown. Not a required check.
# Hosted WSL2 was proven in #27783: WSL 2.7.14, import 13-18 s, cold boot 8-14 s.
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
paths:
- 'src/main/wsl/**'
- - '!src/main/wsl/**/*.test.ts'
- - 'src/main/wsl/wsl-orcad-real-distro-lane.test.ts'
- # This lane executes only the real distro test above.
+ - 'src/main/runtime/**'
+ - 'src/main/native-chat/**'
+ - 'src/main/claude/**'
+ - 'src/main/claude-accounts/**'
+ - 'src/main/codex/**'
+ - 'src/main/acp/**'
+ - 'src/main/pi/**'
+ - 'src/main/orcad/**'
+ - 'src/main/server/**'
+ - 'src/main/persistence/**'
+ - 'mobile/src/**'
+ - 'src/main/ipc/*wsl*'
+ - 'src/shared/**'
+ - 'src/renderer/src/components/native-chat/**'
+ - 'src/renderer/src/lib/*agent-session*'
+ - 'config/scripts/*orcad*'
+ - 'package.json'
+ - 'pnpm-lock.yaml'
- 'src/main/ssh/orcad-*'
- 'src/main/managed-server/**'
- - '!src/main/ssh/orcad-*.test.ts'
- - '!src/main/managed-server/**/*.test.ts'
- - 'src/main/managed-server/install/remote-install-transport.ts'
- # orcad-entry runs this on every boot of the orcad the lane launches in the distro.
- - 'src/main/orcad/orcad-wsl-*'
- - '!src/main/orcad/orcad-wsl-*.test.ts'
+ - 'src/packages/process-host/**'
+ # Unit tests never run in this lane; only the real distro test below does.
+ - '!**/*.test.ts'
+ - '!**/*.test.tsx'
+ - 'src/main/wsl/wsl-orcad-real-distro-lane.test.ts'
- '.github/workflows/wsl-orcad-e2e.yml'
workflow_dispatch:
@@ -175,6 +189,7 @@ jobs:
# A non-root default user, as on a real install; git for the lane's repo.
$setup = "set -e; id orca >/dev/null 2>&1 || useradd -m -s /bin/bash orca; " +
"printf '[boot]\nsystemd=$($d.Systemd)\n[user]\ndefault=orca\n' > /etc/wsl.conf; " +
+ "printf 'export ORCA_BACKGROUND_LAUNCH=1\nexport USERPROFILE=`$HOME\nexport APPDATA=`$HOME/.config\nexport LOCALAPPDATA=`$HOME/.local/share\nexport XDG_CONFIG_HOME=`$HOME/.config\nexport XDG_DATA_HOME=`$HOME/.local/share\nexport XDG_CACHE_HOME=`$HOME/.cache\nexport XDG_STATE_HOME=`$HOME/.local/state\nexport CLAUDE_CONFIG_DIR=`$HOME/.claude\nexport CODEX_HOME=`$HOME/.codex\n' >> /home/orca/.profile; " +
"command -v git >/dev/null || (apt-get update -qq && apt-get install -y -qq git)"
wsl.exe -d $d.Name -u root --exec sh -c $setup
if ($LASTEXITCODE -ne 0) { throw "provisioning $($d.Name) failed: $LASTEXITCODE" }
@@ -201,6 +216,20 @@ jobs:
with:
path: ${{ runner.temp }}/wsl-rootfs
key: wsl-orcad-rootfs-${{ env.ROOTFS_SHA256 }}
+ - name: Isolate the desktop test home
+ run: |
+ New-Item -ItemType Directory -Force $env:ORCA_WSL_ORCAD_LANE_RECEIPTS | Out-Null
+ $files = @('.claude/settings.json', '.claude.json', '.codex/hooks.json', '.codex/config.toml') | ForEach-Object { Join-Path $env:USERPROFILE $_ }
+ $hashes = @($files | ForEach-Object {
+ @{ Path = $_; Hash = $(if (Test-Path $_) { (Get-FileHash $_ -Algorithm SHA256).Hash } else { 'absent' }) }
+ })
+ $hashes | ConvertTo-Json | Set-Content (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'real-home-hashes.json')
+ $homeDir = Join-Path $env:RUNNER_TEMP 'wsl-orcad-desktop-home'
+ foreach ($name in @('HOME', 'USERPROFILE', 'APPDATA', 'LOCALAPPDATA', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_CACHE_HOME', 'XDG_STATE_HOME', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME')) {
+ $path = Join-Path $homeDir $name
+ New-Item -ItemType Directory -Force $path | Out-Null
+ "$name=$path" >> $env:GITHUB_ENV
+ }
- name: 'Phase 1: deploy, RPC, kill orcad, terminate distro'
timeout-minutes: 20
env:
@@ -212,8 +241,8 @@ jobs:
node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/wsl/wsl-orcad-real-distro-lane.test.ts --reporter=verbose 2>&1 | Tee-Object (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'launch.log')
if ($global:LASTEXITCODE -ne 0) { throw "launch phase failed: $global:LASTEXITCODE" }
# A new process: the holders must have kept both distros Running since phase 1 exited.
- - name: 'Phase 2: holders survive, reconnect, teardown'
- timeout-minutes: 10
+ - name: 'Phase 2: holders survive, chat, reconnect, teardown'
+ timeout-minutes: 15
env:
ORCA_RUN_WSL_ORCAD_LANE: '1'
ORCA_WSL_ORCAD_LANE_PHASE: survive
@@ -221,6 +250,16 @@ jobs:
run: |
node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/wsl/wsl-orcad-real-distro-lane.test.ts --reporter=verbose 2>&1 | Tee-Object (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'survive.log')
if ($global:LASTEXITCODE -ne 0) { throw "survive phase failed: $global:LASTEXITCODE" }
+ - name: Real desktop agent configuration is unchanged
+ if: always()
+ run: |
+ $baseline = Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'real-home-hashes.json'
+ if (Test-Path $baseline) {
+ foreach ($file in (Get-Content $baseline -Raw | ConvertFrom-Json)) {
+ $hash = if (Test-Path $file.Path) { (Get-FileHash $file.Path -Algorithm SHA256).Hash } else { 'absent' }
+ if ($hash -ne $file.Hash) { throw "Real agent configuration changed: $($file.Path)" }
+ }
+ }
# Belt and braces for the runner: a missing VM-wide interop entry is restored, then fails.
- name: WSLInterop is intact
if: ${{ !cancelled() && steps.import.outcome == 'success' }}
diff --git a/README.md b/README.md
index f0623fff44d4..dc6b13791676 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ Ghostty-class terminals with WebGL rendering, infinite splits, and scrollback th
-
+
@@ -96,7 +96,7 @@ Browse PRs, issues, and project boards in-app — open a worktree from any task
-
+
@@ -110,7 +110,7 @@ Run agents on a beefy remote box with full file editing, git, and terminals —
-
+
@@ -152,7 +152,7 @@ Agents drive Orca too — script every workflow with `orca worktree create`, `sn
-
+
@@ -242,9 +242,9 @@ Pair with your desktop app to monitor and steer your agents from your phone.
- **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**.
- **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements.
-- **WeChat:** Group 11 is full. Scan to join the Orca community WeChat group 12.
+- **WeChat:** Group 12 is full. Scan to join the Orca community WeChat group 13.
-
+
- **Feedback & Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues).
- **Privacy:** See the [privacy & telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out.
diff --git a/cloud/.npmrc b/cloud/.npmrc
deleted file mode 100644
index e05d6820ed56..000000000000
--- a/cloud/.npmrc
+++ /dev/null
@@ -1,3 +0,0 @@
-engine-strict=false
-package-manager-strict=true
-
diff --git a/cloud/apps/push/src/durable-push-worker.test.ts b/cloud/apps/push/src/durable-push-worker.test.ts
index 7cbd6074b360..615689a320b3 100644
--- a/cloud/apps/push/src/durable-push-worker.test.ts
+++ b/cloud/apps/push/src/durable-push-worker.test.ts
@@ -1,3 +1,4 @@
+import { setImmediate } from 'node:timers/promises'
import { afterEach, expect, it, vi } from 'vitest'
import type { PushNotification } from '@orca-cloud/push-contract'
import { DurablePushStore } from './durable-push-store.js'
@@ -7,6 +8,7 @@ import { PushDeviceRegistryStore } from './device-registry-store.js'
import { openInMemoryPushDatabase } from './push-database.js'
import type { PushDelivery } from './push-delivery-message.js'
import type { PushProviderOutcome } from './push-provider-outcome.js'
+import { WORKER_DRAINS } from './push-worker-concurrency.js'
const cleanups: (() => Promise)[] = []
afterEach(async () => {
@@ -119,8 +121,9 @@ it('keeps untrackable bells and per-phone deliveries individually replaceable',
).toMatchObject({ orca: { notificationId: 'note-2', notificationSeq: 2 } })
})
-it('persists provider retry delay and resumes it through a new worker', async () => {
+it('preserves provider retry delay when waking a restarted worker', async () => {
const h = await fixture()
+ vi.useFakeTimers()
h.send.mockResolvedValueOnce({
status: 'error',
reason: 'busy',
@@ -132,10 +135,13 @@ it('persists provider retry delay and resumes it through a new worker', async ()
expect(h.send).toHaveBeenCalledOnce()
await h.worker.stop()
const restarted = new DurablePushWorker(h.store, h.dispatcher, { now: h.now, onRetry: h.onRetry })
+ restarted.start()
h.advance(9999)
+ restarted.wake()
await restarted.runDue()
expect(h.send).toHaveBeenCalledOnce()
h.advance(1)
+ restarted.wake()
await restarted.runDue()
expect(h.send).toHaveBeenCalledTimes(2)
expect(h.send.mock.calls.map(([delivery]) => delivery.expiresAt)).toEqual([1_300_000, 1_300_000])
@@ -175,6 +181,7 @@ it('rechecks the device before a persisted retry and does not send after unregis
it('joins active work on shutdown and leaves unclaimed work for the next instance', async () => {
const h = await fixture()
+ vi.useFakeTimers()
let finish!: (outcome: PushProviderOutcome) => void
let started!: () => void
const entered = new Promise((resolve) => {
@@ -187,9 +194,12 @@ it('joins active work on shutdown and leaves unclaimed work for the next instanc
})
})
await h.accept(note(1))
+ h.worker.start()
+ h.worker.wake()
const pending = h.worker.runDue()
await entered
await h.accept(note(2))
+ h.worker.wake()
let stopped = false
const stopping = h.worker.stop().then(() => {
stopped = true
@@ -200,7 +210,10 @@ it('joins active work on shutdown and leaves unclaimed work for the next instanc
await Promise.all([pending, stopping])
expect(stopped).toBe(true)
expect(h.send).toHaveBeenCalledOnce()
+ expect(vi.getTimerCount()).toBe(0)
const resumed = new DurablePushWorker(h.store, h.dispatcher, { now: h.now })
+ resumed.start()
+ await vi.advanceTimersByTimeAsync(1000)
await resumed.runDue()
expect(h.send).toHaveBeenCalledTimes(2)
await resumed.stop()
@@ -219,3 +232,129 @@ it('runs due work on its timer and releases the timer on stop', async () => {
await h.worker.stop()
expect(vi.getTimerCount()).toBe(0)
})
+
+it('wakes only while started and keeps the durability poll', async () => {
+ const h = await fixture()
+ vi.useFakeTimers()
+ const claim = vi.spyOn(h.store, 'claim')
+ await h.accept(note(1))
+ h.worker.wake()
+ expect(claim).not.toHaveBeenCalled()
+ h.worker.start()
+ h.worker.wake()
+ expect(claim).toHaveBeenCalled()
+ await h.worker.runDue()
+ expect(h.send).toHaveBeenCalledOnce()
+ expect(vi.getTimerCount()).toBe(1)
+ await h.worker.stop()
+ const calls = claim.mock.calls.length
+ await h.accept(note(2))
+ h.worker.wake()
+ await vi.advanceTimersByTimeAsync(2000)
+ expect(claim).toHaveBeenCalledTimes(calls)
+ expect(await h.store.pendingCount(h.registrationId)).toBe(1)
+ expect(vi.getTimerCount()).toBe(0)
+})
+
+it('coalesces wakes arriving after every active drain found an empty queue', async () => {
+ const h = await fixture()
+ vi.useFakeTimers()
+ const entered = Promise.withResolvers()
+ const release = Promise.withResolvers()
+ const claim = h.store.claim.bind(h.store)
+ let empty = 0
+ let holding = true
+ vi.spyOn(h.store, 'claim').mockImplementation(async () => {
+ const delivery = await claim()
+ if (!delivery && holding) {
+ if (++empty === WORKER_DRAINS) entered.resolve()
+ await release.promise
+ }
+ return delivery
+ })
+ h.worker.start()
+ const startedAt = Date.now()
+ const pending = h.worker.runDue()
+ try {
+ await entered.promise
+ await h.accept(note(1))
+ for (let count = 0; count < 10; count++) h.worker.wake()
+ expect(h.store.claim).toHaveBeenCalledTimes(WORKER_DRAINS)
+ holding = false
+ release.resolve()
+ await pending
+ await vi.waitFor(() => expect(h.send).toHaveBeenCalledOnce())
+ await vi.waitFor(() => expect(h.store.claim).toHaveBeenCalledTimes(WORKER_DRAINS + 2))
+ expect(Date.now() - startedAt).toBeLessThan(1000)
+ expect(h.send.mock.calls[0]?.[0].orca.notificationSeq).toBe(1)
+ } finally {
+ holding = false
+ release.resolve()
+ }
+})
+
+it('bounds claims by sends plus drains for a burst and a trickle of wakes', async () => {
+ const h = await fixture()
+ const registrations: string[] = []
+ for (let index = 0; index < 50; index++) {
+ const device = await h.devices.upsert({
+ hostFingerprint: `host-${index}`,
+ deviceId: `phone-${index}`,
+ platform: 'android',
+ token: `token-${index}`
+ })
+ if (!device.ok) throw new Error('registration failed')
+ registrations.push(device.registrationId)
+ }
+ const claim = vi.spyOn(h.store, 'claim')
+ const sendOne = async (index: number) => {
+ const status = await h.store.accept(
+ `host-${index % 50}`,
+ registrations[index % 50]!,
+ note(index + 1)
+ )
+ expect(status).toBe('queued')
+ h.worker.wake()
+ }
+ // Fake timers hold the poll, so only wakes claim.
+ vi.useFakeTimers()
+ h.worker.start()
+ const settle = async () => {
+ for (let ticks = 0; ticks < 1000; ticks++) await setImmediate()
+ }
+ await Promise.all(Array.from({ length: 1000 }, (_, index) => sendOne(index)))
+ await settle()
+ // Each drain stops after 25 sends, so the poll finishes a large burst.
+ let polls = 0
+ while (h.send.mock.calls.length < 1000 && polls < 10) {
+ await vi.advanceTimersByTimeAsync(1000)
+ await settle()
+ polls++
+ }
+ expect(h.send).toHaveBeenCalledTimes(1000)
+ expect(claim.mock.calls.length).toBeLessThanOrEqual(1000 + WORKER_DRAINS * (polls + 2))
+ claim.mockClear()
+ // Before one-drain wakes, each idle send started every drain: ~13 claims per send.
+ for (let index = 1000; index < 1020; index++) {
+ await sendOne(index)
+ await settle()
+ }
+ expect(h.send).toHaveBeenCalledTimes(1020)
+ expect(claim).toHaveBeenCalledTimes(40)
+}, 60_000)
+
+it('recovers from a failed automatic wake without leaving another timer', async () => {
+ const h = await fixture()
+ vi.useFakeTimers()
+ vi.spyOn(h.store, 'claim').mockRejectedValueOnce(new Error('database unavailable'))
+ h.worker.start()
+ h.worker.wake()
+ await vi.waitFor(() =>
+ expect(console.warn).toHaveBeenCalledWith(JSON.stringify({ event: 'orca_push_worker_failed' }))
+ )
+ await h.accept(note(1))
+ h.worker.wake()
+ await vi.waitFor(() => expect(h.send).toHaveBeenCalledOnce())
+ await h.worker.stop()
+ expect(vi.getTimerCount()).toBe(0)
+})
diff --git a/cloud/apps/push/src/durable-push-worker.ts b/cloud/apps/push/src/durable-push-worker.ts
index f5bcc20fc2c4..77eacb248843 100644
--- a/cloud/apps/push/src/durable-push-worker.ts
+++ b/cloud/apps/push/src/durable-push-worker.ts
@@ -5,8 +5,9 @@ import { WORKER_DRAINS } from './push-worker-concurrency.js'
export class DurablePushWorker {
private timer?: NodeJS.Timeout
- private running: Promise | null = null
+ private readonly drains = new Set>()
private stopped = false
+ private wakePending = false
constructor(
private readonly store: DurablePushStore,
private readonly dispatcher: PushDispatcher,
@@ -17,32 +18,45 @@ export class DurablePushWorker {
if (this.timer) return
this.stopped = false
this.timer = setInterval(() => {
- void this.runDue().catch(() => {
- console.warn(JSON.stringify({ event: 'orca_push_worker_failed' }))
- })
+ void this.runDue().catch(warnWorkerFailed)
}, 1000)
this.timer.unref()
}
- async runDue(): Promise {
- if (this.running) {
- await this.running
+ // One drain per wake keeps claims O(sends + drains); the poll restores full concurrency.
+ wake(): void {
+ if (!this.timer || this.stopped) return
+ if (this.drains.size >= WORKER_DRAINS) {
+ this.wakePending = true
return
}
+ this.startDrain().catch(warnWorkerFailed)
+ }
+
+ async runDue(): Promise {
if (this.stopped) return
- const drains = Array.from({ length: WORKER_DRAINS }, () => this.drain())
- const pending = Promise.allSettled(drains).then(
- (results) => {
- const failure = results.find((result) => result.status === 'rejected')
- if (failure?.status === 'rejected') throw failure.reason
- }
+ const started = Array.from({ length: WORKER_DRAINS - this.drains.size }, () =>
+ this.startDrain()
)
- this.running = pending
- try {
- await pending
- } finally {
- this.running = null
- }
+ const active = Promise.allSettled(this.drains)
+ const failure = (await Promise.allSettled(started)).find(
+ (result) => result.status === 'rejected'
+ )
+ await active
+ if (failure?.status === 'rejected') throw failure.reason
+ }
+
+ private startDrain(): Promise {
+ const drain: Promise = this.drain().finally(() => {
+ this.drains.delete(drain)
+ // A committed enqueue can arrive after every drain has found an empty queue.
+ if (this.wakePending) {
+ this.wakePending = false
+ this.wake()
+ }
+ })
+ this.drains.add(drain)
+ return drain
}
private async drain(): Promise {
@@ -84,8 +98,13 @@ export class DurablePushWorker {
async stop(): Promise {
this.stopped = true
+ this.wakePending = false
if (this.timer) clearInterval(this.timer)
this.timer = undefined
- await this.running
+ await Promise.allSettled(this.drains)
}
}
+
+function warnWorkerFailed(): void {
+ console.warn(JSON.stringify({ event: 'orca_push_worker_failed' }))
+}
diff --git a/cloud/apps/push/src/push-server-wake.test.ts b/cloud/apps/push/src/push-server-wake.test.ts
new file mode 100644
index 000000000000..156e8936ce31
--- /dev/null
+++ b/cloud/apps/push/src/push-server-wake.test.ts
@@ -0,0 +1,122 @@
+import { afterEach, expect, it, vi } from 'vitest'
+import { startPushBackground } from './push-background.js'
+import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js'
+import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js'
+
+const cleanups: (() => Promise)[] = []
+afterEach(async () => {
+ for (const cleanup of cleanups.splice(0)) await cleanup()
+ vi.useRealTimers()
+ vi.restoreAllMocks()
+})
+
+async function fixture() {
+ const h = await createPushServerHarness()
+ const token = await h.signIn(createPushHostKeypair(31))
+ const registrationId = await h.registerAndroid(token)
+ let stop = async () => {}
+ cleanups.push(async () => {
+ await stop()
+ await h.close()
+ })
+ vi.useFakeTimers()
+ return {
+ ...h,
+ registrationId,
+ start: () => {
+ stop = startPushBackground({ mode: 'active' }, h.server)
+ },
+ send: (event = notification(), ids = [registrationId]) =>
+ h.post('/v1/send', { v: 1, registrationIds: ids, notification: event }, token),
+ invalid: () => h.post('/v1/send', { v: 1 }, token)
+ }
+}
+
+it('delivers a committed send before the next poll without waiting in the HTTP handler', async () => {
+ const h = await fixture()
+ h.start()
+ const startedAt = Date.now()
+ const claim = h.server.deliveryStore.claim.bind(h.server.deliveryStore)
+ const release = Promise.withResolvers()
+ vi.spyOn(h.server.deliveryStore, 'claim').mockImplementation(async () => {
+ await release.promise
+ return claim()
+ })
+ try {
+ const response = await h.send()
+ expect(await response.json()).toEqual({
+ results: [{ registrationId: h.registrationId, status: 'queued' }]
+ })
+ expect(h.fcmRequests).toHaveLength(0)
+ release.resolve()
+ await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1))
+ expect(Date.now() - startedAt).toBeLessThan(1000)
+ expect(JSON.parse(h.fcmRequests[0]!.body).message.data.notificationSeq).toBe('1')
+ } finally {
+ release.resolve()
+ }
+})
+
+it('keeps concurrent accepted retries idempotent while automatic delivery runs', async () => {
+ const h = await fixture()
+ h.start()
+ await Promise.all(Array.from({ length: 10 }, () => h.send()))
+ await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1))
+ await h.send()
+ await h.server.worker.runDue()
+ expect(h.fcmRequests).toHaveLength(1)
+ expect(await h.server.deliveryStore.pendingCount(h.registrationId)).toBe(0)
+})
+
+it('does not wake for invalid, unowned or dead registrations', async () => {
+ const h = await fixture()
+ h.start()
+ const wake = vi.spyOn(h.server.worker, 'wake')
+ expect((await h.invalid()).status).toBe(400)
+ expect(await (await h.send(notification(), ['unknown'])).json()).toEqual({
+ results: [{ registrationId: 'unknown', status: 'error' }]
+ })
+ const device = await h.server.devices.findById(h.registrationId)
+ if (!device) throw new Error('registration missing')
+ await h.server.devices.markDead(device)
+ expect(await (await h.send()).json()).toEqual({
+ results: [{ registrationId: h.registrationId, status: 'dead' }]
+ })
+ expect(wake).not.toHaveBeenCalled()
+ expect(h.fcmRequests).toHaveLength(0)
+})
+
+it.each(['error', 'rate_limited'] as const)('does not wake a refused %s send', async (status) => {
+ const h = await fixture()
+ h.start()
+ const wake = vi.spyOn(h.server.worker, 'wake')
+ vi.spyOn(h.server.deliveryStore, 'accept').mockResolvedValue(status)
+ expect(await (await h.send()).json()).toEqual({
+ results: [{ registrationId: h.registrationId, status }]
+ })
+ expect(wake).not.toHaveBeenCalled()
+})
+
+it.each(['queued', 'sent'] as const)(
+ 'preserves dismissal of an already %s alert',
+ async (phase) => {
+ const h = await fixture()
+ if (phase === 'sent') h.start()
+ await h.send()
+ if (phase === 'sent') await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1))
+ await h.send(notification({ kind: 'dismiss', notificationSeq: 2, sound: false }))
+ if (phase === 'queued') {
+ h.start()
+ await vi.advanceTimersByTimeAsync(1000)
+ }
+ await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(phase === 'sent' ? 2 : 1))
+ const last = h.fcmRequests.at(-1)
+ if (!last) throw new Error('dismissal missing')
+ const message = JSON.parse(last.body).message
+ expect(message.data.kind).toBe('dismiss')
+ expect(message.data.notificationId).toBe('note-1')
+ expect(message.data.notificationSeq).toBe('2')
+ expect(message.data.title).toBeUndefined()
+ expect(message.data.sound).toBeUndefined()
+ }
+)
diff --git a/cloud/apps/push/src/push-server.ts b/cloud/apps/push/src/push-server.ts
index 0aee3765570a..17c72b14b160 100644
--- a/cloud/apps/push/src/push-server.ts
+++ b/cloud/apps/push/src/push-server.ts
@@ -279,6 +279,7 @@ export function createPushServer(
observability.record('send_queued')
results.push({ registrationId, status: 'queued' })
}
+ if (results.some((result) => result.status === 'queued')) worker.wake()
return context.json({ results })
})
diff --git a/cloud/apps/relay-ops/src/environment-config.test.ts b/cloud/apps/relay-ops/src/environment-config.test.ts
index 321167d7cc8a..f3522a300cd3 100644
--- a/cloud/apps/relay-ops/src/environment-config.test.ts
+++ b/cloud/apps/relay-ops/src/environment-config.test.ts
@@ -64,6 +64,7 @@ function terraformCells(environment: 'production' | 'staging'): Array<{
machineType: string
capacityRequests: number
databasePoolMax: number
+ connectionHardCap: number | null
}> {
const tfvars = readFileSync(
fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)),
@@ -77,7 +78,10 @@ function terraformCells(environment: 'production' | 'staging'): Array<{
zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '',
capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]),
- databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10)
+ databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10),
+ connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))(
+ /connection_hard_cap\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]
+ )
}))
.sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId))
}
@@ -97,7 +101,8 @@ describe('relay operations environment config', () => {
zone: cell.zone,
machineType: cell.machineType,
capacityRequests: cell.capacityRequests,
- databasePoolMax: cell.databasePoolMax
+ databasePoolMax: cell.databasePoolMax,
+ connectionHardCap: cell.connectionHardCap
}))
).toEqual(expected)
}
@@ -122,17 +127,20 @@ describe('relay operations environment config', () => {
{
cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev',
region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4',
- capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+ capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
+ configuredAdmission: false
},
{
cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev',
region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4',
- capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+ capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
+ configuredAdmission: false
},
{
cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev',
region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4',
- capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false
+ capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null,
+ configuredAdmission: false
}
])
diff --git a/cloud/apps/relay-ops/src/environment-config.ts b/cloud/apps/relay-ops/src/environment-config.ts
index da3de7765edf..efd890367c8a 100644
--- a/cloud/apps/relay-ops/src/environment-config.ts
+++ b/cloud/apps/relay-ops/src/environment-config.ts
@@ -14,6 +14,8 @@ export type RelayOpsCellConfig = {
machineType: RelayOpsMachineType
capacityRequests: number
databasePoolMax: number
+ // The cell's rendered connectionHardCap; null when Terraform leaves it unset.
+ connectionHardCap: number | null
configuredAdmission: boolean
}
@@ -74,6 +76,9 @@ export function relayOpsCellsFromTerraform(input: {
required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`)
),
databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10),
+ connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))(
+ /\bconnection_hard_cap\s*=\s*(\d+)/.exec(body)?.[1]
+ ),
configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true'
}
})
diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
index 32319df7f759..704f55717fa2 100644
--- a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
+++ b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts
@@ -543,6 +543,92 @@ describe('incident monitor sources', () => {
expect(waits).toEqual([])
})
+ // Why: on 2026-10-19 cell-status refused reserve production-gce-c7 with a 404 on every sample.
+ it('marks a reserve cell unprobed and skips its refused migration reads', async () => {
+ const gcloud: GcloudClient = {
+ accessToken: async () => 'unused',
+ identityToken: async () => 'unused'
+ }
+ const reserveCellId = 'production-gce-c7'
+ const selector = {
+ generation: 1,
+ membership: {
+ existingOnly: [],
+ migrationOnly: ['production-gce-c2'],
+ general: productionCells.filter((cellId) => cellId !== 'production-gce-c2')
+ }
+ }
+ const waits: number[] = []
+ const fetchImpl: typeof fetch = async (input, init) => {
+ const path = new URL(String(input)).pathname
+ const body = JSON.parse(String(init?.body)) as Record
+ if (path === '/v1/admin/admission-selector/status') return Response.json({ selector })
+ if (path === '/v1/admin/cell-status') {
+ if (body.cellId === reserveCellId) {
+ return Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 })
+ }
+ return Response.json({
+ status: {
+ enabled: true,
+ connectionCapacity: { hardCap: 600 },
+ runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true }
+ }
+ })
+ }
+ expect(path).toBe('/v1/admin/evacuation-status')
+ expect(body.sourceCellId).not.toBe(reserveCellId)
+ return Response.json({
+ blocked: 0,
+ blockedExpiredUnregistered: 0,
+ registeredTargetInactive: 0
+ })
+ }
+ const result = await directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => {
+ waits.push(ms)
+ })
+ expect(waits).toEqual([])
+ expect(result.cells.find((cell) => cell.cellId === reserveCellId)).toMatchObject({
+ reserve: true
+ })
+ expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).not.toHaveProperty(
+ 'reserve'
+ )
+ const signals = result.source.signals
+ expect(signals[`cell.${reserveCellId}.admission_state`]).toMatchObject({ value: 2 })
+ // Why: 10-19 run 39225932592 froze reserve c7 at 731 connections against the 500 default.
+ const configuredCap = RELAY_OPS_ENVIRONMENTS.production.cells.find(
+ (cell) => cell.cellId === reserveCellId
+ )?.connectionHardCap
+ expect(configuredCap).toBe(1_000)
+ expect(signals[`cell.${reserveCellId}.connection_hard_cap`]).toMatchObject({
+ value: configuredCap
+ })
+ for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked']) {
+ expect(signals[`cell.${reserveCellId}.${suffix}`]).toBeUndefined()
+ }
+ expect(signals['cell.production-gce-c1.heartbeat_age_ms']).toMatchObject({ value: 1_000 })
+ })
+
+ it('still fails a cell-status 404 that is not the reserve refusal', async () => {
+ const gcloud: GcloudClient = {
+ accessToken: async () => 'unused',
+ identityToken: async () => 'unused'
+ }
+ const selector = {
+ generation: 1,
+ membership: { existingOnly: [], migrationOnly: [], general: productionCells }
+ }
+ const collect = (error: string) =>
+ directorSignals('production', selector, gcloud, now, async (input) => {
+ const path = new URL(String(input)).pathname
+ if (path === '/v1/admin/admission-selector/status') return Response.json({ selector })
+ return Response.json({ error }, { status: 404 })
+ }, async () => {})
+ for (const error of ['cell_not_found', 'cell_in_reserve_mode_soon', '']) {
+ await expect(collect(error)).rejects.toThrow('Relay admin telemetry returned 404')
+ }
+ })
+
// A 404 the director means (wrong role) must not be retried either.
it('does not retry a director-only 404', async () => {
const gcloud: GcloudClient = {
diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts
index 9e041940af7f..f69d57b11bdc 100644
--- a/cloud/apps/relay-ops/src/incident-monitor-sources.ts
+++ b/cloud/apps/relay-ops/src/incident-monitor-sources.ts
@@ -452,10 +452,7 @@ function transientFetchFailure(error: unknown): boolean {
return name === 'AbortError' || name === 'TimeoutError'
}
-function retryableAdminStatus(status: number, body: string): boolean {
- if (ADMIN_RETRYABLE_STATUSES.has(status)) return true
- // 401/403/400/413 and a 404 for 'director_only' are decisions, not weather.
- if (status !== 404) return false
+function adminErrorMessage(body: string): string | null {
const parsed = AdminErrorBodySchema.safeParse(
((): unknown => {
try {
@@ -465,12 +462,27 @@ function retryableAdminStatus(status: number, body: string): boolean {
}
})()
)
+ return parsed.success ? parsed.data.error : null
+}
+
+function retryableAdminStatus(status: number, body: string): boolean {
+ if (ADMIN_RETRYABLE_STATUSES.has(status)) return true
+ // 401/403/400/413 and a 404 for 'director_only' are decisions, not weather.
+ if (status !== 404) return false
+ const message = adminErrorMessage(body)
return (
- parsed.success &&
- ADMIN_TRANSIENT_ERROR_MESSAGES.some((message) => parsed.data.error.includes(message))
+ message !== null &&
+ ADMIN_TRANSIENT_ERROR_MESSAGES.some((transient) => message.includes(transient))
)
}
+// Step 5: cell-status refuses a reserve cell with exactly this 404 body; any other 404 stays a failure.
+class AdminReserveCellRefusal extends Error {
+ constructor(status: number) {
+ super(`Relay admin telemetry returned ${status}`)
+ }
+}
+
async function adminPost(
fetchImpl: typeof fetch,
origin: string,
@@ -502,6 +514,9 @@ async function adminPost(
// Status only: the body carries a director error message, which must not
// reach the log the way an identity would.
const errorBody = await response.text().catch(() => '')
+ if (response.status === 404 && adminErrorMessage(errorBody) === 'cell_in_reserve_mode') {
+ throw new AdminReserveCellRefusal(response.status)
+ }
if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) {
throw new Error(`Relay admin telemetry returned ${response.status}`)
}
@@ -545,23 +560,32 @@ export async function directorSignals(
}
const statuses: Array<{
cell: RelayOpsEnvironment['cells'][number]
- status: z.infer['status']
+ status: ({ reserve: false } & z.infer['status']) | { reserve: true }
}> = []
for (const cell of environment.cells) {
+ let response: unknown
+ try {
+ response = await adminPost(
+ fetchImpl,
+ environment.directorOrigin,
+ token,
+ '/v1/admin/cell-status',
+ { v: 1, cellId: cell.cellId },
+ wait
+ )
+ } catch (error) {
+ if (!(error instanceof AdminReserveCellRefusal)) throw error
+ statuses.push({ cell, status: { reserve: true } })
+ continue
+ }
statuses.push({
cell,
- status: CellStatusSchema.parse(
- await adminPost(
- fetchImpl,
- environment.directorOrigin,
- token,
- '/v1/admin/cell-status',
- { v: 1, cellId: cell.cellId },
- wait
- )
- ).status
+ status: { reserve: false, ...CellStatusSchema.parse(response).status }
})
}
+ const reserveCellIds = new Set(
+ statuses.filter(({ status }) => status.reserve).map(({ cell }) => cell.cellId)
+ )
const migrationEntries: Array<{
sourceCellId: string
migration: z.infer
@@ -570,6 +594,8 @@ export async function directorSignals(
for (const source of environment.cells) {
for (const target of environment.cells) {
if (source.cellId === target.cellId || !migrationTargets.has(target.cellId)) continue
+ // The director refuses migration reads, and every migration, on a reserve cell.
+ if (reserveCellIds.has(source.cellId)) continue
migrationEntries.push({
sourceCellId: source.cellId,
migration: MigrationStatusSchema.parse(
@@ -601,6 +627,24 @@ export async function directorSignals(
const signals: Record = {}
for (const { cell, status } of statuses) {
const prefix = `cell.${cell.cellId}`
+ if (status.reserve) {
+ // cell-status carries the live cap; unset in Terraform keeps the evaluator's default, as there.
+ if (cell.connectionHardCap !== null) {
+ addSignal(signals, `${prefix}.connection_hard_cap`, cell.connectionHardCap, nowAt)
+ }
+ // Legacy admission needs relay_cells.enabled, which only cell-status reads: left missing.
+ if (selector.generation !== 0) {
+ addSignal(
+ signals,
+ `${prefix}.admission_state`,
+ ['existing-only', 'migration-only', 'general'].indexOf(
+ selectorCellState(selector, cell.cellId)
+ ),
+ nowAt
+ )
+ }
+ continue
+ }
const admissionState = effectiveAdmissionState(
selector,
status.enabled,
@@ -645,12 +689,13 @@ export async function directorSignals(
return {
source: { observedAt: nowAt, signals },
selector,
- cells: statuses.map(({ cell }) => ({
+ cells: statuses.map(({ cell, status }) => ({
cellId: cell.cellId,
region: cell.region,
runtimeKnown: true,
powered: true,
- expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId)
+ expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId),
+ ...(status.reserve ? { reserve: true } : {})
}))
}
}
diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts
index db1d8797d1c3..a0eadd1b1636 100644
--- a/cloud/apps/relay-ops/src/incident-monitor.test.ts
+++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts
@@ -643,6 +643,71 @@ describe('incident monitor evaluator', () => {
expect.objectContaining({ code: 'selector_mismatch' })
)
})
+
+ // Why: the director refuses cell-status and migration reads on a reserve cell (2026-10-19, c7).
+ it('judges a reserve cell on probes, logs and selector, without its refused reads', () => {
+ const reserveSample = (): IncidentSample => {
+ const sample = healthySample()
+ sample.cells[0] = { ...sample.cells[0]!, reserve: true }
+ const admin = sample.sources['director-admin']!.signals
+ delete admin['cell.production-gce-c1.heartbeat_fresh']
+ delete admin['cell.production-gce-c1.heartbeat_age_ms']
+ delete admin['cell.production-gce-c1.migration_blocked']
+ delete admin['cell.production-gce-c1.migration_target_inactive']
+ return sample
+ }
+ expect(evaluateIncidentSample(reserveSample(), startedAt)).toMatchObject({
+ status: 'green',
+ failures: []
+ })
+
+ const crowded = reserveSample()
+ crowded.sources['relay-logs']!.signals['cell.production-gce-c1.queued_bytes'] = signal(
+ INCIDENT_MONITOR_THRESHOLDS.cellQueuedBytes + 1
+ )
+ expect(evaluateIncidentSample(crowded, startedAt).failures).toContainEqual(
+ expect.objectContaining({ signal: 'cell.production-gce-c1.queued_bytes' })
+ )
+
+ // The connections ceiling is never skipped: the collector's configured cap, else the default.
+ const busy = reserveSample()
+ busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(731)
+ expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual(
+ expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 500 })
+ )
+ busy.sources['director-admin']!.signals['cell.production-gce-c1.connection_hard_cap'] =
+ signal(1_000)
+ expect(evaluateIncidentSample(busy, startedAt).status).toBe('green')
+ busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(1_000)
+ expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual(
+ expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 999 })
+ )
+
+ const unready = reserveSample()
+ unready.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0)
+ expect(evaluateIncidentSample(unready, startedAt).failures).toContainEqual(
+ expect.objectContaining({ signal: 'cell.production-gce-c1.ready' })
+ )
+
+ const unknownAdmission = reserveSample()
+ delete unknownAdmission.sources['director-admin']!.signals[
+ 'cell.production-gce-c1.admission_state'
+ ]
+ expect(evaluateIncidentSample(unknownAdmission, startedAt).failures).toContainEqual({
+ code: 'signal_missing',
+ source: 'director-admin',
+ signal: 'cell.production-gce-c1.admission_state'
+ })
+
+ const notReserve = reserveSample()
+ notReserve.cells[0] = { ...notReserve.cells[0]!, reserve: false }
+ expect(evaluateIncidentSample(notReserve, startedAt).failures).toContainEqual(
+ expect.objectContaining({
+ code: 'signal_missing',
+ signal: 'cell.production-gce-c1.heartbeat_fresh'
+ })
+ )
+ })
})
describe('incident monitor lifecycle', () => {
diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts
index 94c7a4791498..bcb3c30bf092 100644
--- a/cloud/apps/relay-ops/src/incident-monitor.ts
+++ b/cloud/apps/relay-ops/src/incident-monitor.ts
@@ -182,6 +182,8 @@ export type IncidentCellExpectation = {
runtimeKnown: boolean
powered: boolean
expectedAdmissionState: AdmissionState
+ // Step 5 reserve mode: the director's cell-status and migration reads refuse the cell.
+ reserve?: boolean
}
export type IncidentSample = {
@@ -558,6 +560,15 @@ function checkCell(
]
] as const
for (const [source, signals, signalName, threshold, comparison] of checks) {
+ // Probes, logs and selector still judge a reserve cell; the reserve watch reads its runtime.
+ if (
+ cell.reserve &&
+ ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked', 'migration_target_inactive'].some(
+ (suffix) => signalName.endsWith(suffix)
+ )
+ ) {
+ continue
+ }
if (
migrationPolicy === 'recover-forward' &&
cell.cellId === recoverySourceCellId &&
diff --git a/cloud/apps/relay/src/activity-lease-reader-census.test.ts b/cloud/apps/relay/src/activity-lease-reader-census.test.ts
new file mode 100644
index 000000000000..b8bc4aadc202
--- /dev/null
+++ b/cloud/apps/relay/src/activity-lease-reader-census.test.ts
@@ -0,0 +1,182 @@
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import ts from 'typescript'
+import { afterEach, describe, expect, it } from 'vitest'
+import { RelayAssignmentStore } from './assignment-store.js'
+import type { RelayCellConfig } from './config.js'
+import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
+
+// Step 6 (design §3.5): a phone on a `phonePath=memory` cell writes no `confirmation:` or
+// `splice:` lease. Every reader of the lease table is classified by why that is safe; a new
+// reader fails this test until someone decides. While the cell is recorded as reserve its
+// control holds no lease either, so step 5's guards already cover it; after a flip back a
+// memory phone looks like today's phone once its 90 s lease has expired.
+const ACTIVITY_LEASE_READERS: Record = {
+ // Reads control leases only, so phone leases never change the answer.
+ 'assignment-store.ts:activateControlOnce': 'control-only',
+ 'assignment-store.ts:reconcileIdleRegionalRehome': 'control-only',
+ 'assignment-store.ts:renewTransactionalControlActivity': 'control-only',
+ 'assignment-store.ts:startRegionalRehomeCandidate': 'control-only',
+ 'control-renewal-statement.ts:CONTROL_RENEWAL_BATCH_SQL': 'control-only',
+ 'idle-regional-rehome-selection.ts:selectIdleRegionalRehomes': 'control-only',
+ // Refuses or skips a cell recorded as reserve (or an unreadable set).
+ 'assignment-store.ts:cellDeploymentStatus': 'refuses-or-skips-reserve-cells',
+ 'assignment-store.ts:cellEvacuationCapacity': 'refuses-or-skips-reserve-cells',
+ 'assignment-store.ts:releaseExpiredActivity': 'refuses-or-skips-reserve-cells',
+ 'assignment-store.ts:releaseExpiredActivityLeases': 'refuses-or-skips-reserve-cells',
+ 'assignment-store.ts:startEvacuation': 'refuses-or-skips-reserve-cells',
+ // Its only caller drops candidates whose source or target is a reserve cell.
+ 'idle-regional-rehome-selection.ts:sourceRequestUnits': 'refuses-or-skips-reserve-cells',
+ // Only hosts in an open drain, migration or rehome, which no reserve cell can have:
+ // setCellAdmitMode refuses reserve while one is open, and every start refuses a reserve cell.
+ 'assignment-store.ts:activeCellMigrations': 'open-flow-only',
+ 'assignment-store.ts:additionalActivityCellIds': 'open-flow-only',
+ 'assignment-store.ts:assertCellReservationAccounting': 'open-flow-only',
+ 'assignment-store.ts:beginCellDrainSend': 'open-flow-only',
+ 'assignment-store.ts:beginCellDrainSendOnce': 'open-flow-only',
+ 'assignment-store.ts:completeReadyEvacuations': 'open-flow-only',
+ 'assignment-store.ts:completeReadyRegionalRehomes': 'open-flow-only',
+ 'assignment-store.ts:reconcileDrainMigrationAccounting': 'open-flow-only',
+ 'assignment-store.ts:reconcileReservationAccounting': 'open-flow-only',
+ 'assignment-store.ts:refreshDrainMigrationLeasesOnce': 'open-flow-only',
+ 'registered-migration-abandonment.ts:ABANDONED_REGISTERED_MIGRATION': 'open-flow-only',
+ // One host's rows under lock; each caller is classified in the reserve-mode census.
+ 'assignment-store.ts:lockAssignmentActivities': 'host-lock',
+ // Metrics and previews: they report, and act on nothing.
+ 'assignment-inventory-snapshot.ts:readAssignmentInventorySnapshot': 'report-only',
+ 'region-correction-preview.ts:previewRegionalRehomeEligibility': 'report-only',
+ 'registered-migration-inventory.ts:readRegisteredMigrationInventory': 'report-only',
+ // Step 5's existing-only stranded release, fixed separately in #30609 before the first flip.
+ 'assignment-store.ts:assignmentStrandedOnUnservedCell': 'step-5-stranded-release'
+}
+
+// A read is FROM, JOIN or USING on the table; `DELETE FROM` alone only writes. Not caught: a
+// second read inside an already listed function, and counters derived from leases elsewhere.
+const LEASE_TABLE = 'relay_assignment_activity_leases'
+// Bare table-name values that never reach SQL.
+const BARE_TABLE_NAME_USES: Record = {
+ 'database.ts:POSTGRES_TRANSACTION_PHASES': 'labels a failed statement by the table it names'
+}
+const LEASE_READ = /(? {
+ let database: RelayDatabase | undefined
+
+ afterEach(async () => {
+ await database?.close()
+ database = undefined
+ })
+
+ it('classifies every reader of the activity-lease table', () => {
+ const { readers, bareTableNames } = activityLeaseReaders()
+ // A bare name could reach `FROM ${table}`, which the scan cannot see: write the table into the SQL.
+ expect(bareTableNames.filter((use) => !(use in BARE_TABLE_NAME_USES))).toEqual([])
+ expect(readers.filter((name) => !(name in ACTIVITY_LEASE_READERS))).toEqual([])
+ expect(Object.keys(ACTIVITY_LEASE_READERS).filter((name) => !readers.includes(name))).toEqual([])
+ })
+
+ async function setup(now: () => number) {
+ database = await openInMemoryRelayDatabase()
+ const store = new RelayAssignmentStore(database, now)
+ await store.reconcileCells(CELLS)
+ return store
+ }
+
+ // A host whose source cell kept no lease for it, as a memory cell's control and phones do.
+ async function registeredEvacuationWithoutSourceLeases(store: RelayAssignmentStore) {
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ expect(first.cellId).toBe('cell-a')
+ const migration = await store.startEvacuation(identity, 'cell-b')
+ await store.activateControl(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch,
+ generation: 1
+ })
+ await store.markMigrationTargetRegistered(identity, {
+ cellId: 'cell-b',
+ assignmentEpoch: migration.assignmentEpoch
+ })
+ await store.releaseActivity(identity, `control-pending:${first.assignmentEpoch}`)
+ return { identity, migration }
+ }
+
+ it('completes no evacuation from a reserve source, though it holds no source lease', async () => {
+ const now = () => 1_000
+ const store = await setup(now)
+ const { identity, migration } = await registeredEvacuationWithoutSourceLeases(store)
+ // setCellAdmitMode refuses this with the migration open; written directly to reach the second guard.
+ await database!.query(
+ `INSERT INTO relay_cell_admit_modes (cell_id, admit_mode, updated_at) VALUES ('cell-a', 'reserve', 1000)`
+ )
+ const fresh = new RelayAssignmentStore(database!, now)
+ await expect(fresh.completeEvacuation(identity, migration.assignmentEpoch)).rejects.toThrow(
+ 'cell_in_reserve_mode'
+ )
+ await expect(fresh.completeReadyEvacuations()).rejects.toThrow('cell_in_reserve_mode')
+ expect(
+ await database!.query(`SELECT completed_at FROM relay_assignment_migrations`)
+ ).toEqual([{ completed_at: null }])
+ // The same host on a db source completes: the guard, not the fixture, held it.
+ await store.setCellAdmitMode('cell-a', 'db')
+ expect(await store.completeReadyEvacuations()).toBe(1)
+ })
+
+ it('refuses evacuation status for a reserve cell, where activeCellMigrations would count no units', async () => {
+ const store = await setup(() => 1_000)
+ await store.setCellAdmitMode('cell-a', 'reserve')
+ await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow(
+ 'cell_in_reserve_mode'
+ )
+ await expect(store.cellEvacuationStatus('cell-b', 'cell-a', false)).rejects.toThrow(
+ 'cell_in_reserve_mode'
+ )
+ })
+})
+
+function activityLeaseReaders(): { readers: string[]; bareTableNames: string[] } {
+ const directory = fileURLToPath(new URL('.', import.meta.url))
+ const readers = new Set()
+ const bareTableNames: string[] = []
+ const files = readdirSync(directory, { recursive: true, encoding: 'utf8' }).filter(
+ (file) => file.endsWith('.ts') && !file.includes('.test.') && !file.split(/[\\/]/).includes('test-fixtures')
+ )
+ for (const file of files) {
+ const text = readFileSync(join(directory, file), 'utf8')
+ if (!text.includes(LEASE_TABLE)) continue
+ const source = ts.createSourceFile(file, text, ts.ScriptTarget.ES2022, true)
+ const visit = (node: ts.Node): void => {
+ if (ts.isNoSubstitutionTemplateLiteral(node) || ts.isTemplateExpression(node) || ts.isStringLiteral(node)) {
+ if ((ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) && node.text.trim() === LEASE_TABLE) {
+ bareTableNames.push(`${file}:${enclosingName(node)}`)
+ }
+ if (LEASE_READ.test(node.getText())) {
+ readers.add(`${file}:${enclosingName(node)}`)
+ return
+ }
+ }
+ ts.forEachChild(node, visit)
+ }
+ visit(source)
+ }
+ return { readers: [...readers].sort(), bareTableNames }
+}
+
+function enclosingName(node: ts.Node): string {
+ for (let current = node.parent; current; current = current.parent) {
+ if ((ts.isMethodDeclaration(current) || ts.isFunctionDeclaration(current)) && current.name) {
+ return current.name.getText()
+ }
+ // A module-level SQL constant, never a local `const rows = ...` inside a method.
+ if (ts.isVariableStatement(current) && ts.isSourceFile(current.parent)) {
+ return current.declarationList.declarations.map((declaration) => declaration.name.getText()).join(',')
+ }
+ }
+ return ''
+}
diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts
index a9d6c370c168..694bf72f113a 100644
--- a/cloud/apps/relay/src/app.ts
+++ b/cloud/apps/relay/src/app.ts
@@ -31,6 +31,7 @@ import {
RelayAssignmentRowBusyError,
RelayCellInReserveModeError,
RelayHomeCellUnavailableError,
+ type AssignOptions,
type CellFenceAttemptEvidence,
type RelayAssignment,
type RelayAssignmentStore,
@@ -143,6 +144,7 @@ export function createRelayApp(
cellReserverPoll?: () => void
cellAdmitModeEffective?: () => 'db' | 'reserve'
cellAdmitModeRaw?: () => 'db' | 'reserve'
+ cellPhonePathEffective?: () => 'db' | 'memory'
cellReserveCounts?: () => {
bookings: number
intake: { perSec: number; burst: number; tokens: number }
@@ -459,13 +461,14 @@ export function createRelayApp(
.setExpirationTime('5m')
.sign(config.assignmentSigningKey)
// Step 5: switched-on cells answer from the map; anything else keeps today's path below.
- let step5: { epochFloor?: number; placeFresh?: () => Promise } = {}
- if (operations.reservePlacement && !body.data.regionCorrection) {
+ let step5: Pick = {}
+ if (operations.reservePlacement && body.data.regionCorrection?.action !== 'report') {
let plan: ReservePlan
try {
plan = await operations.reservePlacement.plan(identity, {
reconnect: Boolean(body.data.reconnect),
- region: targetRegion
+ region: targetRegion,
+ regionCorrection: body.data.regionCorrection !== undefined
})
} catch (error) {
if (!(error instanceof ReservePaceError)) throw error
@@ -491,7 +494,7 @@ export function createRelayApp(
context.header('Retry-After', String(plan.retryAfterSeconds))
return context.json({ error: 'assignment_temporarily_unavailable' }, 503)
}
- step5 = { epochFloor: plan.epochFloor, placeFresh: plan.placeFresh }
+ step5 = { epochFloor: plan.epochFloor, placeFresh: plan.placeFresh, regionReserve: plan.regionReserve }
}
let admission: { release(): void } | null = null
let lane: AssignmentAdmissionLane = 'placement'
@@ -891,6 +894,9 @@ export function createRelayApp(
...(operations.cellAdmitModeEffective
? { admitModeEffective: operations.cellAdmitModeEffective() }
: {}),
+ ...(operations.cellPhonePathEffective
+ ? { phonePathEffective: operations.cellPhonePathEffective() }
+ : {}),
// Placement as built, not as configured: `on` only when this director books reserve cells.
...(config.role === 'director'
? { reservePlacement: operations.reservePlacement?.placementMode ?? 'off' }
diff --git a/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts b/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts
index 460a0afcf887..306c25043d93 100644
--- a/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts
+++ b/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts
@@ -1,6 +1,7 @@
import { createHash } from 'node:crypto'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js'
+import { ReservePaceError } from './reserve-assignment.js'
import {
encodeMembership,
type CellAdmissionMembership,
@@ -564,6 +565,100 @@ describe('re-placing a host off a cell isolated for a roll', () => {
expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
})
+ describe('with a reserve cell in its own region', () => {
+ const onReserve = {
+ ...IDENTITY,
+ cellId: 'asia-reserve',
+ cellUrl: 'https://asia-reserve.example.com',
+ assignmentEpoch: 9,
+ leaseExpiresAt: START_MS + 1_000_000,
+ region: 'asia-east2' as const
+ }
+
+ it('books the reserve cell before leaving the region, and the canary still counts the move', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const { store, database, isolateForRoll } = await setup()
+ const first = await store.assign(IDENTITY, 'asia-east2')
+ await isolateForRoll(first.cellId)
+ warn.mockClear()
+ const regionReserve = vi.fn(async () => onReserve)
+ await expect(
+ store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
+ ).resolves.toEqual(onReserve)
+ expect(regionReserve).toHaveBeenCalledOnce()
+ expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
+ {
+ event: 'orca_relay_sticky_replaced_off_isolated_cell',
+ fromCellId: first.cellId,
+ fromRegion: 'asia-east2',
+ admissionState: 'migration-only',
+ toCellId: 'asia-reserve',
+ region: 'asia-east2',
+ via: 'reserve'
+ }
+ ])
+ // The refused in-region attempt wrote nothing.
+ expect(
+ await database.query(`SELECT cell_id, assignment_epoch FROM relay_assignments`)
+ ).toEqual([{ cell_id: first.cellId, assignment_epoch: first.assignmentEpoch }])
+ })
+
+ it('leaves the region as before when the reserve cell has no room either', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const { store, isolateForRoll } = await setup()
+ const first = await store.assign(IDENTITY, 'asia-east2')
+ await isolateForRoll(first.cellId)
+ warn.mockClear()
+ const regionReserve = vi.fn(async () => null)
+ const moved = await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
+ expect(moved).toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
+ expect(regionReserve).toHaveBeenCalledOnce()
+ expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
+ { fromCellId: first.cellId, region: 'us-central1' }
+ ])
+ })
+
+ it('treats a throttled reserve cell as no room, so the drain does not wait on its intake', async () => {
+ const { store, isolateForRoll } = await setup()
+ const first = await store.assign(IDENTITY, 'asia-east2')
+ await isolateForRoll(first.cellId)
+ const regionReserve = vi.fn(async () => {
+ throw new ReservePaceError(4)
+ })
+ await expect(
+ store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
+ ).resolves.toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
+ expect(regionReserve).toHaveBeenCalledOnce()
+ })
+
+ it('counts a share-draw booking off the isolated cell for the canary', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const { store, isolateForRoll } = await setup()
+ const first = await store.assign(IDENTITY, 'asia-east2')
+ await isolateForRoll(first.cellId)
+ warn.mockClear()
+ await expect(
+ store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { placeFresh: async () => onReserve })
+ ).resolves.toEqual(onReserve)
+ expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
+ { fromCellId: first.cellId, toCellId: 'asia-reserve', via: 'reserve' }
+ ])
+ })
+
+ it('does not hold a host whose preferred region is elsewhere', async () => {
+ const { store, isolateForRoll, park } = await setup()
+ const first = await store.assign(IDENTITY, 'us-central1')
+ const other = CELLS.find((cell) => cell.region === 'us-central1' && cell.id !== first.cellId)!
+ await park(other.id, 'migration-only')
+ await isolateForRoll(first.cellId)
+ const regionReserve = vi.fn(async () => onReserve)
+ await expect(
+ store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
+ ).resolves.toMatchObject({ cellId: 'asia-c1' })
+ expect(regionReserve).not.toHaveBeenCalled()
+ })
+ })
+
it('takes the dead-cell path when a stamped cell stops heartbeating', async () => {
// Why: the fence is the only proof that a cell we cannot reach has stopped
// serving the sockets it still holds. A stamp does not make it reachable.
diff --git a/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts b/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts
index a10b49940132..077552c8f03c 100644
--- a/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts
+++ b/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts
@@ -228,6 +228,27 @@ describe('reserve-mode census', () => {
await restore()
})
+ it('keeps a connected host pinned when its reserve cell goes existing-only after a ledger write', async () => {
+ let now = 1_000
+ const store = await setup(() => now)
+ const identity = { userId: 'user-a', relayHostId: 'host000000000001' }
+ const first = await store.assign(identity)
+ await setReserve(store, [first.cellId])
+ // The reserve ledger stamps last_activity_at and the cell keeps no activity rows.
+ now += 5 * 60_000
+ await database!.query('DELETE FROM relay_assignment_activity_leases WHERE user_id = ?', [identity.userId])
+ await database!.query('UPDATE relay_assignments SET last_activity_at = ? WHERE user_id = ?', [now, identity.userId])
+ await store.setCellAdmissionState(first.cellId, 'existing-only')
+ now += 2 * 60_000
+ expect(await store.assign(identity)).toMatchObject({
+ cellId: first.cellId,
+ assignmentEpoch: first.assignmentEpoch
+ })
+ const { unknown, restore } = await unknownSet(() => now)
+ expect((await unknown.assign(identity)).assignmentEpoch).toBe(first.assignmentEpoch)
+ await restore()
+ })
+
// Only the first reserve-set read is held; it is released (or failed) by the test.
function holdFirstRead(options: { fail?: boolean } = {}) {
const query = database!.query.bind(database!)
diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts
index 66d90177bdac..38ce3e919a25 100644
--- a/cloud/apps/relay/src/assignment-store.ts
+++ b/cloud/apps/relay/src/assignment-store.ts
@@ -12,6 +12,7 @@ import {
type RegionalRehomePollGate
} from './regional-rehome-poll-telemetry.js'
import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js'
+import { ReservePaceError } from './reserve-assignment.js'
import {
previewRegionalRehomeEligibility,
type RegionCorrectionPreview
@@ -145,6 +146,8 @@ type RelayAssignmentStoreOptions = {
export type AssignOptions = {
epochFloor?: number
placeFresh?: () => Promise
+ // A reserve cell in the placement region with room: the database lane leaves the region only after it.
+ regionReserve?: () => Promise
hostRequested?: boolean
}
@@ -523,7 +526,7 @@ class AssignmentInventoryLockUnavailable extends Error {
// Postgres holds row locks to COMMIT, so the next tier needs a new transaction.
class AssignmentIsolatedTierExhausted extends Error {
- constructor() {
+ constructor(readonly source: { cellId: string; region: RelayRegion }) {
super('assignment_isolated_tier_exhausted')
}
}
@@ -534,6 +537,17 @@ class AssignmentInventoryScopeChanged extends Error {
}
}
+// Sticky's "re-place me": the host's pin is on a cell isolated for a roll.
+type IsolatedPin = { isolatedFrom: { cellId: string; region: RelayRegion } }
+
+// The placement region has no database cell with room; nothing was written.
+class AssignmentRegionFull extends Error {
+ // Set when the host is leaving a roll-isolated cell in that region.
+ constructor(readonly isolatedSource?: { cellId: string; region: RelayRegion }) {
+ super('assignment_region_full')
+ }
+}
+
// Why a reason of its own: a host whose home cell is fenced-but-unattested is
// refused regardless of fleet headroom, so reporting it as capacity sends
// operators after capacity that was never short. Every cell boot and every
@@ -1003,30 +1017,67 @@ export class RelayAssignmentStore {
step5: AssignOptions = {}
): Promise {
const sticky = await this.assignStickyWithLockRetry(identity, lockMode, preferredRegion)
- if (sticky) return sticky
- const booked = await step5.placeFresh?.()
+ if (sticky && !('isolatedFrom' in sticky)) return sticky
+ let isolatedFrom = sticky?.isolatedFrom
+ // Off a roll-isolated cell: the canary counts the move whichever pool takes it, and a
+ // throttled reserve cell is no room, so the drain does not wait on reserve intake.
+ const bookReserve = async (book: () => Promise) => {
+ let reserved: RelayAssignment | null
+ try {
+ reserved = await book()
+ } catch (error) {
+ if (!(isolatedFrom && error instanceof ReservePaceError)) throw error
+ reserved = null
+ }
+ if (reserved && isolatedFrom) {
+ console.warn(
+ JSON.stringify({
+ event: 'orca_relay_sticky_replaced_off_isolated_cell',
+ fromCellId: isolatedFrom.cellId,
+ fromRegion: isolatedFrom.region,
+ admissionState: ROLL_ISOLATED_ADMISSION,
+ toCellId: reserved.cellId,
+ region: reserved.region,
+ via: 'reserve'
+ })
+ )
+ }
+ return reserved
+ }
+ const booked = step5.placeFresh ? await bookReserve(step5.placeFresh) : null
if (booked) return booked
// Only placement needs the global inventory critical section; queueing those
// attempts locally avoids turning true placement bursts into NOWAIT storms.
- return await this.serializeAssignment(
- async () =>
- await this.assignWithLockRetry(
- identity,
- lockMode,
- preferredRegion,
- placementRegion,
- step5.epochFloor ?? 0,
- // Only through the step-5 plan: its floor holds every epoch the map knows for the host.
- step5.hostRequested === true && step5.epochFloor !== undefined
- )
- )
+ const placeOnDatabase = async (keepRegion: boolean) =>
+ await this.serializeAssignment(
+ async () =>
+ await this.assignWithLockRetry(
+ identity,
+ lockMode,
+ preferredRegion,
+ placementRegion,
+ step5.epochFloor ?? 0,
+ // Only through the step-5 plan: its floor holds every epoch the map knows for the host.
+ step5.hostRequested === true && step5.epochFloor !== undefined,
+ keepRegion
+ )
+ )
+ const regionReserve = step5.regionReserve
+ if (!regionReserve) return await placeOnDatabase(false)
+ try {
+ return await placeOnDatabase(true)
+ } catch (error) {
+ if (!(error instanceof AssignmentRegionFull)) throw error
+ isolatedFrom ??= error.isolatedSource
+ }
+ return (await bookReserve(regionReserve)) ?? (await placeOnDatabase(false))
}
private async assignStickyWithLockRetry(
identity: AssignmentIdentity,
lockMode: CellInventoryLockMode,
preferredRegion?: RelayRegion
- ): Promise {
+ ): Promise {
return await this.withAssignmentLockRetry(
async (inventoryFirst) =>
await this.assignStickyOnce(identity, inventoryFirst, lockMode, preferredRegion)
@@ -1039,7 +1090,8 @@ export class RelayAssignmentStore {
preferredRegion?: RelayRegion,
placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION,
epochFloor = 0,
- hostRequested = false
+ hostRequested = false,
+ keepRegion = false
): Promise {
const deadline = Date.now() + ASSIGNMENT_LOCK_RETRY_DEADLINE_MS
let inventoryScope: AssignmentInventoryScope = 'none'
@@ -1052,7 +1104,8 @@ export class RelayAssignmentStore {
preferredRegion,
placementRegion,
epochFloor,
- hostRequested
+ hostRequested,
+ keepRegion
)
} catch (error) {
if (error instanceof AssignmentInventoryScopeChanged) {
@@ -1060,6 +1113,10 @@ export class RelayAssignmentStore {
continue
}
if (error instanceof AssignmentIsolatedTierExhausted) {
+ // An in-region reserve cell comes before the other regions.
+ if (keepRegion && error.source.region === placementRegion) {
+ throw new AssignmentRegionFull(error.source)
+ }
inventoryScope = 'isolated-other-regions'
continue
}
@@ -1097,7 +1154,7 @@ export class RelayAssignmentStore {
inventoryFirst: boolean,
lockMode: CellInventoryLockMode,
preferredRegion?: RelayRegion
- ): Promise {
+ ): Promise {
const now = this.now()
await this.reserveModeCells()
return await this.database.transaction(async (transaction) => {
@@ -1177,7 +1234,8 @@ export class RelayAssignmentStore {
pinnedAdmission
)
) {
- return null
+ const cellId = text(existing, 'cell_id')
+ return { isolatedFrom: { cellId, region: await this.cellRegion(transaction, cellId) } }
}
const currentCellId = text(existing, 'cell_id')
@@ -1290,6 +1348,11 @@ export class RelayAssignmentStore {
if (state !== 'existing-only') {
return false
}
+ // A reserve cell keeps no activity rows and its ledger stamps last_activity_at, so zero
+ // leases proves nothing; a dead one is left to the dead-reserve path and its fence.
+ if (await this.mayBeReserveCell(text(existing, 'cell_id'), transaction)) {
+ return false
+ }
const liveLeases = (
await transaction.query(
`SELECT COUNT(*) AS live FROM relay_assignment_activity_leases
@@ -1320,7 +1383,9 @@ export class RelayAssignmentStore {
preferredRegion?: RelayRegion,
placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION,
epochFloor = 0,
- hostRequested = false
+ hostRequested = false,
+ // Refuse to spill out of placementRegion: AssignmentRegionFull instead.
+ keepRegion = false
): Promise {
const now = this.now()
// An isolated retry takes its tier's rows before anything reassigns this,
@@ -1414,7 +1479,7 @@ export class RelayAssignmentStore {
// Same region full: try the other regions. All full: decide under the
// all-rows lock, which keeps the pin as before.
if (!isolatedTarget) {
- if (isolatedTier === 'same-region') throw new AssignmentIsolatedTierExhausted()
+ if (isolatedTier === 'same-region') throw new AssignmentIsolatedTierExhausted(source)
throw new AssignmentInventoryScopeChanged()
}
isolatedIncumbent = source
@@ -1582,8 +1647,15 @@ export class RelayAssignmentStore {
// cross-region spill it exists to refuse.
const target =
isolatedTarget ??
- (await this.leastLoadedCell(transaction, lockedCells, placementRegion))
- if (!target) throw new Error('relay_capacity_exhausted')
+ (await this.leastLoadedCell(
+ transaction,
+ lockedCells,
+ placementRegion,
+ keepRegion ? 'require' : 'prefer'
+ ))
+ if (!target) {
+ throw keepRegion ? new AssignmentRegionFull() : new Error('relay_capacity_exhausted')
+ }
if (deadReserveSource) {
events.push(
JSON.stringify({
@@ -8132,15 +8204,18 @@ export class RelayAssignmentStore {
mode: CellInventoryLockMode
): Promise {
if (sourceCellId === undefined) throw new AssignmentInventoryScopeChanged()
+ const sourceRegion = await this.cellRegion(database, sourceCellId)
const candidates = await database.query(
`SELECT admission.cell_id FROM relay_cell_admission admission
LEFT JOIN relay_cell_regions region ON region.cell_id = admission.cell_id
WHERE admission.admission_state = 'general' AND admission.cell_id <> ?
AND COALESCE(region.region, ?) ${tier === 'same-region' ? '=' : '<>'} ?`,
- [sourceCellId, RELAY_DEFAULT_REGION, await this.cellRegion(database, sourceCellId)]
+ [sourceCellId, RELAY_DEFAULT_REGION, sourceRegion]
)
if (candidates.length === 0) {
- if (tier === 'same-region') throw new AssignmentIsolatedTierExhausted()
+ if (tier === 'same-region') {
+ throw new AssignmentIsolatedTierExhausted({ cellId: sourceCellId, region: sourceRegion })
+ }
throw new AssignmentInventoryScopeChanged()
}
return await this.lockCellRows(
diff --git a/cloud/apps/relay/src/cell-flags.ts b/cloud/apps/relay/src/cell-flags.ts
index ccf22204d591..a3c3e08bcc8f 100644
--- a/cloud/apps/relay/src/cell-flags.ts
+++ b/cloud/apps/relay/src/cell-flags.ts
@@ -19,6 +19,8 @@ export type CellFlags = {
reserveDryRun: boolean
// Flip-back leases in flight at once; absent is a third of the pool. Speeds up one cell.
reregisterInFlight?: number
+ // Step 6: `memory` resumes paired phones from one device read; it acts only in reserve mode.
+ phonePath: 'db' | 'memory'
}
export const CELL_FLAG_DEFAULTS: CellFlags = {
@@ -26,7 +28,8 @@ export const CELL_FLAG_DEFAULTS: CellFlags = {
ticketCheck: 'off',
rejectionFence: true,
admitMode: 'db',
- reserveDryRun: false
+ reserveDryRun: false,
+ phonePath: 'db'
}
export const READ_TIMEOUT_MARGIN_MIN_MS = 1_000
@@ -52,7 +55,8 @@ const CellFlagsSchema = z.object({
admitMode: z.enum(['db', 'reserve']).default(CELL_FLAG_DEFAULTS.admitMode),
intakePerSec: z.number().positive().max(1_000).optional(),
reserveDryRun: z.boolean().default(CELL_FLAG_DEFAULTS.reserveDryRun),
- reregisterInFlight: z.number().int().min(1).max(16).optional()
+ reregisterInFlight: z.number().int().min(1).max(16).optional(),
+ phonePath: z.enum(['db', 'memory']).default(CELL_FLAG_DEFAULTS.phonePath)
})
const KNOWN_FLAG_KEYS = new Set(Object.keys(CellFlagsSchema.shape))
diff --git a/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts
new file mode 100644
index 000000000000..fb6fb7abe140
--- /dev/null
+++ b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts
@@ -0,0 +1,530 @@
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { reserveDeadManWindowMs } from './cell-reserve-dead-man.js'
+import { openRelayDatabase } from './database.js'
+import { nextFrame, RESERVE_MODE_CELL_ID, startReserveModeCell } from './test-fixtures/reserve-mode-cell.js'
+import { StatementRecordingDatabase } from './test-fixtures/statement-recording-database.js'
+
+// Step 6: a reserve-mode cell with phonePath=memory resumes paired phones without the database.
+
+// Phone credentials are 32 random bytes.
+const token = (fill: number) => Buffer.alloc(32, fill).toString('base64url')
+const TOKEN = token(1)
+const OTHER_CELL = { id: 'production-gce-c4', url: 'https://c4.relay.onorca.dev', capacityRequests: 900 }
+
+// A paired phone's accept and attach with the switch off, as main issued them at 9ea54343710.
+const DATABASE_PATH_ACCEPT = [
+ 'SELECT * FROM relay_devices WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)',
+ 'SELECT assignment.*, cell.cell_url, region.region FROM relay_assignments assignment JOIN relay_cells cell ON cell.cell_id = assignment.cell_id LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id WHERE assignment.user_id = ? AND assignment.relay_host_id = ?',
+ 'BEGIN',
+ 'SELECT * FROM relay_invites WHERE relay_host_id = ? AND token_hash = ?',
+ 'SELECT * FROM relay_devices WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)',
+ 'COMMIT',
+ 'BEGIN',
+ 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ? [locked]',
+ 'SELECT * FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC [locked]',
+ 'UPDATE relay_assignments SET pending_confirmations = CASE WHEN pending_confirmations + ? < 0 THEN 0 ELSE pending_confirmations + ? END, lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?',
+ 'INSERT INTO relay_assignment_activity_leases (user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units, expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+ 'UPDATE relay_cells SET reserved_requests = CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, updated_at = ? WHERE cell_id = ? AND (? <= 0 OR reserved_requests + ? <= capacity_requests) RETURNING cell_id',
+ 'COMMIT',
+ 'BEGIN',
+ 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ? [locked]',
+ 'SELECT * FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC [locked]',
+ 'UPDATE relay_assignments SET reserved_splices = CASE WHEN reserved_splices + ? < 0 THEN 0 ELSE reserved_splices + ? END, lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?',
+ 'INSERT INTO relay_assignment_activity_leases (user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units, expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+ 'UPDATE relay_cells SET reserved_requests = CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, updated_at = ? WHERE cell_id = ? AND (? <= 0 OR reserved_requests + ? <= capacity_requests) RETURNING cell_id',
+ 'COMMIT',
+ 'INSERT INTO relay_connection_bases (basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation, credential_kind, invite_token_hash, accepted_credential_version, accepted_as, deadline, active, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
+]
+
+const CONFIRM_DEVICE_READ =
+ 'SELECT * FROM relay_devices WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?'
+
+// The flag-off confirm, as main issued it at 9ea54343710.
+const DATABASE_PATH_CONFIRM = [
+ 'BEGIN',
+ 'SELECT basis_conn_id, result_json FROM relay_confirm_results WHERE user_id = ? AND relay_host_id = ? AND req_id = ?',
+ 'SELECT * FROM relay_connection_bases WHERE basis_conn_id = ?',
+ 'UPDATE relay_devices SET updated_at = updated_at WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?',
+ 'SELECT * FROM relay_devices WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?',
+ 'UPDATE relay_devices SET current_expires_at = ?, updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?',
+ 'INSERT INTO relay_confirm_results (user_id, relay_host_id, req_id, basis_conn_id, tuple_json, result_json, committed_at) VALUES (?, ?, ?, ?, ?, ?, ?)',
+ 'INSERT INTO relay_audit_events (id, at, type, user_id, relay_host_id, relay_device_id, detail_json) VALUES (?, ?, ?, ?, ?, ?, ?)',
+ 'COMMIT',
+ 'BEGIN',
+ 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ? [locked]',
+ 'SELECT * FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC [locked]',
+ 'DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?',
+ 'UPDATE relay_assignments SET pending_confirmations = CASE WHEN pending_confirmations + ? < 0 THEN 0 ELSE pending_confirmations + ? END, lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?',
+ 'UPDATE relay_cells SET reserved_requests = CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, updated_at = ? WHERE cell_id = ? AND (? <= 0 OR reserved_requests + ? <= capacity_requests) RETURNING cell_id',
+ 'COMMIT'
+]
+
+async function until(check: () => Promise, timeoutMs = 10_000): Promise {
+ const deadline = Date.now() + timeoutMs
+ while (!(await check())) {
+ if (Date.now() > deadline) throw new Error('condition not reached')
+ await new Promise((resolve) => setTimeout(resolve, 50))
+ }
+}
+
+describe('the memory phone path', () => {
+ const cleanup: Array<() => Promise | void> = []
+
+ afterEach(async () => {
+ for (const close of cleanup.splice(0).reverse()) await close()
+ vi.restoreAllMocks()
+ })
+
+ async function cell(now?: () => number) {
+ vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-phone-path-'))
+ cleanup.push(() => rmSync(dataDir, { recursive: true, force: true }))
+ const inner = await openRelayDatabase({ dataDir })
+ cleanup.push(() => inner.close())
+ const database = new StatementRecordingDatabase(inner)
+ const reserve = await startReserveModeCell({ database, dataDir, cleanup, ...(now ? { now } : {}) })
+ // The admit-effective writer ticks on its own; every other statement is the phone's.
+ const phoneStatements = () =>
+ database.statements.filter((sql) => !sql.includes('relay_cell_admit_effective'))
+ // The test's own reads and writes go around the recorder.
+ return { ...reserve, database, raw: inner, phoneStatements }
+ }
+
+ async function seatedPhoneHost(phonePath: 'db' | 'memory') {
+ const reserve = await cell()
+ reserve.setAdmitMode('reserve', { phonePath })
+ const seated = await reserve.bookedHost(3)
+ await reserve.installDevice(seated.identity.hostId, TOKEN)
+ reserve.database.statements.length = 0
+ return { reserve, seated }
+ }
+
+ // A phone resumed and spliced to its desktop, waiting to confirm.
+ async function splicedPhone(
+ phonePath: 'db' | 'memory',
+ device: { expiresAt?: number } = {},
+ clock: { now?: () => number; beforeDial?: () => void } = {}
+ ) {
+ const reserve = await cell(clock.now)
+ reserve.setAdmitMode('reserve', { phonePath })
+ const seated = await reserve.bookedHost(3)
+ await reserve.installDevice(seated.identity.hostId, TOKEN, device)
+ clock.beforeDial?.()
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
+ const basisConnId = String((await connOpen).connId)
+ await reserve.attachData(await connOpen, seated.ack.generation)
+ expect(await phone.hello).toMatchObject({ ok: true })
+ reserve.database.statements.length = 0
+ // The desktop's confirm, sent in reply to the phone over the end-to-end channel.
+ const confirm = async (reqId: string, basis = basisConnId) => {
+ const answer = Promise.race([
+ nextFrame(seated.socket, 'device-resume-confirmed'),
+ nextFrame(seated.socket, 'control-error')
+ ])
+ seated.socket.send(JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId: basis }))
+ return await answer
+ }
+ return { reserve, seated, phone, basisConnId, confirm }
+ }
+
+ const deviceRow = async (reserve: Awaited>) =>
+ (await reserve.raw.query(`SELECT current_expires_at FROM relay_devices`))[0]
+
+ const warnings = (event: string) =>
+ vi
+ .mocked(console.warn)
+ .mock.calls.map(([line]) => String(line))
+ .filter((line) => line.includes(event))
+ .map((line) => JSON.parse(line))
+
+ it('acts only while the cell is in reserve mode, so admitMode=db or a dead-man trip turns it off', async () => {
+ let now = Date.now()
+ const reserve = await cell(() => now)
+ expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
+ reserve.setAdmitMode('reserve', { phonePath: 'memory' })
+ expect(reserve.relay.sessions.phonePathEffective()).toBe('memory')
+ reserve.setAdmitMode('db', { phonePath: 'memory' })
+ expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
+ reserve.setAdmitMode('reserve', { phonePath: 'memory' })
+ expect(reserve.relay.sessions.phonePathEffective()).toBe('memory')
+ now += reserveDeadManWindowMs(reserve.config.cellId) + 1_000
+ expect(reserve.relay.sessions.inReserveMode()).toBe(false)
+ expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
+ })
+
+ it('admits a seated phone resume from one device read and writes no activity lease', async () => {
+ const { reserve, seated } = await seatedPhoneHost('memory')
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
+ expect(await connOpen).toMatchObject({ kind: 'resume', relayDeviceId: 'device-1' })
+ expect(reserve.phoneStatements()).toEqual([DATABASE_PATH_ACCEPT[0]])
+ await reserve.attachData(await connOpen, seated.ack.generation)
+ expect(await phone.hello).toMatchObject({ type: 'relay-hello', ok: true, credentialKind: 'resume' })
+ expect(reserve.phoneStatements()).toEqual([DATABASE_PATH_ACCEPT[0]])
+ expect(await reserve.activityLeases()).toEqual([])
+ })
+
+ it('keeps the database path statement for statement while phonePath is db', async () => {
+ const { reserve, seated } = await seatedPhoneHost('db')
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
+ await reserve.attachData(await connOpen, seated.ack.generation)
+ expect(await phone.hello).toMatchObject({ type: 'relay-hello', ok: true })
+ expect(reserve.phoneStatements()).toEqual(DATABASE_PATH_ACCEPT)
+ })
+
+ it('closes a revoked or expired device 4401 and never tries it as an invite', async () => {
+ const reserve = await cell()
+ reserve.setAdmitMode('reserve', { phonePath: 'memory' })
+ const seated = await reserve.bookedHost(3)
+ await reserve.installDevice(seated.identity.hostId, token(2), { revokedAt: Date.now() - 1_000 })
+ await reserve.installDevice(seated.identity.hostId, token(3), {
+ relayDeviceId: 'device-2',
+ expiresAt: Date.now() - 1_000
+ })
+ for (const refused of [token(2), token(3)]) {
+ reserve.database.statements.length = 0
+ const phone = await reserve.dialPhone(seated.identity.hostId, refused)
+ expect(await phone.hello).toEqual({ type: 'relay-hello', ok: false, code: 4401 })
+ expect((await phone.closed).code).toBe(4401)
+ expect(reserve.phoneStatements()).toEqual([DATABASE_PATH_ACCEPT[0]])
+ }
+ })
+
+ it('closes 4503 when the device read fails, without trying the invite path', async () => {
+ const { reserve, seated } = await seatedPhoneHost('memory')
+ vi.spyOn(reserve.relay.store, 'lookupResume').mockRejectedValue(new Error('connection terminated'))
+ const invite = vi.spyOn(reserve.relay.store, 'resolveInviteForMove')
+ for (let dial = 0; dial < 2; dial += 1) {
+ const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN)
+ expect((await phone.closed).code).toBe(4503)
+ }
+ expect(invite).not.toHaveBeenCalled()
+ expect(reserve.phoneStatements()).toEqual([])
+ // One line opens the window; the repeat waits for its closing line.
+ expect(warnings('orca_relay_client_accept_db_error')).toEqual([
+ { event: 'orca_relay_client_accept_db_error', cause: 'error', cellId: reserve.config.cellId }
+ ])
+ })
+
+ it('falls through to the invite path only when no device row matches', async () => {
+ const { reserve, seated } = await seatedPhoneHost('memory')
+ const created = nextFrame(seated.socket, 'invite-created')
+ seated.socket.send(JSON.stringify({ type: 'invite-create', reqId: 'invite-1', relayDeviceId: 'device-9' }))
+ const invite = await created
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ await reserve.dialPhone(seated.identity.hostId, String(invite.inviteToken))
+ expect(await connOpen).toMatchObject({ kind: 'invite', relayDeviceId: 'device-9' })
+ // Invites keep today's leases: install validates their basis in Postgres.
+ expect((await reserve.activityLeases()).map((lease) => lease.activity_kind)).toEqual(['invite', 'invite'])
+ })
+
+ it('answers a phone whose host has no seat here from one assignment read', async () => {
+ const reserve = await cell()
+ reserve.setAdmitMode('reserve', { phonePath: 'memory' })
+ await reserve.relay.assignments.reconcileCells([...reserve.config.cells, OTHER_CELL])
+ const dial = async (row: 'elsewhere' | 'here' | 'none') => {
+ const host = await reserve.host()
+ if (row !== 'none') await reserve.insertRow(host.hostId, 3, row === 'here' ? reserve.config.cellId : OTHER_CELL.id)
+ await reserve.installDevice(host.hostId, TOKEN)
+ reserve.database.statements.length = 0
+ const phone = await reserve.dialPhone(host.hostId, TOKEN)
+ const closed = await phone.closed
+ expect(reserve.phoneStatements()).toEqual(DATABASE_PATH_ACCEPT.slice(0, 2))
+ return closed.code
+ }
+ expect(await dial('elsewhere')).toBe(4409)
+ expect(await dial('none')).toBe(4404)
+ expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([])
+ expect(await dial('here')).toBe(4404)
+ expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([
+ expect.objectContaining({ cellId: reserve.config.cellId, relayHostIdDigest: expect.any(String) })
+ ])
+ vi.spyOn(reserve.relay.assignments, 'resolve').mockRejectedValueOnce(new Error('connection terminated'))
+ const host = await reserve.host()
+ await reserve.installDevice(host.hostId, TOKEN)
+ expect((await (await reserve.dialPhone(host.hostId, TOKEN)).closed).code).toBe(4409)
+ expect(warnings('orca_relay_client_miss_read_error')).toEqual([
+ { event: 'orca_relay_client_miss_read_error', cause: 'error', cellId: reserve.config.cellId }
+ ])
+ })
+
+ it('takes the database path for a phone whose switch turned off during its device read', async () => {
+ const { reserve, seated } = await seatedPhoneHost('memory')
+ const lookup = reserve.relay.store.lookupResume.bind(reserve.relay.store)
+ vi.spyOn(reserve.relay.store, 'lookupResume').mockImplementation(async (...args) => {
+ const found = await lookup(...args)
+ reserve.setAdmitMode('reserve', { phonePath: 'db' })
+ return found
+ })
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ await reserve.dialPhone(seated.identity.hostId, TOKEN)
+ expect(await connOpen).toMatchObject({ kind: 'resume' })
+ expect((await reserve.activityLeases()).map((lease) => lease.activity_kind)).toEqual(['confirmation'])
+ })
+
+ it('confirms from the memory basis with one fresh read, renewing a device due for it', async () => {
+ const { reserve, confirm } = await splicedPhone('memory')
+ const before = Number((await deviceRow(reserve))?.current_expires_at)
+ const answer = await confirm('confirm-1')
+ expect(answer).toMatchObject({ type: 'device-resume-confirmed', reqId: 'confirm-1', renewed: true, acceptedAs: 'current' })
+ const after = Number((await deviceRow(reserve))?.current_expires_at)
+ expect(after).toBeGreaterThan(before)
+ expect(answer.resumeExpiresAt).toBe(after)
+ expect(reserve.phoneStatements()).toEqual([
+ CONFIRM_DEVICE_READ,
+ expect.stringMatching(/^UPDATE relay_devices SET current_expires_at = \?.*RETURNING current_expires_at$/)
+ ])
+ expect(await reserve.activityLeases()).toEqual([])
+ // Its audit row leaves with the next batch.
+ await reserve.relay.store.resumeAudit.flush()
+ const audits = await reserve.raw.query(
+ `SELECT detail_json FROM relay_audit_events WHERE type = 'resume-confirmed'`
+ )
+ expect(audits.map((row) => JSON.parse(String(row.detail_json)))).toEqual([
+ { reqId: 'confirm-1', basisConnId: expect.any(String), renewed: true }
+ ])
+ // A retry of the same request answers from memory; another basis cannot reuse its reqId.
+ reserve.database.statements.length = 0
+ expect(await confirm('confirm-1')).toEqual(answer)
+ expect(reserve.phoneStatements()).toEqual([])
+ expect(await confirm('confirm-1', 'another-basis')).toMatchObject({
+ type: 'control-error',
+ code: 'confirmation_tuple_mismatch'
+ })
+ })
+
+ it('answers renewed:true with the stored expiry when renewal is not due, or its write fails', async () => {
+ const fresh = Date.now() + 29.5 * 24 * 60 * 60 * 1000
+ const notDue = await splicedPhone('memory', { expiresAt: fresh })
+ expect(await notDue.confirm('confirm-1')).toMatchObject({ renewed: true, resumeExpiresAt: fresh })
+ expect(notDue.reserve.phoneStatements()).toEqual([CONFIRM_DEVICE_READ])
+
+ const due = Date.now() + 20 * 24 * 60 * 60 * 1000
+ const failing = await splicedPhone('memory', { expiresAt: due })
+ failing.reserve.database.failing = (sql) => sql.startsWith('UPDATE relay_devices')
+ expect(await failing.confirm('confirm-1')).toMatchObject({ renewed: true, resumeExpiresAt: due })
+ expect(Number((await deviceRow(failing.reserve))?.current_expires_at)).toBe(due)
+ // Never silent: a renewal left failing ages phones toward re-pairing.
+ expect(warnings('orca_relay_resume_renewal_failed')).toEqual([
+ { event: 'orca_relay_resume_renewal_failed', cause: 'error', cellId: failing.reserve.config.cellId }
+ ])
+ })
+
+ it('confirms from memory after the switch turned off between accept and confirm, taking no lease', async () => {
+ const { reserve, confirm } = await splicedPhone('memory')
+ reserve.setAdmitMode('reserve', { phonePath: 'db' })
+ expect(await confirm('confirm-1')).toMatchObject({ type: 'device-resume-confirmed', renewed: true })
+ expect(reserve.phoneStatements()[0]).toBe(CONFIRM_DEVICE_READ)
+ expect(await reserve.activityLeases()).toEqual([])
+ })
+
+ it('replays a stored answer after its splice closed, and drops the basis itself', async () => {
+ const { reserve, phone, confirm } = await splicedPhone('memory')
+ const answer = await confirm('confirm-1')
+ phone.socket.close()
+ await phone.closed
+ await new Promise((resolve) => setTimeout(resolve, 50))
+ reserve.database.statements.length = 0
+ expect(await confirm('confirm-1')).toEqual(answer)
+ expect(await confirm('confirm-1', 'another-basis')).toMatchObject({ code: 'confirmation_tuple_mismatch' })
+ expect(reserve.phoneStatements()).toEqual([])
+ // A new request on the closed basis finds none in memory, so it goes to the database.
+ expect(await confirm('confirm-2')).toMatchObject({ code: 'confirmation_not_active' })
+ expect(reserve.phoneStatements()[0]).toBe('BEGIN')
+ })
+
+ it('answers a grace-version confirm renewed:false with its expiry unchanged, as today', async () => {
+ const { reserve, confirm } = await splicedPhone('memory')
+ const stored = Number((await deviceRow(reserve))?.current_expires_at)
+ // An install rotated the credential after the accept read, so the accepted version is grace.
+ await reserve.raw.query(
+ `UPDATE relay_devices SET grace_hash = current_hash, grace_version = 1, grace_expires_at = ?,
+ current_hash = 'rotated', current_version = 2`,
+ [Date.now() + 60_000]
+ )
+ expect(await confirm('confirm-1')).toMatchObject({
+ type: 'device-resume-confirmed',
+ currentVersion: 2,
+ acceptedAs: 'current',
+ renewed: false,
+ resumeExpiresAt: stored
+ })
+ expect(reserve.phoneStatements()).toEqual([CONFIRM_DEVICE_READ])
+ })
+
+ it('answers every device change since the accept read as the database confirm does', async () => {
+ const changes: Record = {
+ revoked: ['UPDATE relay_devices SET revoked_at = ?', [Date.now() - 1]],
+ expired: ['UPDATE relay_devices SET current_expires_at = ?', [Date.now() - 1]],
+ retired: ["UPDATE relay_devices SET current_hash = 'rotated-twice', current_version = 3, grace_version = 2", []],
+ unchanged: ['UPDATE relay_devices SET updated_at = updated_at', []]
+ }
+ const outcome = async (phonePath: 'db' | 'memory', [sql, params]: [string, unknown[]]) => {
+ const { reserve, confirm } = await splicedPhone(phonePath)
+ await reserve.raw.query(sql, params)
+ const answer = await confirm('confirm-1')
+ return answer.type === 'control-error' ? answer.code : `${answer.type}:${answer.acceptedAs}:${answer.renewed}`
+ }
+ for (const [name, change] of Object.entries(changes)) {
+ expect(await outcome('memory', change), name).toBe(await outcome('db', change))
+ }
+ expect(await outcome('memory', changes.revoked!)).toBe('reject-revoked')
+ })
+
+ it('sends a confirm with no memory basis to the database, which holds none either', async () => {
+ const { reserve, phone, confirm } = await splicedPhone('memory')
+ phone.socket.close()
+ await phone.closed
+ await new Promise((resolve) => setTimeout(resolve, 50))
+ expect(await confirm('confirm-1')).toMatchObject({ type: 'control-error', code: 'confirmation_not_active' })
+ expect(reserve.phoneStatements()[0]).toBe('BEGIN')
+ expect(warnings('orca_relay_resume_confirm_failed')).toEqual([
+ {
+ event: 'orca_relay_resume_confirm_failed',
+ code: 'confirmation_not_active',
+ phonePath: 'db',
+ cellId: reserve.config.cellId
+ }
+ ])
+ })
+
+ it('does not count a lease release that failed after a committed confirm as a failed confirm', async () => {
+ const { reserve, confirm } = await splicedPhone('db')
+ vi.spyOn(reserve.relay.assignments, 'releaseActivity').mockRejectedValueOnce(new Error('connection terminated'))
+ // As today, the desktop still hears the release failure.
+ expect(await confirm('confirm-1')).toMatchObject({ type: 'control-error' })
+ expect(warnings('orca_relay_resume_confirm_failed')).toEqual([])
+ })
+
+ it('keeps the database confirm statement for statement while phonePath is db', async () => {
+ const { reserve, confirm } = await splicedPhone('db')
+ expect(await confirm('confirm-1')).toMatchObject({ type: 'device-resume-confirmed', renewed: true })
+ expect(reserve.phoneStatements()).toEqual(DATABASE_PATH_CONFIRM)
+ })
+
+ describe('rolling a memory cell back', () => {
+ // Flip back by the switch, or by the dead-man; either way the confirm finishes from memory.
+ type Cell = Awaited>
+ type Advance = (ms: number) => void
+ type Rollback = { beforeDial?: (advance: Advance) => void; roll: (reserve: Cell, advance: Advance) => void }
+ // The dead-man trips just after the phone's accept: its window ran from the host's booking.
+ const window = reserveDeadManWindowMs(RESERVE_MODE_CELL_ID)
+ const rollbacks: Record = {
+ 'admitMode=db': { roll: (reserve) => reserve.setAdmitMode('db', { phonePath: 'memory' }) },
+ 'phonePath=db': { roll: (reserve) => reserve.setAdmitMode('reserve', { phonePath: 'db' }) },
+ 'a dead-man trip': { beforeDial: (advance) => advance(window - 5_000), roll: (_reserve, advance) => advance(6_000) }
+ }
+
+ for (const [name, rollback] of Object.entries(rollbacks)) {
+ it(`finishes an in-flight confirm on its memory basis after ${name}, writing and leaking no lease`, async () => {
+ let now = Date.now()
+ const advance: Advance = (ms) => {
+ now += ms
+ }
+ const spliced = await splicedPhone('memory', {}, {
+ now: () => now,
+ beforeDial: () => rollback.beforeDial?.(advance)
+ })
+ const { reserve, seated } = spliced
+ // The flag tool records reserve in Postgres before the cell admits from memory.
+ await reserve.relay.assignments.reconcileCells([...reserve.config.cells, OTHER_CELL])
+ await reserve.relay.assignments.setCellAdmitMode(reserve.config.cellId, 'reserve')
+ rollback.roll(reserve, advance)
+ expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
+ reserve.database.statements.length = 0
+ expect(await spliced.confirm('confirm-1')).toMatchObject({ type: 'device-resume-confirmed', renewed: true })
+ expect(reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))).toEqual([
+ CONFIRM_DEVICE_READ,
+ expect.stringMatching(/^UPDATE relay_devices SET current_expires_at/)
+ ])
+ // Only the control lease a flip back re-registers; nothing for the phone.
+ const leases = async () => (await reserve.activityLeases()).map((lease) => lease.activity_kind)
+ expect((await leases()).filter((kind) => kind !== 'control')).toEqual([])
+ // Postgres still records reserve, so evacuation leaves the cell alone.
+ await expect(
+ reserve.relay.assignments.startEvacuation(
+ { userId: 'user-1', relayHostId: seated.identity.hostId },
+ OTHER_CELL.id
+ )
+ ).rejects.toThrow('cell_in_reserve_mode')
+ // The splice still runs, and closing it releases nothing it never took.
+ expect(spliced.phone.socket.readyState).toBe(spliced.phone.socket.OPEN)
+ // Let a flip back finish leasing its control (and fold its relay_cells write) first.
+ await until(async () => reserve.relay.sessions.reregistrationPending() === 0)
+ await new Promise((resolve) => setTimeout(resolve, 250))
+ reserve.database.statements.length = 0
+ spliced.phone.socket.close()
+ await spliced.phone.closed
+ await new Promise((resolve) => setTimeout(resolve, 50))
+ expect(reserve.phoneStatements()).toEqual([])
+ expect((await leases()).filter((kind) => kind !== 'control')).toEqual([])
+ })
+ }
+ })
+
+ // E-dep-6 in miniature: the staging drill blocks one memory cell's database for 10 minutes.
+ it('rides out a database outage: splices keep forwarding, new work fails cleanly and recovers', async () => {
+ let now = Date.now()
+ const reserve = await cell(() => now)
+ reserve.setAdmitMode('reserve', { phonePath: 'memory' })
+ const seated = await reserve.bookedHost(3)
+ const confirm = async (reqId: string, basisConnId: string) => {
+ const answer = Promise.race([
+ nextFrame(seated.socket, 'device-resume-confirmed'),
+ nextFrame(seated.socket, 'control-error')
+ ])
+ seated.socket.send(JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId }))
+ return await answer
+ }
+ const splice = async (fill: number) => {
+ await reserve.installDevice(seated.identity.hostId, token(fill), { relayDeviceId: `device-${fill}` })
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ const phone = await reserve.dialPhone(seated.identity.hostId, token(fill))
+ const basisConnId = String((await connOpen).connId)
+ const data = await reserve.attachData(await connOpen, seated.ack.generation)
+ expect(await phone.hello).toMatchObject({ ok: true })
+ return { phone, data, basisConnId }
+ }
+ const confirmed = await splice(1)
+ expect(await confirm('confirm-a', confirmed.basisConnId)).toMatchObject({ type: 'device-resume-confirmed' })
+ const inFlight = await splice(2)
+
+ reserve.database.failing = () => true
+ // A seated phone keeps working.
+ const relayed = new Promise((resolve) =>
+ confirmed.data.once('message', (frame) => resolve(String(frame)))
+ )
+ confirmed.phone.socket.send('still here')
+ expect(await relayed).toBe('still here')
+ // A new resume and a confirm fail cleanly; nothing crashes.
+ expect((await (await reserve.dialPhone(seated.identity.hostId, token(1))).closed).code).toBe(4503)
+ expect(await confirm('confirm-b', inFlight.basisConnId)).toMatchObject({ type: 'control-error' })
+ // Flipped back during the block: the in-flight confirm still finishes on its memory basis.
+ reserve.setAdmitMode('db', { phonePath: 'memory' })
+ reserve.database.failing = null
+ reserve.database.statements.length = 0
+ expect(await confirm('confirm-c', inFlight.basisConnId)).toMatchObject({ type: 'device-resume-confirmed' })
+ expect(reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))[0]).toBe(CONFIRM_DEVICE_READ)
+ // And new resumes take the database path again.
+ const connOpen = nextFrame(seated.socket, 'conn-open')
+ await reserve.dialPhone(seated.identity.hostId, token(1))
+ expect(await connOpen).toMatchObject({ kind: 'resume' })
+
+ // Audit rows held past 10 minutes of outage are dropped and counted, never written late.
+ reserve.database.failing = (sql) => sql.startsWith('INSERT INTO relay_audit_events')
+ now += 11 * 60 * 1000
+ await reserve.relay.store.resumeAudit.flush()
+ reserve.database.failing = null
+ const written = await reserve.raw.query(`SELECT id FROM relay_audit_events WHERE type = 'resume-confirmed'`)
+ expect(written).toEqual([])
+ expect(reserve.relay.runtimeCounts().resumeAuditDropped).toBe(2)
+ })
+})
+
diff --git a/cloud/apps/relay/src/cell-seat-feed.test.ts b/cloud/apps/relay/src/cell-seat-feed.test.ts
index 487f6356709e..1945af12b8cc 100644
--- a/cloud/apps/relay/src/cell-seat-feed.test.ts
+++ b/cloud/apps/relay/src/cell-seat-feed.test.ts
@@ -72,7 +72,8 @@ const SeatFeedReplySchema = z.object({
ticketCheck: z.enum(['off', 'shadow', 'enforce']),
rejectionFence: z.boolean(),
admitMode: z.enum(['db', 'reserve']),
- reserveDryRun: z.boolean()
+ reserveDryRun: z.boolean(),
+ phonePath: z.enum(['db', 'memory'])
}),
ignoredKeys: z.array(z.string()).optional()
})
@@ -200,6 +201,7 @@ function createCell(relayConfig = config(), cellFlags?: () => AppliedControlFlag
cellIncarnation: incarnation,
cellSeatFeed: (sinceSeq) => registry.seatFeed(sinceSeq),
cellFlags,
+ cellPhonePathEffective: () => registry.phonePathEffective(),
isDraining: () => registry.isDraining(),
runtimeCounts: () => ({
totalConnections: 0,
@@ -349,6 +351,8 @@ describe('cell seat feed', () => {
})
expect(await runtime.json()).toMatchObject({
flagsApplied: applied,
+ // Not reserve mode, so db whatever the switch says.
+ phonePathEffective: 'db',
databasePoolMax: expect.any(Number),
// What the flag tool checks a write against before it writes.
supportedFlags: {
@@ -359,7 +363,8 @@ describe('cell seat feed', () => {
admitMode: { type: 'enum', values: ['db', 'reserve'] },
intakePerSec: { type: 'number', max: 1_000 },
reserveDryRun: { type: 'boolean' },
- reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }
+ reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true },
+ phonePath: { type: 'enum', values: ['db', 'memory'] }
}
})
})
diff --git a/cloud/apps/relay/src/control-close-reason-text.ts b/cloud/apps/relay/src/control-close-reason-text.ts
new file mode 100644
index 000000000000..19ac79ea648f
--- /dev/null
+++ b/cloud/apps/relay/src/control-close-reason-text.ts
@@ -0,0 +1,7 @@
+// Peer-supplied close reasons are logged; keep them printable and short.
+export function printableCloseReason(reason: Buffer | string): string {
+ return reason
+ .toString()
+ .replace(/[^\x20-\x7e]/g, '')
+ .slice(0, 80)
+}
diff --git a/cloud/apps/relay/src/credential-store.ts b/cloud/apps/relay/src/credential-store.ts
index 8d17ba623f39..58e9ae3c47aa 100644
--- a/cloud/apps/relay/src/credential-store.ts
+++ b/cloud/apps/relay/src/credential-store.ts
@@ -7,6 +7,7 @@ import {
} from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
import { HeapWindowReaper } from './heap-window-reaper.js'
+import { ResumeConfirmedAuditBuffer } from './resume-confirmed-audit-buffer.js'
const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000
// Released desktops validate invite expiry against their own clock with zero
@@ -95,6 +96,64 @@ function string(row: SqlRow, field: string): string {
return value
}
+// Memory confirms renew a device at most this often (design decision 3: at most once a day of use).
+export const RESUME_RENEWAL_INTERVAL_MS = 24 * 60 * 60 * 1000
+
+function tokenFamily(device: SqlRow) {
+ return {
+ currentVersion: number(device, 'current_version'),
+ currentHash: string(device, 'current_hash'),
+ currentExpiresAt: number(device, 'current_expires_at'),
+ graceVersion: optionalNumber(device, 'grace_version'),
+ graceHash: typeof device.grace_hash === 'string' ? device.grace_hash : undefined,
+ graceExpiresAt: optionalNumber(device, 'grace_expires_at'),
+ revokedAt: optionalNumber(device, 'revoked_at')
+ }
+}
+
+// A device row's answer to a resume token: null when revoked, or neither hash is still valid.
+function resumeReservation(
+ device: SqlRow,
+ relayHostId: string,
+ tokenHash: string,
+ reservationId: string,
+ now: number
+): CredentialReservation | null {
+ if (optionalNumber(device, 'revoked_at') !== undefined) return null
+ const currentHash = string(device, 'current_hash')
+ const currentExpiresAt = number(device, 'current_expires_at')
+ const graceHash = device.grace_hash
+ const graceExpiresAt = optionalNumber(device, 'grace_expires_at')
+ let acceptedAs: 'current' | 'grace'
+ let acceptedCredentialVersion: number
+ if (equalHash(currentHash, tokenHash) && currentExpiresAt > now) {
+ acceptedAs = 'current'
+ acceptedCredentialVersion = number(device, 'current_version')
+ } else if (
+ typeof graceHash === 'string' &&
+ equalHash(graceHash, tokenHash) &&
+ (graceExpiresAt ?? 0) > now
+ ) {
+ acceptedAs = 'grace'
+ acceptedCredentialVersion = number(device, 'grace_version')
+ } else {
+ return null
+ }
+ return {
+ userId: string(device, 'user_id'),
+ relayHostId,
+ relayDeviceId: string(device, 'relay_device_id'),
+ credentialKind: 'resume',
+ tokenHash,
+ reservationId,
+ leaseExpiresAt: now + RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
+ acceptedCredentialVersion,
+ acceptedAs,
+ resumeExpiresAt: currentExpiresAt,
+ graceExpiresAt
+ }
+}
+
export class RelayCredentialStore {
// committed_at has no index, so a LIMIT delete would plan as a sequential scan of the whole table.
private readonly confirmResultReaper = new HeapWindowReaper(
@@ -103,10 +162,15 @@ export class RelayCredentialStore {
CONFIRM_RESULT_REAP_BUDGET
)
+ // Only memory confirms add to it, so its flush timer never starts on a director.
+ readonly resumeAudit: ResumeConfirmedAuditBuffer
+
constructor(
private readonly database: RelayDatabase,
private readonly now: () => number = Date.now
- ) {}
+ ) {
+ this.resumeAudit = new ResumeConfirmedAuditBuffer(database, now)
+ }
async createInvite(identity: RelayIdentity, relayDeviceId: string): Promise<{
inviteToken: string
@@ -216,42 +280,33 @@ export class RelayCredentialStore {
[relayHostId, tokenHash, tokenHash]
)
const device = deviceRows[0]
- if (!device || optionalNumber(device, 'revoked_at') !== undefined) return null
- const currentHash = string(device, 'current_hash')
- const currentExpiresAt = number(device, 'current_expires_at')
- const graceHash = device.grace_hash
- const graceExpiresAt = optionalNumber(device, 'grace_expires_at')
- let acceptedAs: 'current' | 'grace'
- let acceptedCredentialVersion: number
- if (equalHash(currentHash, tokenHash) && currentExpiresAt > now) {
- acceptedAs = 'current'
- acceptedCredentialVersion = number(device, 'current_version')
- } else if (
- typeof graceHash === 'string' &&
- equalHash(graceHash, tokenHash) &&
- (graceExpiresAt ?? 0) > now
- ) {
- acceptedAs = 'grace'
- acceptedCredentialVersion = number(device, 'grace_version')
- } else {
- return null
- }
- return {
- userId: string(device, 'user_id'),
- relayHostId,
- relayDeviceId: string(device, 'relay_device_id'),
- credentialKind: 'resume',
- tokenHash,
- reservationId,
- leaseExpiresAt: now + RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
- acceptedCredentialVersion,
- acceptedAs,
- resumeExpiresAt: currentExpiresAt,
- graceExpiresAt
- }
+ return device ? resumeReservation(device, relayHostId, tokenHash, reservationId, now) : null
})
}
+ // Step 6's one-read resume. Only `none` may fall through to the invite path: a device row
+ // that refuses this token closes the phone (4401), as reserveCredential would.
+ async lookupResume(
+ relayHostId: string,
+ token: string,
+ reservationId: string = randomUUID()
+ ): Promise<
+ | { outcome: 'none' }
+ | { outcome: 'refused' }
+ | { outcome: 'resume'; reservation: CredentialReservation }
+ > {
+ const tokenHash = hashCredential(token)
+ const rows = await this.database.query(
+ `SELECT * FROM relay_devices
+ WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)`,
+ [relayHostId, tokenHash, tokenHash]
+ )
+ const device = rows[0]
+ if (!device) return { outcome: 'none' }
+ const reservation = resumeReservation(device, relayHostId, tokenHash, reservationId, this.now())
+ return reservation ? { outcome: 'resume', reservation } : { outcome: 'refused' }
+ }
+
async recordConnectionBasis(input: CredentialReservation & {
basisConnId: string
owningControlGeneration: number
@@ -483,19 +538,7 @@ export class RelayCredentialStore {
const device = devices[0]
if (!device) throw new RelayStoreError('credential_not_found')
const acceptedVersion = number(basis, 'accepted_credential_version')
- const decision = decideResumeCommit(
- {
- currentVersion: number(device, 'current_version'),
- currentHash: string(device, 'current_hash'),
- currentExpiresAt: number(device, 'current_expires_at'),
- graceVersion: optionalNumber(device, 'grace_version'),
- graceHash: typeof device.grace_hash === 'string' ? device.grace_hash : undefined,
- graceExpiresAt: optionalNumber(device, 'grace_expires_at'),
- revokedAt: optionalNumber(device, 'revoked_at')
- },
- acceptedVersion,
- now
- )
+ const decision = decideResumeCommit(tokenFamily(device), acceptedVersion, now)
if (decision.startsWith('reject-')) throw new RelayStoreError(decision)
const renewed = decision === 'renew-current'
const resumeExpiresAt = renewed
@@ -547,6 +590,82 @@ export class RelayCredentialStore {
})
}
+ // Step 6's confirm for a basis the cell holds in memory: one fresh device read, and a renewal
+ // write at most once a day per device. A current-version confirm always answers renewed:true
+ // with the expiry the database now holds, so a phone keeps scheduling its renewals.
+ async confirmMemoryResume(
+ input: RelayIdentity & {
+ reqId: string
+ basisConnId: string
+ relayDeviceId: string
+ acceptedCredentialVersion: number
+ acceptedAs: 'current' | 'grace'
+ },
+ options: { onRenewalFailure?: (error: unknown) => void } = {}
+ ): Promise {
+ const now = this.now()
+ const devices = await this.database.query(
+ `SELECT * FROM relay_devices
+ WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`,
+ [input.userId, input.relayHostId, input.relayDeviceId]
+ )
+ const device = devices[0]
+ if (!device) throw new RelayStoreError('credential_not_found')
+ const decision = decideResumeCommit(tokenFamily(device), input.acceptedCredentialVersion, now)
+ if (decision.startsWith('reject-')) throw new RelayStoreError(decision)
+ const renewed = decision === 'renew-current'
+ let resumeExpiresAt = number(device, 'current_expires_at')
+ const renewalDue = resumeExpiresAt < now + RELAY_PROTOCOL_LIMITS.resumeTtlMs - RESUME_RENEWAL_INTERVAL_MS
+ if (renewed && renewalDue) {
+ // A failing renewal still answers the expiry the database holds, but must stay visible:
+ // left failing, phones age toward re-pairing.
+ resumeExpiresAt = await this.renewResume(input, device, now).catch((error: unknown) => {
+ options.onRenewalFailure?.(error)
+ return resumeExpiresAt
+ })
+ }
+ this.resumeAudit.add({
+ userId: input.userId,
+ relayHostId: input.relayHostId,
+ relayDeviceId: input.relayDeviceId,
+ detail: { reqId: input.reqId, basisConnId: input.basisConnId, renewed }
+ })
+ return {
+ v: 1,
+ reqId: input.reqId,
+ currentVersion: number(device, 'current_version'),
+ acceptedAs: input.acceptedAs,
+ renewed,
+ resumeExpiresAt,
+ ...(optionalNumber(device, 'grace_expires_at') === undefined
+ ? {}
+ : { graceExpiresAt: optionalNumber(device, 'grace_expires_at') })
+ }
+ }
+
+ // Guarded by the row just read, so a revoke or install that committed since leaves it alone;
+ // the answer then keeps the expiry that was read, which the database still holds.
+ private async renewResume(
+ identity: RelayIdentity & { relayDeviceId: string },
+ device: SqlRow,
+ now: number
+ ): Promise {
+ const renewedTo = now + RELAY_PROTOCOL_LIMITS.resumeTtlMs
+ const rows = await this.database.query(
+ `UPDATE relay_devices SET current_expires_at = ?, updated_at = ?
+ WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?
+ AND current_version = ? AND current_hash = ? AND revoked_at IS NULL
+ AND current_expires_at < ?
+ RETURNING current_expires_at`,
+ [
+ renewedTo, now, identity.userId, identity.relayHostId, identity.relayDeviceId,
+ number(device, 'current_version'), string(device, 'current_hash'),
+ now + RELAY_PROTOCOL_LIMITS.resumeTtlMs - RESUME_RENEWAL_INTERVAL_MS
+ ]
+ )
+ return rows[0] ? number(rows[0], 'current_expires_at') : number(device, 'current_expires_at')
+ }
+
async deactivateBasis(basisConnId: string): Promise {
await this.database.query(`UPDATE relay_connection_bases SET active = ? WHERE basis_conn_id = ?`, [
0,
@@ -580,25 +699,9 @@ export class RelayCredentialStore {
relayHostId: string,
token: string
): Promise<{ userId: string; relayDeviceId: string } | null> {
- const tokenHash = hashCredential(token)
- const rows = await this.database.query(
- `SELECT * FROM relay_devices
- WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)`,
- [relayHostId, tokenHash, tokenHash]
- )
- const device = rows[0]
- if (!device || optionalNumber(device, 'revoked_at') !== undefined) return null
- const now = this.now()
- const currentValid =
- equalHash(string(device, 'current_hash'), tokenHash) &&
- number(device, 'current_expires_at') > now
- const graceHash = device.grace_hash
- const graceValid =
- typeof graceHash === 'string' &&
- equalHash(graceHash, tokenHash) &&
- (optionalNumber(device, 'grace_expires_at') ?? 0) > now
- return currentValid || graceValid
- ? { userId: string(device, 'user_id'), relayDeviceId: string(device, 'relay_device_id') }
+ const found = await this.lookupResume(relayHostId, token)
+ return found.outcome === 'resume'
+ ? { userId: found.reservation.userId, relayDeviceId: found.reservation.relayDeviceId }
: null
}
diff --git a/cloud/apps/relay/src/floating-promise-census.test.ts b/cloud/apps/relay/src/floating-promise-census.test.ts
index fcc056a54462..10574467484d 100644
--- a/cloud/apps/relay/src/floating-promise-census.test.ts
+++ b/cloud/apps/relay/src/floating-promise-census.test.ts
@@ -15,6 +15,7 @@ const NEVER_REJECTS = new Map([
['cell-reserve-client.ts#send', 'one try/catch around the request; resolves every item'],
['cell-reserve-client.ts#demote', 'one try/catch around the request'],
['reserve-ledger-writer.ts#flush', 'one try/catch around the write; retries with backoff'],
+ ['resume-confirmed-audit-buffer.ts#flush', 'catches the insert; a failed batch goes back to the buffer'],
['control-renewal-batch.ts#flush', 'rejects the waiters, never itself'],
['regional-rehome-worker.ts#run', 'one try/catch around the whole poll'],
['relay-control-flag-channel.ts#poll', 'one catch on the whole read']
diff --git a/cloud/apps/relay/src/host-session-client-accept.test.ts b/cloud/apps/relay/src/host-session-client-accept.test.ts
index 7806e8d25b37..7448631383a2 100644
--- a/cloud/apps/relay/src/host-session-client-accept.test.ts
+++ b/cloud/apps/relay/src/host-session-client-accept.test.ts
@@ -114,7 +114,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) {
recordSql: vi.fn(),
recordClientAcceptAbandoned: vi.fn(),
recordClientAcceptCompleted: vi.fn(),
- recordControlRtt: vi.fn()
+ recordControlRtt: vi.fn(),
+ recordPhoneRejection: vi.fn(),
+ recordPhoneAttachDeadline: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
@@ -369,6 +371,42 @@ describe('client accept abandoned mid-DB-phase', () => {
}
})
+ it('logs and counts an attach deadline with what the cell knew of the control', async () => {
+ let now = 1_700_000_000_000
+ const h = harness({ now: () => now })
+ await activeHost(h)
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const client = new FakeSocket()
+ await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'credential')
+ now += RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs
+ vi.advanceTimersByTime(RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs)
+ // The phone-only reason; relay-hello itself is unchanged for strict old parsers.
+ expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE, 'attach-timeout')
+ expect(client.send).toHaveBeenCalledWith(JSON.stringify({ type: 'relay-hello', ok: false, code: 4404 }))
+ expect(h.observer.recordPhoneAttachDeadline).toHaveBeenCalledOnce()
+ expect(h.observer.recordPhoneRejection).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE)
+ const lines = warn.mock.calls
+ .map(([line]) => String(line))
+ .filter((line) => line.includes('orca_relay_attach_deadline_offline'))
+ expect(lines.map((line) => JSON.parse(line))).toEqual([
+ {
+ event: 'orca_relay_attach_deadline_offline',
+ relayHostIdDigest: expect.any(String),
+ credentialKind: 'resume',
+ msSinceLastPong: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
+ pingOutstanding: false,
+ controlAgeMs: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs,
+ controlRttMs: null,
+ pong: 'fresh',
+ cellId: config.cellId
+ }
+ ])
+ expect(lines[0]).not.toContain(identity.relayHostId)
+ warn.mockRestore()
+ h.registry.drain(0)
+ vi.advanceTimersByTime(0)
+ })
+
it('still opens the connection when the phone is holding on', async () => {
const h = harness()
const control = await activeHost(h)
@@ -437,7 +475,8 @@ describe('successful client accept timing', () => {
expect(ownerProbe).toHaveBeenCalledOnce()
expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({
totalMs: 49,
- stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }
+ stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 },
+ phonePath: 'db'
})
const line = log.mock.calls
.map((call) => String(call[0]))
diff --git a/cloud/apps/relay/src/host-session-registry.test.ts b/cloud/apps/relay/src/host-session-registry.test.ts
index d58899e84b61..0db469ad457e 100644
--- a/cloud/apps/relay/src/host-session-registry.test.ts
+++ b/cloud/apps/relay/src/host-session-registry.test.ts
@@ -7,11 +7,11 @@ import {
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS
} from '@orca-cloud/relay-contract'
-import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
-import type { RelayCredentialStore } from './credential-store.js'
+import { RelayStoreError, type RelayCredentialStore } from './credential-store.js'
import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js'
import {
CONTROL_RENEWAL_STATEMENT_OUTCOMES,
@@ -130,6 +130,7 @@ function createRegistry(
observer: {
recordAuth: ReturnType
recordControlClose: ReturnType
+ recordPostProofClose: ReturnType
recordSpliceClose: ReturnType
}
} {
@@ -177,6 +178,7 @@ function createRegistry(
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordControlClose: vi.fn(),
+ recordPostProofClose: vi.fn(),
recordSpliceClose: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
@@ -2038,3 +2040,234 @@ describe('paced drain', () => {
expect(sockets.map((socket) => socket.send.mock.calls.length)).toEqual(sendsAfterEmergency)
})
})
+
+describe('controls closed after host proof but before wiring', () => {
+ let warn: MockInstance
+ beforeEach(() => {
+ vi.useFakeTimers()
+ warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ })
+ afterEach(() => {
+ warn.mockRestore()
+ vi.clearAllTimers()
+ vi.useRealTimers()
+ })
+ const refusedLines = (): string[] =>
+ warn.mock.calls
+ .map((call) => String(call[0]))
+ .filter((line) => line.includes('control refused after proof'))
+
+ function expectRefusal(
+ observer: { recordPostProofClose: ReturnType },
+ code: number | null,
+ reason: string,
+ cause?: string
+ ): void {
+ expect(observer.recordPostProofClose).toHaveBeenCalledWith(reason)
+ const [line] = refusedLines()
+ expect(line).toContain(`host=${relayHostLogDigest(identity.relayHostId)} gen=1 app="1.4.173"`)
+ expect(line).toContain(`code=${code ?? 'none'} reason=${JSON.stringify(reason)}`)
+ if (cause) expect(line).toContain(`cause=${cause}`)
+ expect(line).not.toContain(identity.relayHostId)
+ expect(line).not.toContain(identity.sub)
+ }
+
+ it('logs an idle cutover refusal', async () => {
+ const h = createRegistry(vi.fn())
+ // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: beginIdleWork only checks the host key's presence.
+ const idle = h.registry as unknown as { idleAttempts: Map }
+ idle.idleAttempts.set(identity.relayHostId, {})
+ const socket = new FakeSocket()
+ await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
+ expect(socket.close).toHaveBeenCalledWith(
+ RELAY_CLOSE_CODE.WRONG_CELL,
+ 'idle cutover in progress'
+ )
+ expectRefusal(h.observer, RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
+ })
+
+ it('logs a stalled activation queue once', async () => {
+ const stalled = deferred()
+ const h = createRegistry(
+ vi.fn().mockReturnValueOnce(stalled.promise)
+ )
+ const first = h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1)
+ const waiting = new FakeSocket()
+ const second = h.activate(waiting as unknown as WebSocket, identity, null, 1, false, 1)
+ await Promise.resolve()
+ await Promise.resolve()
+ vi.advanceTimersByTime(30_000)
+ expectRefusal(h.observer, RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
+ stalled.resolve('control:production-gce-c3:1')
+ await Promise.all([first, second])
+ expect(h.observer.recordPostProofClose).toHaveBeenCalledOnce()
+ })
+
+ it('logs a superseded activation', async () => {
+ const h = createRegistry(vi.fn())
+ // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only compared by identity against the empty session map.
+ const stale = {} as HostSession
+ await h.activate(new FakeSocket() as unknown as WebSocket, identity, stale, 1, false, 1)
+ expectRefusal(h.observer, RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
+ })
+
+ it.each([
+ { error: new Error('wrong_assignment'), cause: 'wrong_assignment' },
+ { error: Object.assign(new Error('timeout'), { code: '57014' }), cause: 'database_transient' },
+ // Assembled at runtime so the secret scanner does not read the fixture as a real URL.
+ {
+ error: new Error(`connect ${['postgres', '//user:secret@db/relay'].join(':')}`),
+ cause: 'error'
+ }
+ ])('logs a failed host-proof assignment check as $cause', async ({ error, cause }) => {
+ const h = createRegistry(
+ vi.fn().mockRejectedValueOnce(error)
+ )
+ await h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1)
+ expectRefusal(
+ h.observer,
+ RELAY_CLOSE_CODE.WRONG_CELL,
+ 'assignment changed during host proof',
+ cause
+ )
+ expect(refusedLines()[0]).not.toContain('secret')
+ })
+
+ it('logs a drain that lands during activation', async () => {
+ const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
+ h.registry.drain(100)
+ await h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1)
+ expectRefusal(h.observer, RELAY_CLOSE_CODE.DRAINING, 'relay draining')
+ })
+
+ it('logs a host that hangs up during activation, with no cell close code', async () => {
+ const blocked = deferred()
+ const h = createRegistry(
+ vi.fn().mockReturnValueOnce(blocked.promise)
+ )
+ const socket = new FakeSocket()
+ const activation = h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
+ await vi.advanceTimersByTimeAsync(0)
+ socket.close()
+ blocked.resolve('control:production-gce-c3:1')
+ await activation
+ expectRefusal(h.observer, null, 'host closed during activation')
+ })
+
+ it('logs a host that hangs up while queued, with no cell close code', async () => {
+ const stalled = deferred()
+ const h = createRegistry(
+ vi
+ .fn()
+ .mockReturnValueOnce(stalled.promise)
+ .mockResolvedValue('control:production-gce-c3:1')
+ )
+ const first = h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1)
+ const queued = new FakeSocket()
+ const second = h.activate(queued as unknown as WebSocket, identity, null, 1, false, 1)
+ queued.close()
+ stalled.resolve('control:production-gce-c3:1')
+ await Promise.all([first, second])
+ expectRefusal(h.observer, null, 'host closed while queued')
+ })
+
+ it('logs the hang-up, not a stall, when the queued host left before the deadline', async () => {
+ const stalled = deferred()
+ const h = createRegistry(
+ vi.fn().mockReturnValueOnce(stalled.promise)
+ )
+ const first = h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1)
+ const queued = new FakeSocket()
+ const second = h.activate(queued as unknown as WebSocket, identity, null, 1, false, 1)
+ await Promise.resolve()
+ await Promise.resolve()
+ queued.readyState = queued.CLOSED
+ vi.advanceTimersByTime(30_000)
+ expect(queued.close).toHaveBeenCalledWith(
+ RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
+ 'control activation queue stalled'
+ )
+ expectRefusal(h.observer, null, 'host closed while queued')
+ stalled.resolve('control:production-gce-c3:1')
+ await Promise.all([first, second])
+ })
+
+ it('does not log a control that was wired', async () => {
+ const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1'))
+ await h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1)
+ expect(h.observer.recordPostProofClose).not.toHaveBeenCalled()
+ expect(refusedLines()).toEqual([])
+ })
+})
+
+describe('control error codes', () => {
+ beforeEach(() => vi.useFakeTimers())
+ afterEach(() => {
+ vi.clearAllTimers()
+ vi.useRealTimers()
+ })
+
+ async function revokeFailure(error: Error): Promise<{ sent: string[]; warned: string[] }> {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ const activateControl = vi
+ .fn()
+ .mockResolvedValue('control:production-gce-c3:1')
+ const { registry, activate } = createRegistry(activateControl, {
+ revoke: vi.fn().mockRejectedValue(error)
+ })
+ const socket = new FakeSocket()
+ await activate(socket as unknown as WebSocket, identity, null, 1, false, 1)
+ socket.emit(
+ 'message',
+ Buffer.from(JSON.stringify({ type: 'device-revoke', reqId: 'revoke-1', relayDeviceId: 'device-1' })),
+ false
+ )
+ await vi.advanceTimersByTimeAsync(0)
+ const sent = socket.send.mock.calls.map((call) => String(call[0]))
+ const warned = warn.mock.calls.map((call) => String(call[0]))
+ warn.mockRestore()
+ registry.drain(0)
+ vi.advanceTimersByTime(0)
+ return { sent, warned }
+ }
+
+ function controlErrors(sent: string[]): unknown[] {
+ return sent
+ .map((line) => JSON.parse(line) as Record)
+ .filter((message) => message.type === 'control-error')
+ }
+
+ it('never sends a database error message, which can carry the connection URL', async () => {
+ // Assembled at runtime so secret scanners do not flag a fixture URL.
+ const url = ['postgres', '://relay:', 'fake-pw', '@db.internal:5432/relay'].join('')
+ const error = Object.assign(new Error(`connect to ${url} failed: Connection terminated`), {
+ code: '08006'
+ })
+ const { sent, warned } = await revokeFailure(error)
+
+ expect(controlErrors(sent)).toEqual([
+ { type: 'control-error', reqId: 'revoke-1', code: 'database_transient' }
+ ])
+ expect(sent.join('\n')).not.toContain('fake-pw')
+ expect(warned).toContain('[orca-relay] control device-revoke failed: Error: redacted code=08006')
+ expect(warned.join('\n')).not.toContain('fake-pw')
+ })
+
+ it('maps any other unexpected message to a fixed code', async () => {
+ const { sent } = await revokeFailure(new Error('Unexpected token < in JSON'))
+
+ expect(controlErrors(sent)).toEqual([
+ { type: 'control-error', reqId: 'revoke-1', code: 'control_operation_failed' }
+ ])
+ })
+
+ it.each([
+ ['a store code', new RelayStoreError('reject-revoked'), 'reject-revoked'],
+ ['an invariant slug', new Error('authorization_expired'), 'authorization_expired']
+ ])('keeps %s on the wire', async (_kind, error, code) => {
+ const { sent, warned } = await revokeFailure(error)
+
+ expect(controlErrors(sent)).toEqual([{ type: 'control-error', reqId: 'revoke-1', code }])
+ expect(warned.join('\n')).not.toContain('control device-revoke failed')
+ })
+})
diff --git a/cloud/apps/relay/src/host-session-registry.ts b/cloud/apps/relay/src/host-session-registry.ts
index 4bbbf896830f..1ccbabb98e78 100644
--- a/cloud/apps/relay/src/host-session-registry.ts
+++ b/cloud/apps/relay/src/host-session-registry.ts
@@ -18,10 +18,12 @@ import {
RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME,
RELAY_PROTOCOL_LIMITS,
RELAY_CLOSE_CODE,
+ RELAY_PHONE_CLOSE_REASON,
type IdleRegionalRehomeCommit,
type IdleRegionalRehomeDeferReason,
type IdleRegionalRehomeResult,
type RelayHostCloseReason,
+ type RelayPhoneCloseReason,
type RelayRegion
} from '@orca-cloud/relay-contract'
import nacl from 'tweetnacl'
@@ -35,9 +37,18 @@ import type { CellAdmitMode, CellReserveBook } from './cell-reserve-book.js'
import type { DemoteRequest, ReserveOutcome, ReserveRequest } from './cell-reserve-contract.js'
import { CellSeatLog, type CellSeatChange, type CellSeatFeedPage } from './cell-seat-log.js'
import { isRelayDatabaseTransientError } from './database.js'
+import { printableCloseReason } from './control-close-reason-text.js'
import { ControlRenewalBatch } from './control-renewal-batch.js'
-import { RelayCredentialStore, type CredentialReservation } from './credential-store.js'
+import { RelayCredentialStore, RelayStoreError, type CredentialReservation } from './credential-store.js'
import { HostCloseReasonMemory } from './host-close-reason-memory.js'
+import { MemoryResumeConfirmations } from './memory-resume-confirmations.js'
+import { PhoneConnectionLog, type PhoneDatabaseCause } from './phone-connection-log.js'
+import {
+ postProofCloseCause,
+ PostProofCloseLog,
+ type PostProofCloseCause
+} from './post-proof-close-log.js'
+import { describeRelayFailure, relayErrorSlug } from './relay-error-redaction.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import {
@@ -52,14 +63,6 @@ import type { PendingHostDataReservation } from './relay-connection-ledger.js'
import { closeRelayWebSocket } from './relay-websocket-close.js'
import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js'
-// Peer-supplied close reasons are logged; keep them printable and short.
-function printableCloseReason(reason: Buffer | string): string {
- return reason
- .toString()
- .replace(/[^\x20-\x7e]/g, '')
- .slice(0, 80)
-}
-
type VerifyRelayToken = (token: string) => Promise
type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed'
@@ -134,6 +137,8 @@ type PendingConnection = {
client: WebSocket
attachTimer: ReturnType
credentialActivityId: string | null
+ // `memory`: a resume admitted from the seat, which writes no activity leases.
+ phonePath: 'db' | 'memory'
capacityReservation?: PendingHostDataReservation
timing: ClientAcceptTiming
}
@@ -249,6 +254,8 @@ export type CellReserveAdmission = {
directorContact?: () => void
// The switch file's flip-back pace, when set.
reregisterInFlight?: () => number | undefined
+ // The switch file's phone path; it acts only while the cell is in reserve mode.
+ phonePath?: () => 'db' | 'memory'
// The pool-pressure shed's condition: a hello memory cannot admit is shed, not queued.
databaseShedding: () => boolean
book: CellReserveBook
@@ -299,6 +306,9 @@ export class HostSessionRegistry {
// draining" is not the same question as "this host has been told to leave".
private readonly drainSentHosts = new Set()
private readonly seatLog: CellSeatLog
+ private readonly postProofCloses: PostProofCloseLog
+ private readonly phoneConnections: PhoneConnectionLog
+ private readonly memoryConfirmations: MemoryResumeConfirmations
// Seat key -> the demoted control socket. Its own set, not drainSentHosts: it ends with
// that socket (close, rebind or a new join), so a later booking here is a fresh seat.
private readonly demotedSeats = new Map()
@@ -442,6 +452,11 @@ export class HostSessionRegistry {
private readonly reserveAdmission?: CellReserveAdmission
) {
this.seatLog = new CellSeatLog(undefined, now)
+ this.postProofCloses = new PostProofCloseLog(observer)
+ this.phoneConnections = new PhoneConnectionLog(config.cellId)
+ this.memoryConfirmations = new MemoryResumeConfirmations(store, now, (error) =>
+ this.recordRenewalFailure(error)
+ )
}
// Reserve mode is only ever a cell's, and only while its applied switch says so.
@@ -453,6 +468,12 @@ export class HostSessionRegistry {
return this.reserveMode()
}
+ // Step 6: the seat is authoritative only in reserve mode, so a dead-man trip or admitMode=db
+ // turns the memory phone path off for new phones too.
+ phonePathEffective(): 'db' | 'memory' {
+ return this.reserveMode() && this.reserveAdmission?.phonePath?.() === 'memory' ? 'memory' : 'db'
+ }
+
// Leaving reserve mode: every control admitted from memory gets the database lease it never
// took, while it stays connected. Rescans until none lacks one, so a hello whose proof lands
// after the flip is caught too. A control whose row never names this cell is closed (4409).
@@ -833,34 +854,47 @@ export class HostSessionRegistry {
stageMs[stage] = at - stageCursor
stageCursor = at
}
- if (this.config.role === 'cell') {
- // Each lookup is its own pooled round trip; stop between them once the phone
- // has left instead of running the rest of the chain for nobody.
- let outerIdentity = await this.store.resolveResume(hostId, credential)
- if (abandonedByClient('assignment')) return
- if (!outerIdentity) {
- outerIdentity = await this.store.resolveInviteForMove(hostId, credential)
- if (abandonedByClient('assignment')) return
- }
- const assignment = outerIdentity
- ? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId })
- : null
- if (!assignment || assignment.cellId !== this.config.cellId) {
+ // Step 6: a resume is admitted from one device read and the seat, writing no leases.
+ let memoryReservation: CredentialReservation | null = null
+ let memoryMissedDevice = false
+ if (this.config.role === 'cell' && this.phonePathEffective() === 'memory') {
+ markStage('assignment')
+ let found: Awaited>
+ try {
+ found = await this.store.lookupResume(hostId, credential)
+ } catch (error) {
+ // Never the invite path: an unread device is not a missing one.
capacityReservation?.release()
- this.observer.recordAuth(false)
- this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
+ this.observer.recordPhoneAcceptDatabaseError?.()
+ this.phoneConnections.databaseError(databaseCause(error))
+ this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING)
return
}
- if (abandonedByClient('assignment')) return
- }
- markStage('assignment')
- const reservation = await this.store.reserveCredential(hostId, credential)
- if (!reservation) {
- capacityReservation?.release()
- this.observer.recordAuth(false)
- this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL)
- return
+ if (abandonedByClient('credential')) return
+ // The switch turned off during the read: today's whole path, never a lease-less memory phone.
+ if (this.phonePathEffective() === 'memory') {
+ if (found.outcome === 'refused') {
+ capacityReservation?.release()
+ this.observer.recordAuth(false)
+ this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL)
+ return
+ }
+ if (found.outcome === 'resume') memoryReservation = found.reservation
+ else memoryMissedDevice = true
+ }
}
+ const reservation =
+ memoryReservation ??
+ (await this.reserveCredentialFromDatabase(
+ socket,
+ hostId,
+ credential,
+ memoryMissedDevice,
+ capacityReservation,
+ abandonedByClient,
+ markStage
+ ))
+ if (!reservation) return
this.observer.recordAuth(true)
if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return
markStage('credential')
@@ -879,6 +913,10 @@ export class HostSessionRegistry {
session.socket.readyState !== session.socket.OPEN
) {
capacityReservation?.release()
+ if (memoryReservation) {
+ await this.rejectMissedSeat(socket, reservation, sessionKey)
+ return
+ }
await this.store.failReservation(reservation)
// The only rejection that can name a cause: the host is genuinely absent.
// The attach-deadline 4404 below fires while control is still connected.
@@ -900,7 +938,7 @@ export class HostSessionRegistry {
const connTicket = randomBytes(32).toString('base64url')
const identity = { userId: reservation.userId, relayHostId: hostId }
const credentialActivityId =
- this.config.role === 'cell'
+ this.config.role === 'cell' && !memoryReservation
? `${reservation.credentialKind === 'invite' ? 'invite' : 'confirmation'}:${connId}`
: null
if (credentialActivityId) {
@@ -945,7 +983,13 @@ export class HostSessionRegistry {
capacityReservation?.release()
this.failReservationBestEffort(reservation)
if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId)
- this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE)
+ this.recordAttachDeadline(session, reservation)
+ // OTA phones redial this cell once, quickly: the control is still seated here.
+ this.rejectClient(
+ socket,
+ RELAY_CLOSE_CODE.HOST_OFFLINE,
+ RELAY_PHONE_CLOSE_REASON.HOST_ATTACH_TIMEOUT
+ )
}, RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs)
const pending: PendingConnection = {
connId,
@@ -954,6 +998,7 @@ export class HostSessionRegistry {
client: socket,
attachTimer,
credentialActivityId,
+ phonePath: memoryReservation ? 'memory' : 'db',
capacityReservation,
// Attach starts where the activity stage ended, so the conn-open send is
// charged to it and no wall-clock gap goes unattributed.
@@ -982,6 +1027,74 @@ export class HostSessionRegistry {
})
}
+ // Today's credential path: invites, and every resume while the phone path is db. `deviceMissed`:
+ // the memory path already found no device row, so only the invite lookup is left.
+ private async reserveCredentialFromDatabase(
+ socket: WebSocket,
+ hostId: string,
+ credential: string,
+ deviceMissed: boolean,
+ capacityReservation: PendingHostDataReservation | undefined,
+ abandonedByClient: (stage: RelayClientAcceptStage, cleanup?: () => void) => boolean,
+ markStage: (stage: RelayClientAcceptStage) => void
+ ): Promise {
+ if (this.config.role === 'cell') {
+ // Each lookup is its own pooled round trip; stop between them once the phone
+ // has left instead of running the rest of the chain for nobody.
+ let outerIdentity = deviceMissed ? null : await this.store.resolveResume(hostId, credential)
+ if (abandonedByClient('assignment')) return null
+ if (!outerIdentity) {
+ outerIdentity = await this.store.resolveInviteForMove(hostId, credential)
+ if (abandonedByClient('assignment')) return null
+ }
+ const assignment = outerIdentity
+ ? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId })
+ : null
+ if (!assignment || assignment.cellId !== this.config.cellId) {
+ capacityReservation?.release()
+ this.observer.recordAuth(false)
+ this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
+ return null
+ }
+ if (abandonedByClient('assignment')) return null
+ }
+ markStage('assignment')
+ const reservation = await this.store.reserveCredential(hostId, credential)
+ if (!reservation) {
+ capacityReservation?.release()
+ this.observer.recordAuth(false)
+ this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL)
+ return null
+ }
+ return reservation
+ }
+
+ // No seat for a memory resume: one assignment read tells "moved" from "offline". A row that
+ // names this cell may be a lost ledger write (stale up to the 10 min reconcile), so it is logged.
+ private async rejectMissedSeat(
+ socket: WebSocket,
+ reservation: CredentialReservation,
+ sessionKey: string
+ ): Promise {
+ let elsewhere: boolean
+ try {
+ const row = await this.assignments.resolve({
+ userId: reservation.userId,
+ relayHostId: reservation.relayHostId
+ })
+ elsewhere = row !== null && row.cellId !== this.config.cellId
+ if (row?.cellId === this.config.cellId) this.phoneConnections.missRowNamesHere(reservation.relayHostId)
+ } catch (error) {
+ this.phoneConnections.missReadError(databaseCause(error))
+ elsewhere = true
+ }
+ if (elsewhere) {
+ this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL)
+ return
+ }
+ this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE, this.hostCloseReasons.read(sessionKey))
+ }
+
async acceptHostData(
socket: WebSocket,
connId: string,
@@ -1041,7 +1154,8 @@ export class HostSessionRegistry {
userId: pending.reservation.userId,
relayHostId: pending.reservation.relayHostId
}
- const spliceActivityId = this.config.role === 'cell' ? `splice:${connId}` : null
+ const spliceActivityId =
+ this.config.role === 'cell' && pending.phonePath === 'db' ? `splice:${connId}` : null
try {
if (spliceActivityId) {
await this.assignments.acquireActivity(identity, {
@@ -1050,12 +1164,16 @@ export class HostSessionRegistry {
cellId: this.config.cellId
})
}
- await this.store.recordConnectionBasis({
- ...pending.reservation,
- basisConnId: connId,
- owningControlGeneration: session.generation,
- deadline: basisDeadline
- })
+ if (pending.phonePath === 'memory') {
+ this.openMemoryBasis(pending, connId, session.generation, basisDeadline)
+ } else {
+ await this.store.recordConnectionBasis({
+ ...pending.reservation,
+ basisConnId: connId,
+ owningControlGeneration: session.generation,
+ deadline: basisDeadline
+ })
+ }
} catch {
session.activeConnIds.delete(connId)
await this.store.failReservation(pending.reservation)
@@ -1078,7 +1196,7 @@ export class HostSessionRegistry {
) {
session.activeConnIds.delete(connId)
pending.capacityReservation?.release()
- this.deactivateBasisBestEffort(connId)
+ this.closeBasis(pending, connId)
this.failReservationBestEffort(pending.reservation)
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
if (pending.credentialActivityId) {
@@ -1095,7 +1213,7 @@ export class HostSessionRegistry {
onClose: () => {
session.activeConnIds.delete(connId)
session.activeSplices.delete(connId)
- this.deactivateBasisBestEffort(connId)
+ this.closeBasis(pending, connId)
if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId)
if (pending.credentialActivityId) {
this.releaseActivityBestEffort(identity, pending.credentialActivityId)
@@ -1161,13 +1279,14 @@ export class HostSessionRegistry {
basis: nonNegativeMs(helloAt - attachedAt)
}
const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt)
- this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs })
+ this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs, phonePath: pending.phonePath })
console.log(
JSON.stringify({
event: 'orca_relay_client_accept_completed',
...this.logIdentity(),
...this.sessionPlacementLogFields(session),
credentialKind: pending.reservation.credentialKind,
+ phonePath: pending.phonePath,
stageMs,
totalMs,
relayHostIdDigest: relayHostLogDigest(session.relayHostId)
@@ -1627,9 +1746,15 @@ export class HostSessionRegistry {
// Present when the hello was admitted from memory: no database registration.
reserved?: { reservedBy?: string }
): Promise {
+ const refused = { relayHostId: identity.relayHostId, generation, appVersion }
const release = this.beginIdleWork(identity.relayHostId)
if (!release) {
- socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress')
+ this.refuseProvenControl(
+ socket,
+ refused,
+ RELAY_CLOSE_CODE.WRONG_CELL,
+ 'idle cutover in progress'
+ )
return Promise.resolve()
}
const key = this.key(identity.sub, identity.relayHostId)
@@ -1639,16 +1764,36 @@ export class HostSessionRegistry {
let queueWaitExpired = false
const queueWaitTimer = setTimeout(() => {
queueWaitExpired = true
- socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled')
+ this.refuseProvenControl(
+ socket,
+ refused,
+ RELAY_CLOSE_CODE.LIMIT_EXCEEDED,
+ 'control activation queue stalled',
+ undefined,
+ 'host closed while queued'
+ )
}, ACTIVATION_QUEUE_WAIT_MS)
queueWaitTimer.unref?.()
const activation = previous
.catch(() => undefined)
.then(async () => {
clearTimeout(queueWaitTimer)
- if (queueWaitExpired || socket.readyState !== socket.OPEN) return
+ if (queueWaitExpired) return
+ if (socket.readyState !== socket.OPEN) {
+ this.postProofCloses.record({
+ ...refused,
+ code: null,
+ reason: 'host closed while queued'
+ })
+ return
+ }
if ((this.sessions.get(key) ?? null) !== existing) {
- socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded')
+ this.refuseProvenControl(
+ socket,
+ refused,
+ RELAY_CLOSE_CODE.PEER_DROPPED,
+ 'control activation superseded'
+ )
return
}
await this.activateCurrent(
@@ -1683,6 +1828,7 @@ export class HostSessionRegistry {
connectionInclusionWatermark?: number,
reserved?: { reservedBy?: string }
): Promise {
+ const refused = { relayHostId: identity.relayHostId, generation, appVersion }
let controlActivityId: string | null = null
// Admitted from memory, but the cell left reserve mode before the proof landed: lease now.
if (this.config.role === 'cell' && !(reserved && this.reserveMode())) {
@@ -1704,7 +1850,7 @@ export class HostSessionRegistry {
{ userId: identity.sub, relayHostId: identity.relayHostId },
{ cellId: this.config.cellId, assignmentEpoch }
)
- } catch {
+ } catch (error) {
// A failure after activateControl succeeded must not strand the
// acquired control activity until lease expiry.
if (controlActivityId) {
@@ -1713,7 +1859,13 @@ export class HostSessionRegistry {
controlActivityId
)
}
- socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'assignment changed during host proof')
+ this.refuseProvenControl(
+ socket,
+ refused,
+ RELAY_CLOSE_CODE.WRONG_CELL,
+ 'assignment changed during host proof',
+ postProofCloseCause(error, isRelayDatabaseTransientError)
+ )
return
}
}
@@ -1725,7 +1877,13 @@ export class HostSessionRegistry {
)
}
if (socket.readyState === socket.OPEN) {
- socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining')
+ this.refuseProvenControl(socket, refused, RELAY_CLOSE_CODE.DRAINING, 'relay draining')
+ } else {
+ this.postProofCloses.record({
+ ...refused,
+ code: null,
+ reason: 'host closed during activation'
+ })
}
return
}
@@ -1828,6 +1986,22 @@ export class HostSessionRegistry {
this.appendSeatChange(session, 'join', undefined, reserved?.reservedBy)
}
+ private refuseProvenControl(
+ socket: WebSocket,
+ refused: { relayHostId: string; generation: number; appVersion: string },
+ code: number,
+ reason: string,
+ cause?: PostProofCloseCause,
+ hungUpReason = 'host closed during activation'
+ ): void {
+ // A host that already hung up got no close frame from us; log what happened instead.
+ const open = socket.readyState === socket.OPEN
+ socket.close(code, reason)
+ this.postProofCloses.record(
+ open ? { ...refused, code, reason, cause } : { ...refused, code: null, reason: hungUpReason }
+ )
+ }
+
seatFeed(sinceSeq: number | null): CellSeatFeedPage {
return { ...this.seatLog.read(sinceSeq), seats: this.seatLog.seatCount() }
}
@@ -2322,12 +2496,22 @@ export class HostSessionRegistry {
}
if (type === 'device-resume-confirm') {
const request = DeviceResumeConfirmSchema.parse(payload(raw, type))
- const result = await this.store.confirmResume({
- ...identity,
- ...request,
- owningControlGeneration: session.generation
- })
- await this.assignments.releaseActivity(identity, `confirmation:${request.basisConnId}`)
+ const confirm = { ...identity, ...request, owningControlGeneration: session.generation }
+ const phonePath = this.memoryConfirmations.owns(confirm) ? 'memory' : 'db'
+ let result
+ try {
+ result =
+ phonePath === 'memory'
+ ? await this.memoryConfirmations.confirm(confirm)
+ : await this.store.confirmResume(confirm)
+ } catch (error) {
+ this.recordConfirmFailure(error, phonePath)
+ throw error
+ }
+ // After a committed confirm: a failed release is not a failed confirm.
+ if (phonePath === 'db') {
+ await this.assignments.releaseActivity(identity, `confirmation:${request.basisConnId}`)
+ }
send(session.socket, 'device-resume-confirmed', result)
return
}
@@ -2345,28 +2529,51 @@ export class HostSessionRegistry {
? String(candidate.reqId)
: undefined
})()
- this.sendControlError(
- session,
- reqId,
- error instanceof Error ? error.message : 'control_operation_failed'
- )
+ this.sendControlError(session, reqId, controlErrorCode(type, error))
}
}
+ private recordRenewalFailure(error: unknown): void {
+ this.observer.recordResumeRenewalFailure?.()
+ this.phoneConnections.renewalFailed(databaseCause(error))
+ }
+
+ // Never the raw message as a key: pg errors can carry the database URL.
+ private recordConfirmFailure(error: unknown, phonePath: 'db' | 'memory'): void {
+ const code = error instanceof RelayStoreError ? error.code : databaseCause(error)
+ this.observer.recordResumeConfirmFailure?.(code)
+ this.phoneConnections.confirmFailed(code, phonePath)
+ }
+
+ private recordAttachDeadline(session: HostSession, reservation: CredentialReservation): void {
+ const now = this.now()
+ const msSinceLastPong = now - session.lastPongAt
+ this.observer.recordPhoneAttachDeadline?.()
+ this.phoneConnections.attachDeadline(session.relayHostId, {
+ credentialKind: reservation.credentialKind,
+ msSinceLastPong,
+ pingOutstanding: session.pendingPingAt !== null,
+ controlAgeMs: now - session.controlWiredAt,
+ controlRttMs: session.controlRttSamplesMs.at(-1) ?? null,
+ pong: msSinceLastPong > RELAY_PROTOCOL_LIMITS.controlPingIntervalMs ? 'stale' : 'fresh'
+ })
+ }
+
private sendControlError(session: HostSession, reqId: string | undefined, code: string): void {
if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code })
}
- // hostCloseReason rides the WebSocket close reason, never relay-hello: every
- // shipped phone parses relay-hello with a strict schema that rejects an
- // unknown key, and none of them read the close reason at all.
+ // closeReason rides the WebSocket close reason, never relay-hello: every shipped
+ // phone parses relay-hello with a strict schema that rejects an unknown key, and
+ // ignores a close reason it does not know.
private rejectClient(
socket: WebSocket,
code: number,
- hostCloseReason?: RelayHostCloseReason | null
+ closeReason?: RelayHostCloseReason | RelayPhoneCloseReason | null
): void {
+ this.observer.recordPhoneRejection?.(code)
send(socket, 'relay-hello', { ok: false, code })
- closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected')
+ closeRelayWebSocket(socket, code, closeReason ?? 'relay connection rejected')
}
private releaseControlActivity(session: HostSession): void {
@@ -2393,6 +2600,34 @@ export class HostSessionRegistry {
})
}
+ // A memory resume's basis is the cell's own record; every other basis is a Postgres row.
+ private openMemoryBasis(
+ pending: PendingConnection,
+ connId: string,
+ owningControlGeneration: number,
+ deadline: number
+ ): void {
+ const { reservation } = pending
+ if (reservation.acceptedCredentialVersion === undefined || reservation.acceptedAs === undefined) {
+ throw new Error('memory basis needs an accepted resume credential')
+ }
+ this.memoryConfirmations.open({
+ basisConnId: connId,
+ userId: reservation.userId,
+ relayHostId: reservation.relayHostId,
+ relayDeviceId: reservation.relayDeviceId,
+ acceptedCredentialVersion: reservation.acceptedCredentialVersion,
+ acceptedAs: reservation.acceptedAs,
+ owningControlGeneration,
+ deadline
+ })
+ }
+
+ private closeBasis(pending: PendingConnection, connId: string): void {
+ if (pending.phonePath === 'memory') this.memoryConfirmations.close(connId)
+ else this.deactivateBasisBestEffort(connId)
+ }
+
private deactivateBasisBestEffort(connId: string): void {
void this.store.deactivateBasis(connId).catch(() => {
console.warn('[orca-relay] basis deactivation deferred to credential cleanup')
@@ -2401,3 +2636,16 @@ export class HostSessionRegistry {
}
export type RelayIdentityKey = { userId: string; relayHostId: string }
+
+// Intentional throws are store codes or slugs; anything else could carry the database URL.
+function controlErrorCode(type: string, error: unknown): string {
+ if (error instanceof RelayStoreError) return error.code
+ const slug = relayErrorSlug(error)
+ if (slug) return slug
+ console.warn(`[orca-relay] control ${type} failed: ${describeRelayFailure(error)}`)
+ return isRelayDatabaseTransientError(error) ? 'database_transient' : 'control_operation_failed'
+}
+
+function databaseCause(error: unknown): PhoneDatabaseCause {
+ return isRelayDatabaseTransientError(error) ? 'database_transient' : 'error'
+}
diff --git a/cloud/apps/relay/src/idle-regional-rehome-store.test.ts b/cloud/apps/relay/src/idle-regional-rehome-store.test.ts
index b83cd57f92f9..ce84bf78e531 100644
--- a/cloud/apps/relay/src/idle-regional-rehome-store.test.ts
+++ b/cloud/apps/relay/src/idle-regional-rehome-store.test.ts
@@ -217,6 +217,15 @@ describe('constrained idle regional assignment transaction', () => {
expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
})
+ // sourceRequestUnits reads a reserve cell's missing phone leases as zero units; selection must drop it.
+ it.each(['source', 'target'])('drops every candidate while the %s cell is recorded as reserve', async (cellId) => {
+ const { store, safety } = await setup()
+ await store.setCellAdmitMode(cellId, 'reserve')
+ expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([])
+ await store.setCellAdmitMode(cellId, 'db')
+ expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1)
+ })
+
it('leaves the candidate page offset untouched across a closed dispatch budget', async () => {
const { store, database, safety } = await setup()
const first = await store.selectIdleRegionalRehomeCandidates(safety)
diff --git a/cloud/apps/relay/src/memory-resume-confirmations.ts b/cloud/apps/relay/src/memory-resume-confirmations.ts
new file mode 100644
index 000000000000..c2de7a7be7a8
--- /dev/null
+++ b/cloud/apps/relay/src/memory-resume-confirmations.ts
@@ -0,0 +1,98 @@
+import type { DeviceResumeConfirmed } from '@orca-cloud/relay-contract'
+import { RelayStoreError, type RelayCredentialStore, type RelayIdentity } from './credential-store.js'
+
+// Step 6: what a memory cell keeps instead of relay_connection_bases and relay_confirm_results.
+// Both live and die with this process; a restart takes the splices with it.
+
+export type MemoryConnectionBasis = RelayIdentity & {
+ basisConnId: string
+ relayDeviceId: string
+ acceptedCredentialVersion: number
+ acceptedAs: 'current' | 'grace'
+ owningControlGeneration: number
+ deadline: number
+}
+
+export type ResumeConfirmRequest = RelayIdentity & {
+ reqId: string
+ basisConnId: string
+ owningControlGeneration: number
+}
+
+// A retried confirm of the same request answers from here, as relay_confirm_results does.
+export const MEMORY_CONFIRM_RESULT_RETENTION_MS = 10 * 60 * 1000
+
+type StoredResult = { basisConnId: string; result: DeviceResumeConfirmed; at: number }
+
+function resultKey(request: RelayIdentity & { reqId: string }): string {
+ return `${request.userId}\u0000${request.relayHostId}\u0000${request.reqId}`
+}
+
+export class MemoryResumeConfirmations {
+ private readonly bases = new Map()
+ // Insertion order is age order, so expiry walks from the front.
+ private readonly results = new Map()
+
+ constructor(
+ private readonly store: RelayCredentialStore,
+ private readonly now: () => number = Date.now,
+ private readonly onRenewalFailure?: (error: unknown) => void
+ ) {}
+
+ open(basis: MemoryConnectionBasis): void {
+ this.bases.set(basis.basisConnId, basis)
+ }
+
+ close(basisConnId: string): void {
+ this.bases.delete(basisConnId)
+ }
+
+ // Routed by where the basis lives, not by the switch, so a flip between accept and confirm
+ // neither fails the confirm nor sends it to a database that holds no basis for it.
+ owns(request: ResumeConfirmRequest): boolean {
+ this.expireResults()
+ return this.bases.has(request.basisConnId) || this.results.has(resultKey(request))
+ }
+
+ async confirm(request: ResumeConfirmRequest): Promise {
+ this.expireResults()
+ const key = resultKey(request)
+ const prior = this.results.get(key)
+ if (prior) {
+ if (prior.basisConnId !== request.basisConnId) throw new RelayStoreError('confirmation_tuple_mismatch')
+ return prior.result
+ }
+ const basis = this.bases.get(request.basisConnId)
+ if (
+ !basis ||
+ basis.userId !== request.userId ||
+ basis.relayHostId !== request.relayHostId ||
+ basis.owningControlGeneration !== request.owningControlGeneration ||
+ basis.deadline < this.now()
+ ) {
+ throw new RelayStoreError('confirmation_not_active')
+ }
+ const result = await this.store.confirmMemoryResume(
+ {
+ userId: basis.userId,
+ relayHostId: basis.relayHostId,
+ reqId: request.reqId,
+ basisConnId: basis.basisConnId,
+ relayDeviceId: basis.relayDeviceId,
+ acceptedCredentialVersion: basis.acceptedCredentialVersion,
+ acceptedAs: basis.acceptedAs
+ },
+ { onRenewalFailure: this.onRenewalFailure }
+ )
+ this.results.set(key, { basisConnId: basis.basisConnId, result, at: this.now() })
+ return result
+ }
+
+ private expireResults(): void {
+ const cutoff = this.now() - MEMORY_CONFIRM_RESULT_RETENTION_MS
+ for (const [key, stored] of this.results) {
+ if (stored.at > cutoff) break
+ this.results.delete(key)
+ }
+ }
+}
diff --git a/cloud/apps/relay/src/phone-connection-log.ts b/cloud/apps/relay/src/phone-connection-log.ts
new file mode 100644
index 000000000000..52b8c3ace6fa
--- /dev/null
+++ b/cloud/apps/relay/src/phone-connection-log.ts
@@ -0,0 +1,86 @@
+import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
+import { relayHostLogDigest } from './relay-host-log-digest.js'
+
+// Phone-connection anomalies that repeat per dial (a stuck database, a lost ledger write, a
+// half-open control): per kind, one line opens the window and one closing line carries the rest.
+export const PHONE_CONNECTION_LOG_WINDOW_MS = 60_000
+
+export type PhoneDatabaseCause = 'database_transient' | 'error'
+
+// What the cell knew of the desktop's control when a phone's attach deadline passed: a stale
+// pong is a half-open control, a fresh one a slow or failing desktop data dial.
+export type AttachDeadlineEvidence = {
+ credentialKind: 'invite' | 'resume'
+ msSinceLastPong: number
+ pingOutstanding: boolean
+ controlAgeMs: number
+ controlRttMs: number | null
+ pong: 'stale' | 'fresh'
+}
+
+type PhoneConnectionAnomaly =
+ | { event: 'orca_relay_client_accept_db_error'; cause: PhoneDatabaseCause }
+ | { event: 'orca_relay_client_miss_row_names_here'; relayHostIdDigest: string }
+ | { event: 'orca_relay_client_miss_read_error'; cause: PhoneDatabaseCause }
+ | { event: 'orca_relay_resume_renewal_failed'; cause: PhoneDatabaseCause }
+ | ({ event: 'orca_relay_attach_deadline_offline'; relayHostIdDigest: string } & AttachDeadlineEvidence)
+ | { event: 'orca_relay_resume_confirm_failed'; code: string; phonePath: 'db' | 'memory' }
+
+export class PhoneConnectionLog {
+ private readonly window: AssignmentRejectionLogWindow
+
+ constructor(
+ private readonly cellId: string,
+ private readonly warn: (line: string) => void = (line) => console.warn(line),
+ schedule?: (callback: () => void, delayMs: number) => () => void
+ ) {
+ this.window = new AssignmentRejectionLogWindow({
+ windowMs: PHONE_CONNECTION_LOG_WINDOW_MS,
+ onWindowClosed: ({ suppressed, sample }) => this.write(sample, suppressed),
+ schedule
+ })
+ }
+
+ // The one device read failed; the phone was closed 4503 and never took the invite path.
+ databaseError(cause: PhoneDatabaseCause): void {
+ this.record(cause, { event: 'orca_relay_client_accept_db_error', cause })
+ }
+
+ // No seat, yet the row names this cell: the 4404 may be stale (a lost reserve ledger write).
+ missRowNamesHere(relayHostId: string): void {
+ this.record('', {
+ event: 'orca_relay_client_miss_row_names_here',
+ relayHostIdDigest: relayHostLogDigest(relayHostId)
+ })
+ }
+
+ // The no-seat assignment read failed; the phone was closed 4409 to re-resolve.
+ missReadError(cause: PhoneDatabaseCause): void {
+ this.record(cause, { event: 'orca_relay_client_miss_read_error', cause })
+ }
+
+ // A due renewal write failed; the confirm still answered the stored expiry.
+ renewalFailed(cause: PhoneDatabaseCause): void {
+ this.record(cause, { event: 'orca_relay_resume_renewal_failed', cause })
+ }
+
+ attachDeadline(relayHostId: string, evidence: AttachDeadlineEvidence): void {
+ this.record(`${evidence.credentialKind}:${evidence.pong}`, {
+ event: 'orca_relay_attach_deadline_offline',
+ relayHostIdDigest: relayHostLogDigest(relayHostId),
+ ...evidence
+ })
+ }
+
+ confirmFailed(code: string, phonePath: 'db' | 'memory'): void {
+ this.record(`${code}:${phonePath}`, { event: 'orca_relay_resume_confirm_failed', code, phonePath })
+ }
+
+ private record(kind: string, anomaly: PhoneConnectionAnomaly): void {
+ if (this.window.admit(`${anomaly.event}:${kind}`, anomaly)) this.write(anomaly, 0)
+ }
+
+ private write(anomaly: PhoneConnectionAnomaly, suppressed: number): void {
+ this.warn(JSON.stringify({ ...anomaly, cellId: this.cellId, ...(suppressed > 0 ? { suppressed } : {}) }))
+ }
+}
diff --git a/cloud/apps/relay/src/post-proof-close-log.test.ts b/cloud/apps/relay/src/post-proof-close-log.test.ts
new file mode 100644
index 000000000000..6113c8d72a8a
--- /dev/null
+++ b/cloud/apps/relay/src/post-proof-close-log.test.ts
@@ -0,0 +1,63 @@
+import { describe, expect, it, vi } from 'vitest'
+import { POST_PROOF_CLOSE_LOG_WINDOW_MS, PostProofCloseLog } from './post-proof-close-log.js'
+
+const close = {
+ relayHostId: 'abcdefghijklmnop',
+ generation: 7,
+ appVersion: '1.4.150',
+ code: 4409,
+ reason: 'assignment changed during host proof',
+ cause: 'wrong_assignment' as const
+}
+
+function createLog() {
+ const lines: string[] = []
+ const scheduled: Array<{ callback: () => void; delayMs: number }> = []
+ const observer = {
+ recordAuth: vi.fn(),
+ recordForwardedBytes: vi.fn(),
+ recordHttp: vi.fn(),
+ recordReconnect: vi.fn(),
+ recordSql: vi.fn(),
+ recordPostProofClose: vi.fn()
+ }
+ const log = new PostProofCloseLog(
+ observer,
+ (line) => lines.push(line),
+ (callback, delayMs) => {
+ scheduled.push({ callback, delayMs })
+ return () => undefined
+ }
+ )
+ return { log, lines, scheduled, observer }
+}
+
+describe('post-proof close log', () => {
+ it('counts every close but writes one line per reason and cause per window', () => {
+ const { log, lines, scheduled, observer } = createLog()
+ for (let index = 0; index < 1_000; index++) log.record({ ...close, generation: index })
+ log.record({ ...close, cause: 'database_transient' })
+ log.record({ ...close, code: 4400, reason: 'relay draining', cause: undefined })
+
+ expect(observer.recordPostProofClose).toHaveBeenCalledTimes(1_002)
+ expect(lines).toHaveLength(3)
+ expect(scheduled.map((entry) => entry.delayMs)).toEqual(
+ Array.from({ length: 3 }, () => POST_PROOF_CLOSE_LOG_WINDOW_MS)
+ )
+
+ for (const { callback } of scheduled) callback()
+ // Only the window that held closes back writes a closing line, naming the latest.
+ expect(lines).toHaveLength(4)
+ expect(lines[3]).toContain('gen=999')
+ expect(lines[3]).toContain('suppressed=999')
+ })
+
+ it('names the host by digest and keeps peer-supplied versions printable', () => {
+ const { log, lines } = createLog()
+ log.record({ ...close, appVersion: '1.4.150\n\u0007injected' })
+ expect(lines[0]).toMatch(
+ /^\[orca-relay\] control refused after proof host=\S+ gen=7 app="1\.4\.150injected" code=4409 reason="assignment changed during host proof" cause=wrong_assignment$/
+ )
+ expect(lines[0]).not.toContain(close.relayHostId)
+ })
+})
diff --git a/cloud/apps/relay/src/post-proof-close-log.ts b/cloud/apps/relay/src/post-proof-close-log.ts
new file mode 100644
index 000000000000..8385509309d5
--- /dev/null
+++ b/cloud/apps/relay/src/post-proof-close-log.ts
@@ -0,0 +1,65 @@
+import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js'
+import { printableCloseReason } from './control-close-reason-text.js'
+import { relayHostLogDigest } from './relay-host-log-digest.js'
+import type { RelayRuntimeObserver } from './relay-observability.js'
+
+// A looping host hits a post-proof close about once a second; per reason and
+// cause, one line opens the window and one closing line carries the rest.
+export const POST_PROOF_CLOSE_LOG_WINDOW_MS = 60_000
+
+// Why the assignment check failed: the close reason alone cannot tell a moved
+// assignment from a database timeout. Never the raw message (pg errors carry URLs).
+export type PostProofCloseCause = 'wrong_assignment' | 'database_transient' | 'error'
+
+export type PostProofClose = {
+ relayHostId: string
+ generation: number
+ appVersion: string
+ // Null when the host hung up itself: the cell sent no close frame.
+ code: number | null
+ reason: string
+ cause?: PostProofCloseCause
+}
+
+// Controls closed between host proof and wiring never reach the 'control closed' listener.
+export class PostProofCloseLog {
+ private readonly window: AssignmentRejectionLogWindow
+
+ constructor(
+ private readonly observer: RelayRuntimeObserver,
+ private readonly warn: (line: string) => void = (line) => console.warn(line),
+ schedule?: (callback: () => void, delayMs: number) => () => void
+ ) {
+ this.window = new AssignmentRejectionLogWindow({
+ windowMs: POST_PROOF_CLOSE_LOG_WINDOW_MS,
+ onWindowClosed: ({ suppressed, sample }) => this.warn(formatLine(sample, suppressed)),
+ schedule
+ })
+ }
+
+ record(close: PostProofClose): void {
+ this.observer.recordPostProofClose?.(close.reason)
+ if (!this.window.admit(`${close.reason}:${close.cause ?? ''}`, close)) return
+ this.warn(formatLine(close, 0))
+ }
+}
+
+export function postProofCloseCause(
+ error: unknown,
+ isTransient: (error: unknown) => boolean
+): PostProofCloseCause {
+ if (error instanceof Error && error.message === 'wrong_assignment') return 'wrong_assignment'
+ return isTransient(error) ? 'database_transient' : 'error'
+}
+
+function formatLine(close: PostProofClose, suppressed: number): string {
+ return (
+ `[orca-relay] control refused after proof host=${relayHostLogDigest(close.relayHostId)}` +
+ ` gen=${close.generation}` +
+ ` app=${JSON.stringify(printableCloseReason(close.appVersion))}` +
+ ` code=${close.code ?? 'none'} reason=${JSON.stringify(close.reason)}` +
+ (close.cause ? ` cause=${close.cause}` : '') +
+ // Closing line only: how many more closes the window held back; the fields name the latest.
+ (suppressed > 0 ? ` suppressed=${suppressed}` : '')
+ )
+}
diff --git a/cloud/apps/relay/src/relay-background-operation.ts b/cloud/apps/relay/src/relay-background-operation.ts
index be3943b7b1ad..adf704106cd4 100644
--- a/cloud/apps/relay/src/relay-background-operation.ts
+++ b/cloud/apps/relay/src/relay-background-operation.ts
@@ -1,19 +1,9 @@
+import { describeRelayFailure } from './relay-error-redaction.js'
+
type Warn = (message: string) => void
// A swallowed error hides which sweep step is failing (a silently dead
-// cleanup chain stalled production rehoming for hours), but raw messages can
-// embed secrets such as connection strings. Only two provably inert shapes
-// are logged: this codebase's snake_case invariant slugs, and five-character
-// SQLSTATE codes; everything else stays redacted.
-function describeFailure(error: unknown): string {
- const name = error instanceof Error ? error.name : typeof error
- const message = error instanceof Error ? error.message : ''
- const slug = /^[a-z0-9_]{1,64}$/.test(message) ? message : 'redacted'
- const code = (error as { code?: unknown } | null)?.code
- const sqlState = typeof code === 'string' && /^[0-9A-Z]{5}$/.test(code) ? ` code=${code}` : ''
- return `${name}: ${slug}${sqlState}`
-}
-
+// cleanup chain stalled production rehoming for hours).
export async function runRelayBackgroundOperation(
operation: () => Promise,
failureMessage: string,
@@ -23,6 +13,6 @@ export async function runRelayBackgroundOperation(
await operation()
} catch (error) {
// Dependency outages must fail readiness without crashing liveness.
- warn(`${failureMessage}: ${describeFailure(error)}`)
+ warn(`${failureMessage}: ${describeRelayFailure(error)}`)
}
}
diff --git a/cloud/apps/relay/src/relay-error-redaction.ts b/cloud/apps/relay/src/relay-error-redaction.ts
new file mode 100644
index 000000000000..f67642a04002
--- /dev/null
+++ b/cloud/apps/relay/src/relay-error-redaction.ts
@@ -0,0 +1,14 @@
+// Raw error messages can embed secrets such as connection strings. Only two
+// provably inert shapes leave the process: this codebase's snake_case
+// invariant slugs, and five-character SQLSTATE codes; everything else is redacted.
+export function relayErrorSlug(error: unknown): string | null {
+ const message = error instanceof Error ? error.message : ''
+ return /^[a-z0-9_]{1,64}$/.test(message) ? message : null
+}
+
+export function describeRelayFailure(error: unknown): string {
+ const name = error instanceof Error ? error.name : typeof error
+ const code = typeof error === 'object' && error !== null && 'code' in error ? error.code : null
+ const sqlState = typeof code === 'string' && /^[0-9A-Z]{5}$/.test(code) ? ` code=${code}` : ''
+ return `${name}: ${relayErrorSlug(error) ?? 'redacted'}${sqlState}`
+}
diff --git a/cloud/apps/relay/src/relay-observability.test.ts b/cloud/apps/relay/src/relay-observability.test.ts
index 8e0e209cdf86..6d73c7871e78 100644
--- a/cloud/apps/relay/src/relay-observability.test.ts
+++ b/cloud/apps/relay/src/relay-observability.test.ts
@@ -508,6 +508,8 @@ describe('relay observability', () => {
observability.recordControlClose(1006)
observability.recordControlClose(1006)
observability.recordControlClose(4402)
+ observability.recordPostProofClose('relay draining')
+ observability.recordPostProofClose('relay draining')
observability.recordSpliceClose('host-oversize-frame')
observability.recordSpliceClose('queue-limit')
observability.recordClientAcceptAbandoned('activity', 14_250.4)
@@ -518,18 +520,55 @@ describe('relay observability', () => {
expect(entries[0]).toMatchObject({
controlClosesByCodeDelta: { 1006: 2, 4402: 1 },
+ postProofClosesByReasonDelta: { 'relay draining': 2 },
spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 },
clientAcceptsAbandonedByStageDelta: { activity: 2, credential: 1 },
clientAcceptAbandonedMsMax: 14_250.4
})
expect(entries[1]).toMatchObject({
controlClosesByCodeDelta: {},
+ postProofClosesByReasonDelta: {},
spliceClosesByTriggerDelta: {},
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
})
+ it('counts the phone path, phone refusals by code and the step-6 failures per window', () => {
+ const entries: Array> = []
+ const observability = new RelayObservability(
+ { role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
+ (entry) => entries.push(entry)
+ )
+ const stageMs = { assignment: 0, credential: 170, activity: 0, attach: 300, basis: 0 }
+ observability.recordClientAcceptCompleted({ totalMs: 470, stageMs, phonePath: 'memory' })
+ observability.recordClientAcceptCompleted({ totalMs: 3_600, stageMs, phonePath: 'db' })
+ observability.recordPhoneRejection(4401)
+ observability.recordPhoneRejection(4404)
+ observability.recordPhoneRejection(4404)
+ observability.recordPhoneAttachDeadline()
+ observability.recordPhoneAcceptDatabaseError()
+ observability.recordResumeConfirmFailure('reject-revoked')
+ observability.recordResumeRenewalFailure()
+ observability.flush({ ...counts, resumeAuditDropped: 3 })
+ observability.flush(counts)
+ expect(entries[0]).toMatchObject({
+ clientAcceptsByPhonePathDelta: { memory: 1, db: 1 },
+ phoneRejectionsByCodeDelta: { '4401': 1, '4404': 2 },
+ phoneAttachDeadlinesDelta: 1,
+ phoneAcceptDatabaseErrorsDelta: 1,
+ resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1 },
+ resumeRenewalFailuresDelta: 1,
+ resumeAuditDropped: 3
+ })
+ expect(entries[1]).toMatchObject({
+ phoneRejectionsByCodeDelta: {},
+ phoneAttachDeadlinesDelta: 0,
+ phoneAcceptDatabaseErrorsDelta: 0,
+ resumeConfirmFailuresByCodeDelta: {}
+ })
+ })
+
it('summarises completed client accepts and control round trips per window', () => {
const entries: Array> = []
const observability = new RelayObservability(
@@ -572,6 +611,7 @@ describe('relay observability', () => {
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
+ expect(entries[0]).toMatchObject({ clientAcceptsByPhonePathDelta: {} })
// An empty window publishes counts only: a zero percentile point is
// indistinguishable from a real zero once Cloud Logging aggregates it.
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
diff --git a/cloud/apps/relay/src/relay-observability.ts b/cloud/apps/relay/src/relay-observability.ts
index 6f43248b4457..27ab9178dc8a 100644
--- a/cloud/apps/relay/src/relay-observability.ts
+++ b/cloud/apps/relay/src/relay-observability.ts
@@ -18,6 +18,8 @@ export type RelayRuntimeCounts = {
splices: number
pendingSplices: number
queuedBytes: number
+ // Step 6: resume audit rows a memory cell dropped since boot; the watcher pages on any.
+ resumeAuditDropped?: number
}
export function observedRelayRequests(counts: RelayRuntimeCounts): number {
@@ -101,9 +103,16 @@ export interface RelayRuntimeObserver {
fallback: boolean
}): void
recordControlClose?(code: number): void
+ recordPostProofClose?(reason: string): void
recordSpliceClose?(trigger: string): void
recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void
recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void
+ // Step 6: every phone refusal by close code, and the phone-path failures the watcher reads.
+ recordPhoneRejection?(code: number): void
+ recordPhoneAttachDeadline?(): void
+ recordPhoneAcceptDatabaseError?(): void
+ recordResumeConfirmFailure?(code: string): void
+ recordResumeRenewalFailure?(): void
recordControlRtt?(rttMs: number): void
recordHostPreAuth?(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void
}
@@ -162,6 +171,7 @@ export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [
export type RelayClientAcceptSample = {
totalMs: number
stageMs: Record
+ phonePath?: 'db' | 'memory'
}
type RelayMetricDeltas = {
@@ -200,11 +210,18 @@ type RelayMetricDeltas = {
regionFallbacks: Record
unavailableRegions: Record
controlClosesByCode: Record
+ postProofClosesByReason: Record
spliceClosesByTrigger: Record
clientAcceptsAbandonedByStage: Record
clientAcceptAbandonedMsMax: number
clientAcceptTotalsMs: number[]
clientAcceptStageSamplesMs: Record
+ clientAcceptsByPhonePath: Record
+ phoneRejectionsByCode: Record
+ phoneAttachDeadlines: number
+ phoneAcceptDatabaseErrors: number
+ resumeConfirmFailuresByCode: Record
+ resumeRenewalFailures: number
controlRttSamplesMs: number[]
controlRttObserved: number
controlRenewalLatenciesMs: number[]
@@ -288,6 +305,7 @@ const emptyDeltas = (): RelayMetricDeltas => ({
regionFallbacks: {},
unavailableRegions: {},
controlClosesByCode: {},
+ postProofClosesByReason: {},
spliceClosesByTrigger: {},
clientAcceptsAbandonedByStage: {},
clientAcceptAbandonedMsMax: 0,
@@ -299,6 +317,12 @@ const emptyDeltas = (): RelayMetricDeltas => ({
attach: [],
basis: []
},
+ clientAcceptsByPhonePath: {},
+ phoneRejectionsByCode: {},
+ phoneAttachDeadlines: 0,
+ phoneAcceptDatabaseErrors: 0,
+ resumeConfirmFailuresByCode: {},
+ resumeRenewalFailures: 0,
controlRttSamplesMs: [],
controlRttObserved: 0,
controlRenewalLatenciesMs: [],
@@ -529,6 +553,10 @@ export class RelayObservability implements RelayRuntimeObserver {
this.deltas.controlClosesByCode[key] = (this.deltas.controlClosesByCode[key] ?? 0) + 1
}
+ recordPostProofClose(reason: string): void {
+ increment(this.deltas.postProofClosesByReason, reason)
+ }
+
recordSpliceClose(trigger: string): void {
this.deltas.spliceClosesByTrigger[trigger] =
(this.deltas.spliceClosesByTrigger[trigger] ?? 0) + 1
@@ -543,12 +571,33 @@ export class RelayObservability implements RelayRuntimeObserver {
}
recordClientAcceptCompleted(sample: RelayClientAcceptSample): void {
+ if (sample.phonePath) increment(this.deltas.clientAcceptsByPhonePath, sample.phonePath)
this.deltas.clientAcceptTotalsMs.push(sample.totalMs)
for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) {
this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage])
}
}
+ recordPhoneRejection(code: number): void {
+ increment(this.deltas.phoneRejectionsByCode, String(code))
+ }
+
+ recordPhoneAttachDeadline(): void {
+ this.deltas.phoneAttachDeadlines += 1
+ }
+
+ recordPhoneAcceptDatabaseError(): void {
+ this.deltas.phoneAcceptDatabaseErrors += 1
+ }
+
+ recordResumeConfirmFailure(code: string): void {
+ increment(this.deltas.resumeConfirmFailuresByCode, code)
+ }
+
+ recordResumeRenewalFailure(): void {
+ this.deltas.resumeRenewalFailures += 1
+ }
+
recordControlRtt(rttMs: number): void {
const observedBefore = this.deltas.controlRttObserved++
keepInReservoir(
@@ -670,10 +719,19 @@ export class RelayObservability implements RelayRuntimeObserver {
regionFallbacksDelta: deltas.regionFallbacks,
unavailableRegionsDelta: deltas.unavailableRegions,
controlClosesByCodeDelta: deltas.controlClosesByCode,
+ // Closes after a host proof but before its control was wired; none is in the line above.
+ postProofClosesByReasonDelta: deltas.postProofClosesByReason,
spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger,
clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage,
clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax),
clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length,
+ clientAcceptsByPhonePathDelta: deltas.clientAcceptsByPhonePath,
+ // Phone refusals by close code; attach deadlines are the 4404s sent with the control still up.
+ phoneRejectionsByCodeDelta: deltas.phoneRejectionsByCode,
+ phoneAttachDeadlinesDelta: deltas.phoneAttachDeadlines,
+ phoneAcceptDatabaseErrorsDelta: deltas.phoneAcceptDatabaseErrors,
+ resumeConfirmFailuresByCodeDelta: deltas.resumeConfirmFailuresByCode,
+ resumeRenewalFailuresDelta: deltas.resumeRenewalFailures,
// Accepts are sparse: publishing a zero percentile for every empty window
// would pin the p50 at 0 forever and collapse the p95 at low accept rates.
...(deltas.clientAcceptTotalsMs.length === 0
diff --git a/cloud/apps/relay/src/relay-region-app.test.ts b/cloud/apps/relay/src/relay-region-app.test.ts
index 8c4e87ab8c32..0e7685b5dedb 100644
--- a/cloud/apps/relay/src/relay-region-app.test.ts
+++ b/cloud/apps/relay/src/relay-region-app.test.ts
@@ -140,6 +140,68 @@ describe('Relay region API', () => {
})
})
+ it('gives a window request the in-region reserve fallback, and never a report', async () => {
+ const assignment = {
+ userId: 'user-1',
+ relayHostId: 'abcdefghijklmnop',
+ cellId: 'asia-c1',
+ cellUrl: 'https://asia-c1.relay.example.test',
+ region: 'asia-east2',
+ assignmentEpoch: 7,
+ leaseExpiresAt: Date.now() + 300_000
+ }
+ const assign = vi.fn(async () => assignment)
+ const regionReserve = vi.fn(async () => null)
+ const plan = vi.fn(async () => ({ kind: 'database', regionReserve }))
+ const app = createRelayApp(config(), {
+ store: {} as never,
+ assignments: {
+ assign,
+ resolve: async () => assignment,
+ exchangeRegionCorrection: async () => ({ v: 1, reportStatus: 'accepted' })
+ } as never,
+ reservePlacement: { plan } as never,
+ drain: vi.fn(),
+ ready: async () => true
+ })
+ const response = await app.request(
+ '/v1/assign',
+ assignmentRequest('abcdefghijklmnop', {
+ reconnect: true,
+ preferredRegion: 'asia-east2',
+ regionCorrection: { v: 1, action: 'issue-window' }
+ })
+ )
+ expect(response.status).toBe(200)
+ expect(plan).toHaveBeenCalledWith(
+ { userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
+ { reconnect: true, region: 'asia-east2', regionCorrection: true }
+ )
+ expect(assign).toHaveBeenCalledWith(
+ { userId: 'user-1', relayHostId: 'abcdefghijklmnop' },
+ 'asia-east2',
+ 'asia-east2',
+ undefined,
+ { regionReserve, hostRequested: true }
+ )
+ plan.mockClear()
+ await app.request(
+ '/v1/assign',
+ assignmentRequest('abcdefghijklmnop', {
+ regionCorrection: {
+ v: 1,
+ action: 'report',
+ generation: 2,
+ assignmentEpoch: 7,
+ policyVersion: 1,
+ outcome: 'inconclusive',
+ reason: 'timeout'
+ }
+ })
+ )
+ expect(plan).not.toHaveBeenCalled()
+ })
+
it('returns successful placement when optional window storage is unavailable', async () => {
const app = createRelayApp(config(), {
store: {} as never,
diff --git a/cloud/apps/relay/src/relay-region-placement.test.ts b/cloud/apps/relay/src/relay-region-placement.test.ts
index 76cf5479a415..8754b15b0d5b 100644
--- a/cloud/apps/relay/src/relay-region-placement.test.ts
+++ b/cloud/apps/relay/src/relay-region-placement.test.ts
@@ -1,7 +1,8 @@
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
-import { RelayAssignmentStore } from './assignment-store.js'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { RelayAssignmentStore, type RelayAssignment } from './assignment-store.js'
import type { RelayCellConfig } from './config.js'
import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js'
+import { ReservePaceError } from './reserve-assignment.js'
const CELLS: RelayCellConfig[] = [
{
@@ -179,6 +180,68 @@ describe('Relay regional placement', () => {
).resolves.toBeUndefined()
})
+ describe('with a reserve cell in the region', () => {
+ const identity = { userId: 'spill-user', relayHostId: 'spillhost0000001' }
+ const onReserve: RelayAssignment = {
+ ...identity,
+ cellId: 'asia-reserve',
+ cellUrl: 'https://asia-reserve.relay.example.com',
+ assignmentEpoch: 1,
+ leaseExpiresAt: 1_000_000,
+ region: 'asia-east2'
+ }
+ const assignAsia = async (regionReserve?: () => Promise) =>
+ await store.assign(identity, 'asia-east2', 'asia-east2', undefined, {
+ placeFresh: async () => null,
+ regionReserve
+ })
+ const fillAsia = async () => {
+ for (const cell of CELLS.slice(1)) await store.configureCell(cell, 'migration-only')
+ }
+
+ it('books the reserve cell rather than spill out of a full region', async () => {
+ await fillAsia()
+ const place = vi.fn(async () => onReserve)
+ await expect(assignAsia(place)).resolves.toEqual(onReserve)
+ expect(place).toHaveBeenCalledOnce()
+ // The refused in-region attempt wrote nothing.
+ expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([])
+ })
+
+ it('answers a throttled reserve cell with its pace, not a cell abroad', async () => {
+ await fillAsia()
+ await expect(
+ assignAsia(async () => {
+ throw new ReservePaceError(4)
+ })
+ ).rejects.toBeInstanceOf(ReservePaceError)
+ expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([])
+ })
+
+ it('spills as before when the reserve cell has no room either', async () => {
+ await fillAsia()
+ const place = vi.fn(async () => null)
+ await expect(assignAsia(place)).resolves.toMatchObject({
+ cellId: 'us-c1',
+ region: 'us-central1'
+ })
+ expect(place).toHaveBeenCalledOnce()
+ // No in-region reserve room at plan time: today's single spill.
+ await expect(
+ store.assign({ ...identity, relayHostId: 'spillhost0000002' }, 'asia-east2', 'asia-east2')
+ ).resolves.toMatchObject({ cellId: 'us-c1' })
+ })
+
+ it('leaves a region with database room to the database lane', async () => {
+ const place = vi.fn(async () => onReserve)
+ await expect(assignAsia(place)).resolves.toMatchObject({
+ cellId: 'asia-c1',
+ region: 'asia-east2'
+ })
+ expect(place).not.toHaveBeenCalled()
+ })
+ })
+
it('expires identity-linked region preferences after 30 days', async () => {
const identity = { userId: 'expiry-user', relayHostId: 'expiryhost000001' }
await store.assign(identity, 'asia-east2')
diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts
index 7d98d8adb7b4..3824105810ac 100644
--- a/cloud/apps/relay/src/relay-server.ts
+++ b/cloud/apps/relay/src/relay-server.ts
@@ -372,6 +372,7 @@ export function createRelayServer(
mode: effectiveAdmitMode,
directorContact: () => reserveDeadMan?.contact(),
reregisterInFlight: () => options.cellFlags?.().flags.reregisterInFlight,
+ phonePath: () => options.cellFlags?.().flags.phonePath ?? 'db',
ticketEnforce: () => options.cellFlags?.().flags.ticketCheck === 'enforce',
dryRunEnabled: () => options.cellFlags?.().flags.reserveDryRun === true,
databaseShedding: () => databaseShedding(),
@@ -444,6 +445,7 @@ export function createRelayServer(
effectiveAdmitMode() === 'reserve' || sessions.reregistrationPending() > 0
? 'reserve'
: 'db',
+ cellPhonePathEffective: () => sessions.phonePathEffective(),
cellDemote: (request: DemoteRequest) => sessions.demote(request),
cellReserveCounts: () => sessions.reserveCounts(),
cellPlacementCeiling: placementCeiling
@@ -872,7 +874,12 @@ export function createRelayServer(
controls.close()
clients.close()
dataSockets.close()
+ store.resumeAudit.stop()
})
+ const resumeAuditDropped = (): number => {
+ const drops = store.resumeAudit.drops()
+ return drops.overflow + drops.aged
+ }
const runtimeCounts = () => {
const ledgerCounts = connectionLedger?.counts()
return {
@@ -880,6 +887,7 @@ export function createRelayServer(
preAuthConnections,
...sessions.runtimeCounts(),
queuedBytes: queuedBytes.current(),
+ ...(config.role === 'cell' ? { resumeAuditDropped: resumeAuditDropped() } : {}),
...(ledgerCounts
? {
inFlightConnections: ledgerCounts.inFlightConnections,
diff --git a/cloud/apps/relay/src/relay.blackbox.test.ts b/cloud/apps/relay/src/relay.blackbox.test.ts
index 84f8a87f80bc..35785ac6528a 100644
--- a/cloud/apps/relay/src/relay.blackbox.test.ts
+++ b/cloud/apps/relay/src/relay.blackbox.test.ts
@@ -1542,7 +1542,7 @@ describe('served relay URL', () => {
expect(await failed).toMatchObject({
type: 'control-error',
reqId: 'fault-install',
- code: 'injected SQL failure'
+ code: 'control_operation_failed'
})
const status = nextMessage(host.socket)
host.socket.send(
@@ -1642,7 +1642,7 @@ describe('served relay URL', () => {
expect(await directFailure).toMatchObject({
type: 'control-error',
reqId: 'fallback-install',
- code: 'injected SQL failure'
+ code: 'control_operation_failed'
})
const status = nextMessage(host.socket)
host.socket.send(
diff --git a/cloud/apps/relay/src/reserve-assignment.test.ts b/cloud/apps/relay/src/reserve-assignment.test.ts
index 38a6aa27e08b..fe424f412dbc 100644
--- a/cloud/apps/relay/src/reserve-assignment.test.ts
+++ b/cloud/apps/relay/src/reserve-assignment.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
+import type { RelayRegion } from '@orca-cloud/relay-contract'
import type { CellReserveClient } from './cell-reserve-client.js'
import {
ReserveAssignment,
@@ -13,7 +14,7 @@ import { ShadowSeatDirectory, type SeatFeedCell, type SeatFeedResponse } from '.
const HOST = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }
const US = 'us-central1' as const
-function cellList(ids: string[], region = US): SeatFeedCell[] {
+function cellList(ids: string[], region: RelayRegion = US): SeatFeedCell[] {
return ids.map((cellId) => ({
cellId,
cellUrl: `https://${cellId}.example`,
@@ -322,6 +323,92 @@ describe('reserve assignment on a director', () => {
}
})
+ describe('for a share-draw loser', () => {
+ const ASIA = 'asia-east2' as const
+ const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' }
+ const setupAsia = (reserveCell = feed('c1'), reserve?: (cellId: string) => ReserveAttempt) =>
+ setup({
+ ...(reserve ? { reserve } : {}),
+ cells: [...cellList(['c1', 'c2'], ASIA), ...cellList(['c3'])],
+ feeds: [
+ reserveCell,
+ // Database room far above the reserve cell's, so the 0.25 draw loses.
+ feed('c2', { admitMode: 'db', counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 5_000 } }),
+ feed('c3')
+ ]
+ })
+ const regionReserveOf = async (assignment: ReserveAssignment, identity = HOST) => {
+ const plan = await assignment.plan(identity, { reconnect: false, region: ASIA })
+ expect(plan).toMatchObject({ kind: 'database', epochFloor: 0 })
+ return plan.kind === 'database' ? plan.regionReserve : undefined
+ }
+
+ it('offers the reserve cell in its region only, only while it has a seat', async () => {
+ const { assignment, reserved, directory, now } = setupAsia()
+ const regionReserve = await regionReserveOf(assignment)
+ expect(reserved).toEqual([])
+ await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA })
+ expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
+ // A full reserve cell is not offered, and the US reserve cell never is.
+ const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } })
+ directory.apply('c1', full, now.value, now.value)
+ expect(await regionReserveOf(assignment, OTHER)).toBeUndefined()
+ expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
+ })
+
+ it('paces when the reserve cell has seats but no intake budget, rather than give up the region', async () => {
+ const { assignment, reserved } = setupAsia(feed('c1', { intake: { perSec: 1, burst: 20, tokens: 0 } }))
+ const regionReserve = await regionReserveOf(assignment)
+ await expect(regionReserve?.()).rejects.toBeInstanceOf(ReservePaceError)
+ expect(reserved).toEqual([])
+ })
+
+ it('gives the region up when the reserve cell refuses with a seat still on the map', async () => {
+ for (const outcome of ['full', 'unreachable']) {
+ const { assignment, reserved } = setupAsia(feed('c1'), (cellId) =>
+ cellId === 'c1' ? { outcome } : { outcome: 'ok' }
+ )
+ const regionReserve = await regionReserveOf(assignment)
+ // Null sends the host to the database lane's cross-region fallback, not a 1 s retry.
+ await expect(regionReserve?.()).resolves.toBeNull()
+ expect(reserved.map((call) => call.cellId)).toEqual(['c1'])
+ }
+ })
+ })
+
+ describe('for a region-correction request', () => {
+ const request = { reconnect: true, region: US, regionCorrection: true }
+
+ it('leaves the answer to the database lane, offering only the in-region reserve fallback', async () => {
+ const row = vi.fn(async () => null)
+ // A memory answer would skip the window exchange; the share draw is not offered either.
+ const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], row })
+ const plan = await assignment.plan(HOST, request)
+ expect(plan).toEqual({ kind: 'database', regionReserve: expect.any(Function) })
+ expect(reserved).toEqual([])
+ expect(row).not.toHaveBeenCalled()
+ await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({
+ assignmentEpoch: 8
+ })
+ })
+
+ it('keeps today’s path for a host the map never saw, without reading its row', async () => {
+ const row = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 3 }))
+ const { assignment, reserved } = setup({ row })
+ expect(await assignment.plan(HOST, request)).toEqual({ kind: 'database' })
+ expect(row).not.toHaveBeenCalled()
+ expect(reserved).toEqual([])
+ })
+
+ it('keeps today’s path while the map is incomplete, and in dry run', async () => {
+ const gated = setup({ cells: cellList(['c1', 'c2']), feeds: [feed('c1', { full: [seated(4)] })], startedAt: 100_500 })
+ expect(await gated.assignment.plan(HOST, request)).toEqual({ kind: 'database' })
+ const dryRun = setup({ mode: 'dry-run', feeds: [feed('c1', { full: [seated(4)] }), feed('c2')] })
+ expect(await dryRun.assignment.plan(HOST, request)).toEqual({ kind: 'database' })
+ expect([...gated.reserved, ...dryRun.reserved]).toEqual([])
+ })
+ })
+
it('paces with Retry-After when every reserve cell is out of budget and no database cell exists', async () => {
const { assignment } = setup({ reserve: () => ({ outcome: 'intake' }) })
await expect(assignment.plan(HOST, { reconnect: false, region: US })).rejects.toBeInstanceOf(
diff --git a/cloud/apps/relay/src/reserve-assignment.ts b/cloud/apps/relay/src/reserve-assignment.ts
index a64174571103..eb5d0bc42ae3 100644
--- a/cloud/apps/relay/src/reserve-assignment.ts
+++ b/cloud/apps/relay/src/reserve-assignment.ts
@@ -4,7 +4,7 @@ import {
RELAY_DEFAULT_REGION,
type RelayRegion
} from '@orca-cloud/relay-contract'
-import type { RelayAssignment } from './assignment-store.js'
+import type { AssignOptions, RelayAssignment } from './assignment-store.js'
import { CELL_RESERVE_TTL_MS } from './cell-reserve-contract.js'
import type { CellReserveClient } from './cell-reserve-client.js'
import { relayHostLogDigest } from './relay-host-log-digest.js'
@@ -45,11 +45,7 @@ type Identity = { userId: string; relayHostId: string }
export type ReservePlan =
| { kind: 'answer'; assignment: RelayAssignment; lane: 'sticky' | 'placement' }
| { kind: 'retry'; retryAfterSeconds: number; reason: 'map-incomplete' | 'database' | 'paced' }
- | {
- kind: 'database'
- epochFloor?: number
- placeFresh?: () => Promise
- }
+ | ({ kind: 'database' } & Pick)
// A host whose seat the ledger reconcile demoted mints above its row for this long, so the
// re-assign does not collide with the row again (the mirror refuses an equal epoch).
@@ -141,10 +137,15 @@ export class ReserveAssignment {
return this.input.directory.cellIds().some((cellId) => this.input.directory.placementModeOf(cellId) === 'reserve')
}
- async plan(identity: Identity, request: { reconnect: boolean; region: RelayRegion }): Promise {
+ async plan(
+ identity: Identity,
+ // regionCorrection: the reply carries a region window bound to the database row.
+ request: { reconnect: boolean; region: RelayRegion; regionCorrection?: boolean }
+ ): Promise {
this.flushSummary()
if (this.input.mode === 'off') return { kind: 'database' }
if (this.input.mode === 'dry-run') {
+ if (request.regionCorrection) return { kind: 'database' }
return { kind: 'database', placeFresh: async () => await this.dryRun(identity, request.region) }
}
if (!this.hasReserveCells()) return { kind: 'database' }
@@ -171,7 +172,7 @@ export class ReserveAssignment {
// above it on any director, not just the one that demoted it.
const leftWrongCell =
left?.closeCode === RELAY_CLOSE_CODE.WRONG_CELL && !(this.input.databaseBusy?.() ?? false)
- if (known.length === 0 || leftWrongCell) {
+ if ((known.length === 0 && !request.regionCorrection) || leftWrongCell) {
try {
// Only the never-seen host needs the row; for the WRONG_CELL one it is an optimisation,
// bounded so a database stall cannot hold the reconnect (the reconcile repairs a miss).
@@ -191,6 +192,13 @@ export class ReserveAssignment {
}
const epochFloor = known.length === 0 ? 0 : Math.max(...known)
const lastCellId = seats[0]?.cellId ?? left?.cellId ?? row?.cellId
+ // The database lane answers it, so the window exchange follows: no memory answer, no share
+ // draw, only the in-region fallback, once the map is complete and knows the host.
+ if (request.regionCorrection) {
+ return incomplete || known.length === 0
+ ? { kind: 'database' }
+ : { kind: 'database', regionReserve: this.regionReserve(identity, request.region, epochFloor, seats, now) }
+ }
if (incomplete) {
if (lastCellId !== undefined && directory.cellState(lastCellId)?.status === 'pending') {
this.summary.retries.mapIncomplete += 1
@@ -209,15 +217,36 @@ export class ReserveAssignment {
}
const placeFresh = async (): Promise =>
await this.placeFresh(identity, request.region, epochFloor, seats)
+ const regionReserve = this.regionReserve(identity, request.region, epochFloor, seats, now)
// Where the host was last seen decides who owns it: a database-mode cell keeps today's
// path, sticky included; only a fresh placement may be booked on a reserve-mode cell.
const lastOnReserve = lastCellId !== undefined && directory.placementModeOf(lastCellId) === 'reserve'
if (!lastOnReserve && (seats.length > 0 || left !== undefined || row !== null)) {
- return { kind: 'database', epochFloor, placeFresh }
+ return { kind: 'database', epochFloor, placeFresh, regionReserve }
}
const placed = await placeFresh()
if (placed) return { kind: 'answer', assignment: placed, lane: 'placement' }
- return { kind: 'database', epochFloor }
+ return { kind: 'database', epochFloor, regionReserve }
+ }
+
+ // For the share draw's losers: the database lane tries the region first, then this, then spills.
+ // Undefined while no in-region reserve cell has a seat, so the database lane runs as today.
+ private regionReserve(
+ identity: Identity,
+ region: RelayRegion,
+ epochFloor: number,
+ seats: readonly ShadowSeat[],
+ now: number
+ ): AssignOptions['regionReserve'] {
+ const hasRoom = this.placementCells().some(
+ (cell) => this.input.placer.isCandidate(cell, region, now) && this.input.placer.hasSeat(cell)
+ )
+ if (!hasRoom) return undefined
+ return async () => {
+ const placed = await this.placeFresh(identity, region, epochFloor, seats, true)
+ if (placed) this.summary.regionKept += 1
+ return placed
+ }
}
// The same epoch on the same cell: no booking, no database, no row lock. A cell that
@@ -288,16 +317,19 @@ export class ReserveAssignment {
identity: Identity,
region: RelayRegion,
epochFloor: number,
- seats: readonly ShadowSeat[] = []
+ seats: readonly ShadowSeat[] = [],
+ // The database lane found the region full: no share draw, and no other region.
+ regionFull = false
): Promise {
const placements = this.placementCells()
- if (this.random() >= this.input.placer.reservePoolShare(placements, region)) return null
- const regions = region === RELAY_DEFAULT_REGION ? [region] : [region, RELAY_DEFAULT_REGION]
+ if (!regionFull && this.random() >= this.input.placer.reservePoolShare(placements, region)) return null
+ const regions = regionFull || region === RELAY_DEFAULT_REGION ? [region] : [region, RELAY_DEFAULT_REGION]
for (const target of regions) {
const result = await this.input.placer.place({
cells: placements,
region: target,
epoch: mintEpoch([epochFloor]),
+ regionFull,
reserve: async (cellId, epoch) => {
const cell = this.input.cells().find((entry) => entry.cellId === cellId)
if (!cell) return { outcome: 'unreachable' }
@@ -463,6 +495,8 @@ type PlacementSummary = {
sticky: number
stickyReserves: number
placed: number
+ // Placements the database lane would have sent out of the region.
+ regionKept: number
calls: number
database: number
supersedes: number
@@ -488,6 +522,7 @@ function emptySummary(): PlacementSummary {
sticky: 0,
stickyReserves: 0,
placed: 0,
+ regionKept: 0,
calls: 0,
database: 0,
supersedes: 0,
diff --git a/cloud/apps/relay/src/reserve-placement.ts b/cloud/apps/relay/src/reserve-placement.ts
index 3e677c054f5c..2cba1baaf841 100644
--- a/cloud/apps/relay/src/reserve-placement.ts
+++ b/cloud/apps/relay/src/reserve-placement.ts
@@ -106,8 +106,12 @@ export class ReservePlacer {
// Above every epoch this director knows for the host.
epoch: number
reserve: (cellId: string, epoch: number) => Promise
+ // The database lane found the region full: a throttled reserve cell paces instead.
+ regionFull?: boolean
}): Promise {
const tried = new Set()
+ // Refused for a reason other than intake: no seat to wait for this time.
+ const refused = new Set()
let epoch = input.epoch
let calls = 0
while (calls < RESERVE_MAX_TRIES) {
@@ -120,23 +124,30 @@ export class ReservePlacer {
if (answer.outcome === 'ok') return { kind: 'placed', cellId: cell.cellId, epoch, calls }
this.uncountBooking(cell, countedAt)
if (answer.outcome === 'intake') this.drain(cell)
+ else refused.add(cell.cellId)
if (answer.outcome === 'seated-newer' && 'epoch' in answer && typeof answer.epoch === 'number') {
epoch = Math.max(epoch, answer.epoch + 1)
}
}
- return this.fallback(input.cells, input.region, calls)
+ return this.fallback(input.cells, input.region, calls, { regionFull: input.regionFull, refused })
}
- // Database cells in the region come first, then pacing; a region with neither falls to the
- // database path too, which spills to another region as today.
- fallback(cells: readonly PlacementCell[], region: RelayRegion, calls: number): PlacementResult {
+ // Database cells in the region come first (unless known full), then pacing on a cell that
+ // only lacks budget; anything else falls to the database path, which spills as today.
+ fallback(
+ cells: readonly PlacementCell[],
+ region: RelayRegion,
+ calls: number,
+ options: { regionFull?: boolean; refused?: ReadonlySet } = {}
+ ): PlacementResult {
const now = this.now()
- const databaseCells = cells.some(
- (cell) => !cell.reserve && cell.region === region && cell.general && !cell.draining
- )
+ const databaseCells =
+ !options.regionFull &&
+ cells.some((cell) => !cell.reserve && cell.region === region && cell.general && !cell.draining)
if (databaseCells) return { kind: 'database', calls }
let refillMs: number | null = null
for (const cell of cells) {
+ if (options.refused?.has(cell.cellId)) continue
if (!this.isCandidate(cell, region, now) || !this.hasSeat(cell)) continue
const estimate = this.refill(cell, now)
const rate = this.ratePerSec(cell)
diff --git a/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts b/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts
new file mode 100644
index 000000000000..a213074f03c0
--- /dev/null
+++ b/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts
@@ -0,0 +1,129 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js'
+import {
+ RESUME_AUDIT_BUFFER_MAX_ROWS,
+ RESUME_AUDIT_FLUSH_MS,
+ RESUME_AUDIT_FLUSH_ROWS,
+ RESUME_AUDIT_MAX_AGE_MS,
+ ResumeConfirmedAuditBuffer
+} from './resume-confirmed-audit-buffer.js'
+import { StatementRecordingDatabase } from './test-fixtures/statement-recording-database.js'
+
+const event = (reqId: string) => ({
+ userId: 'user-1',
+ relayHostId: 'host000000000001',
+ relayDeviceId: 'device-1',
+ detail: { reqId, basisConnId: `basis-${reqId}`, renewed: true }
+})
+
+describe('the batched resume-confirmed audit', () => {
+ let inner: RelayDatabase | undefined
+
+ afterEach(async () => {
+ await inner?.close()
+ inner = undefined
+ })
+
+ async function setup(now: () => number = Date.now) {
+ inner = await openInMemoryRelayDatabase()
+ const database = new StatementRecordingDatabase(inner)
+ const lines: Array> = []
+ const buffer = new ResumeConfirmedAuditBuffer(database, now, (line) => lines.push(JSON.parse(line)))
+ const rows = async (): Promise =>
+ await inner!.query(`SELECT type, relay_device_id, detail_json FROM relay_audit_events ORDER BY at, id`)
+ return { database, buffer, lines, rows }
+ }
+
+ it('writes buffered rows as one multi-row insert', async () => {
+ const { database, buffer, rows } = await setup()
+ for (const reqId of ['a', 'b', 'c']) buffer.add(event(reqId))
+ expect(await rows()).toEqual([])
+ await buffer.flush()
+ expect(database.statements).toEqual([
+ expect.stringMatching(/^INSERT INTO relay_audit_events .* ON CONFLICT \(id\) DO NOTHING$/)
+ ])
+ expect((await rows()).map((row) => JSON.parse(String(row.detail_json)).reqId).sort()).toEqual(['a', 'b', 'c'])
+ expect((await rows())[0]).toMatchObject({ type: 'resume-confirmed', relay_device_id: 'device-1' })
+ buffer.stop()
+ })
+
+ it('flushes at once when a batch fills, without waiting for the timer', async () => {
+ const { buffer, rows } = await setup()
+ for (let index = 0; index < RESUME_AUDIT_FLUSH_ROWS; index += 1) buffer.add(event(`r${index}`))
+ await buffer.flush()
+ expect(await rows()).toHaveLength(RESUME_AUDIT_FLUSH_ROWS)
+ buffer.stop()
+ })
+
+ it('retries a flush whose commit outcome was lost without writing any row twice', async () => {
+ const { database, buffer, rows } = await setup()
+ buffer.add(event('a'))
+ buffer.add(event('b'))
+ // The insert commits, then the reply is lost.
+ const query = database.query.bind(database)
+ let lost = true
+ database.query = async (sql, params) => {
+ const result = await query(sql, params)
+ if (lost) {
+ lost = false
+ throw new Error('Connection terminated unexpectedly')
+ }
+ return result
+ }
+ await buffer.flush()
+ expect(buffer.pending()).toBe(2)
+ await buffer.flush()
+ expect(buffer.pending()).toBe(0)
+ expect(await rows()).toHaveLength(2)
+ buffer.stop()
+ })
+
+ it('caps the buffer through an outage and drops rows too old to retry, counting both', async () => {
+ let now = 1_000_000
+ const { database, buffer, lines, rows } = await setup(() => now)
+ database.failing = () => true
+ for (let index = 0; index <= RESUME_AUDIT_BUFFER_MAX_ROWS; index += 1) buffer.add(event(`r${index}`))
+ // The batch the 500th row sent fails and comes back: rows in flight count toward the cap.
+ await buffer.flush()
+ expect(buffer.pending()).toBe(RESUME_AUDIT_BUFFER_MAX_ROWS)
+ expect(lines).toEqual([{ event: 'orca_relay_resume_audit_dropped', overflow: 1, aged: 0 }])
+ now += RESUME_AUDIT_MAX_AGE_MS
+ database.failing = null
+ await buffer.flush()
+ expect(buffer.drops()).toEqual({ overflow: 1, aged: RESUME_AUDIT_BUFFER_MAX_ROWS })
+ expect(lines.at(-1)).toEqual({
+ event: 'orca_relay_resume_audit_dropped',
+ overflow: 1,
+ aged: RESUME_AUDIT_BUFFER_MAX_ROWS
+ })
+ expect(await rows()).toEqual([])
+ // Quiet once nothing more is lost.
+ await buffer.flush()
+ expect(lines).toHaveLength(2)
+ buffer.stop()
+ })
+
+ it('sends one early batch on crossing the batch size, and leaves retries to the timer', async () => {
+ const { database, buffer } = await setup()
+ database.failing = () => true
+ for (let index = 0; index < RESUME_AUDIT_FLUSH_ROWS + 50; index += 1) buffer.add(event(`r${index}`))
+ await buffer.flush()
+ // The 50 adds past the batch size sent nothing more: no insert per confirm into a failing database.
+ expect(database.statements).toHaveLength(1)
+ buffer.stop()
+ })
+
+ it('writes nothing more once stopped', async () => {
+ vi.useFakeTimers()
+ try {
+ const { database, buffer } = await setup()
+ buffer.stop()
+ buffer.add(event('late'))
+ await vi.advanceTimersByTimeAsync(RESUME_AUDIT_FLUSH_MS * 3)
+ expect(database.statements).toEqual([])
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+})
+
diff --git a/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts b/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts
new file mode 100644
index 000000000000..f23239126fa2
--- /dev/null
+++ b/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts
@@ -0,0 +1,114 @@
+import { randomUUID } from 'node:crypto'
+import type { RelayDatabase } from './database.js'
+
+// Step 6 (decision 4): a memory cell's resume-confirmed audit rows leave in batches, off the
+// confirm's round trips. A crash or shutdown loses at most one flush interval of rows: accepted,
+// since a drain at exit would race the database pool closing.
+export const RESUME_AUDIT_FLUSH_MS = 5_000
+export const RESUME_AUDIT_FLUSH_ROWS = 500
+export const RESUME_AUDIT_BUFFER_MAX_ROWS = 5_000
+// An erasure deletes an account's audit rows by hand; a row this old is dropped rather than
+// retried, so a database outage cannot insert it after the account is gone.
+export const RESUME_AUDIT_MAX_AGE_MS = 10 * 60 * 1000
+
+export type ResumeConfirmedAudit = {
+ userId: string
+ relayHostId: string
+ relayDeviceId: string
+ detail: { reqId: string; basisConnId: string; renewed: boolean }
+}
+
+// The id is fixed when the event happens, so a retried flush after an unknown commit outcome
+// cannot write a row twice (ON CONFLICT (id) DO NOTHING).
+type BufferedRow = ResumeConfirmedAudit & { id: string; at: number }
+
+export type ResumeAuditDrops = { overflow: number; aged: number }
+
+export class ResumeConfirmedAuditBuffer {
+ private rows: BufferedRow[] = []
+ private flushing: Promise | null = null
+ private timer: ReturnType | null = null
+ private readonly dropped: ResumeAuditDrops = { overflow: 0, aged: 0 }
+ private reportedDrops = 0
+ private stopped = false
+
+ constructor(
+ private readonly database: RelayDatabase,
+ private readonly now: () => number = Date.now,
+ private readonly warn: (line: string) => void = (line) => console.warn(line)
+ ) {}
+
+ add(event: ResumeConfirmedAudit): void {
+ this.rows.push({ ...event, id: randomUUID(), at: this.now() })
+ this.trimToCap()
+ if (!this.stopped && !this.timer) {
+ this.timer = setInterval(() => void this.flush(), RESUME_AUDIT_FLUSH_MS)
+ this.timer.unref?.()
+ }
+ // Only on crossing the batch size: retries of a failing database wait for the timer.
+ if (this.rows.length === RESUME_AUDIT_FLUSH_ROWS) void this.flush()
+ }
+
+ // Rows lost since boot: the watcher pages on any.
+ drops(): ResumeAuditDrops {
+ return { ...this.dropped }
+ }
+
+ pending(): number {
+ return this.rows.length
+ }
+
+ // One flush in flight at a time; a failed batch goes back to the front with its ids.
+ async flush(): Promise {
+ if (this.flushing) return await this.flushing
+ this.flushing = this.flushOnce().finally(() => {
+ this.flushing = null
+ })
+ return await this.flushing
+ }
+
+ stop(): void {
+ this.stopped = true
+ if (this.timer) clearInterval(this.timer)
+ this.timer = null
+ }
+
+ // The oldest rows go first; rows back from a failed batch count toward the cap.
+ private trimToCap(): void {
+ const excess = this.rows.length - RESUME_AUDIT_BUFFER_MAX_ROWS
+ if (excess <= 0) return
+ this.rows.splice(0, excess)
+ this.dropped.overflow += excess
+ }
+
+ private async flushOnce(): Promise {
+ const aged = this.now() - RESUME_AUDIT_MAX_AGE_MS
+ const before = this.rows.length
+ this.rows = this.rows.filter((row) => row.at > aged)
+ this.dropped.aged += before - this.rows.length
+ const batch = this.rows.splice(0, RESUME_AUDIT_FLUSH_ROWS)
+ if (batch.length > 0) {
+ try {
+ await this.database.query(
+ `INSERT INTO relay_audit_events
+ (id, at, type, user_id, relay_host_id, relay_device_id, detail_json)
+ VALUES ${batch.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')}
+ ON CONFLICT (id) DO NOTHING`,
+ batch.flatMap((row) => [
+ row.id, row.at, 'resume-confirmed', row.userId, row.relayHostId, row.relayDeviceId,
+ JSON.stringify(row.detail)
+ ])
+ )
+ } catch {
+ this.rows.unshift(...batch)
+ this.trimToCap()
+ }
+ }
+ // At most one line per flush, carrying the totals since boot.
+ const total = this.dropped.overflow + this.dropped.aged
+ if (total > this.reportedDrops) {
+ this.reportedDrops = total
+ this.warn(JSON.stringify({ event: 'orca_relay_resume_audit_dropped', ...this.drops() }))
+ }
+ }
+}
diff --git a/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts b/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts
index 0242f499a631..265e5bfaa615 100644
--- a/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts
+++ b/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts
@@ -28,7 +28,7 @@ async function unusedPort(): Promise {
return address.port
}
-function nextMessage(socket: WebSocket): Promise> {
+export function nextMessage(socket: WebSocket): Promise> {
return new Promise((resolve, reject) => {
socket.once('message', (data: RawData) => {
try {
@@ -42,6 +42,19 @@ function nextMessage(socket: WebSocket): Promise> {
})
}
+// The next frame of this type, skipping pings and anything else the socket carries first.
+export function nextFrame(socket: WebSocket, type: string): Promise> {
+ return new Promise((resolve) => {
+ const onMessage = (data: RawData) => {
+ const frame: unknown = JSON.parse(data.toString())
+ if (!frame || typeof frame !== 'object' || !('type' in frame) || frame.type !== type) return
+ socket.off('message', onMessage)
+ resolve(Object.fromEntries(Object.entries(frame)))
+ }
+ socket.on('message', onMessage)
+ })
+}
+
export async function startReserveModeCell(input: {
database: RelayDatabase
dataDir: string
@@ -199,6 +212,64 @@ export async function startReserveModeCell(input: {
return { identity, ...reply }
}
+ // A paired phone's device row, as an install would have written it.
+ const installDevice = async (
+ relayHostId: string,
+ resumeToken: string,
+ options: { expiresAt?: number; revokedAt?: number; relayDeviceId?: string } = {}
+ ) =>
+ await input.database.query(
+ `INSERT INTO relay_devices
+ (user_id, relay_host_id, relay_device_id, current_hash, current_version,
+ current_expires_at, revoked_at, updated_at)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+ [
+ 'user-1', relayHostId, options.relayDeviceId ?? 'device-1',
+ createHash('sha256').update(resumeToken).digest('base64url'), 1,
+ options.expiresAt ?? Date.now() + 20 * 24 * 60 * 60 * 1000, options.revokedAt ?? null, Date.now()
+ ]
+ )
+
+ // A phone's dial: its first relay-hello frame and its close.
+ let phoneSource = 0
+ const dialPhone = async (relayHostId: string, credential: string) => {
+ phoneSource += 1
+ const socket = new WebSocket(`ws://127.0.0.1:${port}/v1/connect/${relayHostId}`, {
+ headers: { 'x-forwarded-for': `spoofed, 192.0.2.${phoneSource}, 35.191.0.1` },
+ perMessageDeflate: false
+ })
+ await new Promise((resolve, reject) => {
+ socket.once('open', () => resolve())
+ socket.once('error', reject)
+ })
+ input.cleanup.push(() => socket.terminate())
+ const closed = new Promise<{ code: number; reason: string }>((resolve) =>
+ socket.once('close', (code, reason) => resolve({ code, reason: reason.toString() }))
+ )
+ const hello = nextMessage(socket)
+ socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential }))
+ return { socket, hello, closed }
+ }
+
+ // The desktop's data leg for a conn-open its control received.
+ const attachData = async (connOpen: Record, generation: unknown) => {
+ const socket = new WebSocket(`ws://127.0.0.1:${port}/v1/host/data/${String(connOpen.connId)}`, {
+ perMessageDeflate: false
+ })
+ await new Promise((resolve, reject) => {
+ socket.once('open', () => resolve())
+ socket.once('error', reject)
+ })
+ input.cleanup.push(() => socket.terminate())
+ socket.send(JSON.stringify({ type: 'host-data-auth', v: 1, connTicket: connOpen.connTicket, generation }))
+ return socket
+ }
+
+ const activityLeases = async () =>
+ await input.database.query(
+ `SELECT activity_id, activity_kind FROM relay_assignment_activity_leases ORDER BY activity_id`
+ )
+
const controlLeases = async () =>
await input.database.query(
`SELECT relay_host_id, cell_id, activity_id FROM relay_assignment_activity_leases
@@ -221,6 +292,10 @@ export async function startReserveModeCell(input: {
insertRow,
bookedHost,
controlLeases,
+ activityLeases,
+ installDevice,
+ dialPhone,
+ attachData,
admitEffectiveRow,
setAdmitMode: (admitMode: 'db' | 'reserve', flags: Partial = {}) => {
applied = { generation: applied.generation + 1, flags: { ...CELL_FLAG_DEFAULTS, ...flags, admitMode } }
diff --git a/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts b/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts
new file mode 100644
index 000000000000..d195ed6c552c
--- /dev/null
+++ b/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts
@@ -0,0 +1,59 @@
+import type {
+ RelayDatabase,
+ RelayLockOptions,
+ RelayTransactionOptions,
+ SqlRow
+} from '../database.js'
+
+// Every statement a relay path issues, in order, so its round trips are counted rather than
+// described. BEGIN and COMMIT are recorded too: each is a trip to a remote database.
+export class StatementRecordingDatabase implements RelayDatabase {
+ readonly dialect: RelayDatabase['dialect']
+ // A statement this matches fails as a dropped connection would, before reaching the database.
+ failing: ((sql: string) => boolean) | null = null
+
+ constructor(
+ private readonly inner: RelayDatabase,
+ readonly statements: string[] = []
+ ) {
+ this.dialect = inner.dialect
+ }
+
+ async query(sql: string, params: unknown[] = []): Promise {
+ this.statements.push(normalized(sql))
+ if (this.failing?.(normalized(sql))) throw new Error('Connection terminated unexpectedly')
+ return await this.inner.query(sql, params)
+ }
+
+ async queryLocked(sql: string, params: unknown[] = [], options: RelayLockOptions = {}): Promise {
+ this.statements.push(`${normalized(sql)} [locked]`)
+ return await this.inner.queryLocked(sql, params, options)
+ }
+
+ async transaction(
+ operation: (transaction: RelayDatabase) => Promise,
+ options: RelayTransactionOptions = {}
+ ): Promise {
+ this.statements.push('BEGIN')
+ try {
+ return await this.inner.transaction(
+ async (transaction) => {
+ const recording = new StatementRecordingDatabase(transaction, this.statements)
+ recording.failing = this.failing
+ return await operation(recording)
+ },
+ options
+ )
+ } finally {
+ this.statements.push('COMMIT')
+ }
+ }
+
+ async close(): Promise {
+ await this.inner.close()
+ }
+}
+
+function normalized(sql: string): string {
+ return sql.replace(/\s+/g, ' ').trim()
+}
diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json
index 89d97154b0af..9067dda7dbec 100644
--- a/cloud/dev/fixtures/terraform-root-partition/families.json
+++ b/cloud/dev/fixtures/terraform-root-partition/families.json
@@ -206,6 +206,7 @@
"google_project_iam_member.github_staging_relay_capacity_mutation",
"google_project_iam_member.github_staging_relay_capacity_viewer",
"google_project_iam_member.github_staging_relay_deploy_compute_viewer",
+ "google_project_iam_member.github_staging_relay_deploy_logging_viewer",
"google_project_iam_member.github_staging_relay_power",
"google_project_iam_member.push_runtime_cloudsql_client",
"google_project_iam_member.push_runtime_fcm_admin",
diff --git a/cloud/dev/scripts/check-relay-same-cap-headroom.mjs b/cloud/dev/scripts/check-relay-same-cap-headroom.mjs
index 95fad7b8543b..6cc0bebf64ea 100644
--- a/cloud/dev/scripts/check-relay-same-cap-headroom.mjs
+++ b/cloud/dev/scripts/check-relay-same-cap-headroom.mjs
@@ -1,5 +1,6 @@
import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
+import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs'
// Drained hosts that find no free slot keep redialling and pin the drained cell (c28,
// 2026-10-01). Hosts are counted as controls, but each moved host also brings its splices;
@@ -57,6 +58,8 @@ async function adminJson(fetchImpl, url, token, body, wait) {
{ wait }
)
const json = await response.json().catch(() => ({}))
+ // null: the director refused the read because the cell is reserve.
+ if (isReserveCellRefusal(response.status, json)) return null
if (!response.ok) throw new Error(`${new URL(url).pathname} returned ${response.status}`)
return json
}
@@ -89,15 +92,18 @@ export async function checkSameCapHeadroom(config, overrides = {}) {
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const cellStatus = async (cellId) => {
- const { status } = await adminJson(
+ const answer = await adminJson(
fetchImpl,
`${config.directorOrigin}/v1/admin/cell-status`,
token,
{ v: 1, cellId },
wait
)
- if (status?.cellId !== cellId) throw new Error('director status does not match the cell')
- return status
+ if (answer === null) return null
+ if (answer.status?.cellId !== cellId) {
+ throw new Error('director status does not match the cell')
+ }
+ return answer.status
}
const runtime = await adminJson(
fetchImpl,
@@ -106,16 +112,23 @@ export async function checkSameCapHeadroom(config, overrides = {}) {
{ v: 1 },
wait
)
- if (runtime.role !== 'cell' || runtime.cellId !== config.cellId) {
+ if (runtime?.role !== 'cell' || runtime.cellId !== config.cellId) {
throw new Error('runtime status does not match the cell')
}
const targetHosts = count(runtime.runtime?.controls, 'target host controls')
- const targetRegion = (await cellStatus(config.cellId)).region ?? null
+ const targetStatus = await cellStatus(config.cellId)
+ if (targetStatus === null) throw reserveCellRollRefusal(config.cellId)
+ const targetRegion = targetStatus.region ?? null
const cells = []
for (const cellId of config.generalCells) {
if (cellId === config.cellId) continue
const status = await cellStatus(cellId)
- cells.push({ cellId, region: status.region ?? null, freeSlots: freeSlots(status) })
+ // A reserve cell's free seats are not in the database: counted as none, never guessed.
+ cells.push(
+ status === null
+ ? { cellId, region: null, freeSlots: 0, reserve: true }
+ : { cellId, region: status.region ?? null, freeSlots: freeSlots(status) }
+ )
}
const total = (filter) =>
cells.filter(filter).reduce((sum, cell) => sum + cell.freeSlots, 0)
diff --git a/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs b/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs
index 66b5eda3bacb..de6304c7037a 100644
--- a/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs
+++ b/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs
@@ -134,6 +134,46 @@ test('no general cells means no headroom', async () => {
assert.equal(result.sufficient, false)
})
+const reserveRefusal = () => Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 })
+
+test('a reserve cell that is not being rolled offers no slots and is named', async () => {
+ const base = harness({ controls: 160 })
+ const result = await run(async (url, options) =>
+ JSON.parse(options.body).cellId === 'production-gce-c29'
+ ? reserveRefusal()
+ : await base(url, options))
+ assert.equal(result.freeSlots, 200)
+ assert.equal(result.sufficient, true)
+ assert.deepEqual(result.cells.find((entry) => entry.cellId === 'production-gce-c29'), {
+ cellId: 'production-gce-c29',
+ region: null,
+ freeSlots: 0,
+ reserve: true
+ })
+})
+
+test('refuses to roll a reserve cell, naming the flip back to db', async () => {
+ const base = harness()
+ await assert.rejects(
+ run(async (url, options) =>
+ JSON.parse(options.body ?? '{}').cellId === config.cellId
+ ? reserveRefusal()
+ : await base(url, options)),
+ /production-gce-c28 is in reserve mode .*flip it to admitMode=db/
+ )
+})
+
+test('any other 404 from cell-status still fails', async () => {
+ const base = harness()
+ await assert.rejects(
+ run(async (url, options) =>
+ JSON.parse(options.body).cellId === 'production-gce-c29'
+ ? Response.json({ error: 'cell_not_found' }, { status: 404 })
+ : await base(url, options)),
+ /cell-status returned 404/
+ )
+})
+
test('rejects a runtime or director answer for another cell', async () => {
const base = harness()
await assert.rejects(
diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs
index 08a18475cabe..f9dfe213daf9 100644
--- a/cloud/dev/scripts/deploy-relay-blue-green.mjs
+++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs
@@ -30,6 +30,14 @@ export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({
})
const DIRECTOR_STARTUP_PROBE =
'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1'
+// Why: Cloud Run routes a new revision's floor instances their first request up to ~110 s after
+// they start, so a one-step move lands every assignment on the one pre-started instance.
+export const DIRECTOR_STAGED_TRAFFIC_PERCENT = 20
+// Floor instances have started up to ~220 s after the revision and taken ~110 s more to route.
+const DIRECTOR_SPREAD_TIMEOUT_MS = 6 * 60 * 1000
+const DIRECTOR_SPREAD_POLL_INTERVAL_MS = 15_000
+// A hung read must not hold the 20/80 split past the deadline.
+const DIRECTOR_SPREAD_READ_TIMEOUT_MS = 30_000
export function taggedRevisionOrigin(serviceOrigin, tag) {
const url = new URL(serviceOrigin)
@@ -461,8 +469,13 @@ export function parseArguments(argv) {
return values
}
-function commandJson(args) {
- const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] })
+function commandJson(args, { timeoutMs } = {}) {
+ const result = spawnSync('gcloud', args, {
+ encoding: 'utf8',
+ stdio: ['ignore', 'pipe', 'pipe'],
+ timeout: timeoutMs
+ })
+ if (result.error) throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.error.message}`)
if (result.status !== 0) {
throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`)
}
@@ -727,6 +740,59 @@ function cellIdentifier(sourceCellId, releaseId) {
return `${sourceCellId.slice(0, available)}--${suffix}`
}
+// Instances that served a request since `since`, read from Cloud Run's own request log.
+function servingInstanceIds(config, revision, since) {
+ const filter = [
+ 'resource.type="cloud_run_revision"',
+ `resource.labels.service_name="${config.service}"`,
+ `resource.labels.revision_name="${revision}"`,
+ `logName="projects/${config.project}/logs/run.googleapis.com%2Frequests"`,
+ `timestamp>="${since}"`
+ ].join(' AND ')
+ const entries = commandJson([
+ 'logging',
+ 'read',
+ filter,
+ '--project',
+ config.project,
+ '--limit',
+ '1000',
+ '--format=json(labels.instanceId)'
+ ], { timeoutMs: DIRECTOR_SPREAD_READ_TIMEOUT_MS })
+ return new Set(entries.map((entry) => entry.labels?.instanceId).filter(Boolean))
+}
+
+export async function waitForServingInstances(
+ config,
+ revision,
+ required,
+ since,
+ {
+ readInstances = servingInstanceIds,
+ pollIntervalMs = DIRECTOR_SPREAD_POLL_INTERVAL_MS,
+ timeoutMs = DIRECTOR_SPREAD_TIMEOUT_MS
+ } = {}
+) {
+ const deadline = Date.now() + timeoutMs
+ let instances = 0
+ for (;;) {
+ try {
+ instances = readInstances(config, revision, since).size
+ console.warn(JSON.stringify({ event: 'director_candidate_spread', revision, instances, required }))
+ if (instances >= required) return
+ } catch (error) {
+ // A throttled log read is not evidence either way; the deadline still bounds the wait.
+ console.warn(JSON.stringify({ event: 'director_candidate_spread_unread', revision, error: String(error) }))
+ }
+ if (Date.now() >= deadline) {
+ // Fewer than the floor here can mean Cloud Run packs the partial share onto fewer instances.
+ console.warn(JSON.stringify({ event: 'director_candidate_spread_timeout', revision, instances, required }))
+ throw new Error(`candidate ${revision} served from ${instances} of ${required} instances before the deadline`)
+ }
+ await new Promise((resolve) => setTimeout(resolve, pollIntervalMs))
+ }
+}
+
async function waitForHealth(origin, connectionCapacityProtocol) {
const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) })
const body = await response.json()
@@ -952,6 +1018,7 @@ export async function deployDirector(config, tag, overrides = {}) {
deleteRevision,
updateTraffic,
waitForHealth,
+ waitForServingInstances,
assertRegionalRehomeDisabled,
readReserveCells,
readDirectorPlacement,
@@ -962,7 +1029,8 @@ export async function deployDirector(config, tag, overrides = {}) {
// 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of
// restating it here; public admission is per-instance, so losing it shrinks fleet capacity.
const initialService = operations.describeService(config)
- const servingRevision = operations.describeRevision(config, activeRevision(initialService))
+ const servingRevisionName = activeRevision(initialService)
+ const servingRevision = operations.describeRevision(config, servingRevisionName)
const shapeBaseline = directorShapeBaseline(
config,
initialService,
@@ -1058,6 +1126,7 @@ export async function deployDirector(config, tag, overrides = {}) {
}
removeDirectorTrafficTags(config, operations)
let deployed
+ let staged = false
let promoted = false
try {
operations.deployCandidate(
@@ -1178,6 +1247,18 @@ export async function deployDirector(config, tag, overrides = {}) {
})
)
}
+ // A floor of one is the pre-started instance itself; nothing else needs to warm.
+ if (requiredMinimumInstances > 1) {
+ const since = new Date().toISOString()
+ staged = true
+ operations.updateTraffic(config, [`--to-tags=${tag}=${DIRECTOR_STAGED_TRAFFIC_PERCENT}`])
+ await operations.waitForServingInstances(
+ config,
+ candidate.revision,
+ requiredMinimumInstances,
+ since
+ )
+ }
operations.updateTraffic(config, [`--to-tags=${tag}=100`])
promoted = true
removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG]))
@@ -1195,6 +1276,13 @@ export async function deployDirector(config, tag, overrides = {}) {
} catch (rollbackError) {
recoveryErrors.push(rollbackError)
}
+ } else if (staged) {
+ // The serving revision still holds most traffic and its warm floor; the cold rollback has neither.
+ try {
+ operations.updateTraffic(config, [`--to-revisions=${servingRevisionName}=100`])
+ } catch (rollbackError) {
+ recoveryErrors.push(rollbackError)
+ }
}
try {
removeDirectorTrafficTags(
diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs
index 0ba40b6f97b1..fac960c12e94 100644
--- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs
+++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs
@@ -32,7 +32,8 @@ import {
taggedRevisionOrigin,
taggedTraffic,
trafficTags,
- waitForEvacuationCapacity
+ waitForEvacuationCapacity,
+ waitForServingInstances
} from './deploy-relay-blue-green.mjs'
// Terraform declares these values; audited blue/green stamps the same values without targeting
@@ -443,7 +444,8 @@ function directorHarness({
placements = {},
callerEmail,
// The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one.
- templateTransport
+ templateTransport,
+ spreadFailure
} = {}) {
const state = {
activeRevision: 'relay-00001-old',
@@ -476,6 +478,8 @@ function directorHarness({
}
const removed = []
const healthProtocols = []
+ const trafficMoves = []
+ const spreadWaits = []
let pendingCleanupFailure = cleanupFailsBeforeRemoval
const operations = {
describeService: () => ({
@@ -563,19 +567,25 @@ function directorHarness({
if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision')
return
}
- const promote = args.find((argument) => argument.startsWith('--to-tags='))
- assert.ok(promote)
- const tag = promote.slice('--to-tags='.length).split('=')[0]
- state.activeRevision = state.tags.get(tag)
+ const move = args.find((argument) => /^--to-(tags|revisions)=/.test(argument))
+ assert.ok(move)
+ trafficMoves.push(move)
+ const [target, percent] = move.slice(move.indexOf('=') + 1).split('=')
+ if (percent !== '100') return
+ state.activeRevision = move.startsWith('--to-tags=') ? state.tags.get(target) : target
},
waitForHealth: async (_origin, connectionCapacityProtocol) => {
healthProtocols.push(connectionCapacityProtocol)
},
+ waitForServingInstances: async (_config, revision, required) => {
+ spreadWaits.push({ revision, required })
+ if (spreadFailure) throw spreadFailure
+ },
readReserveCells: async (_config, origin) => reserveCells[origin],
readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported',
adminIdentityToken: () => adminToken(callerEmail)
}
- return { state, removed, healthProtocols, operations }
+ return { state, removed, healthProtocols, trafficMoves, spreadWaits, operations }
}
test('director deploy removes stale and promoted Cloud Run tags', async () => {
@@ -701,6 +711,73 @@ test('director candidate inherits the serving warm-instance floor', async () =>
assert.equal(rollback.minimum, 0)
})
+test('a multi-instance floor moves part of the traffic until every floor instance serves', async () => {
+ const harness = directorHarness({ servingMinimum: 5 })
+ await deployDirector({}, 'candidate-new', harness.operations)
+ assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=20', '--to-tags=candidate-new=100'])
+ assert.deepEqual(harness.spreadWaits, [{ revision: 'relay-00003-new', required: 5 }])
+ assert.equal(harness.state.activeRevision, 'relay-00003-new')
+})
+
+test('a floor of one moves all traffic at once', async () => {
+ const harness = directorHarness()
+ await deployDirector({}, 'candidate-new', harness.operations)
+ assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=100'])
+ assert.deepEqual(harness.spreadWaits, [])
+})
+
+test('a candidate that never spreads returns traffic to the warm serving revision', async () => {
+ const failure = new Error('candidate served from 1 of 5 instances before the deadline')
+ const harness = directorHarness({ servingMinimum: 5, spreadFailure: failure })
+ await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure)
+ assert.deepEqual(harness.trafficMoves, [
+ '--to-tags=candidate-new=20',
+ '--to-revisions=relay-00001-old=100'
+ ])
+ assert.equal(harness.state.activeRevision, 'relay-00001-old')
+ assert.equal(harness.state.tags.size, 0)
+})
+
+test('the spread wait counts distinct instances that served since the partial move', async () => {
+ const reads = [new Set(['a']), new Set(['a', 'b']), new Set(['a', 'b', 'c'])]
+ const seen = []
+ await waitForServingInstances({}, 'relay-00003-new', 3, '2026-10-20T05:25:00.000Z', {
+ readInstances: (_config, revision, since) => {
+ seen.push([revision, since])
+ return reads.shift()
+ },
+ pollIntervalMs: 0,
+ timeoutMs: 60_000
+ })
+ assert.equal(seen.length, 3)
+ assert.deepEqual(seen[0], ['relay-00003-new', '2026-10-20T05:25:00.000Z'])
+})
+
+test('the spread wait retries a failed log read', async () => {
+ let reads = 0
+ await waitForServingInstances({}, 'relay-00003-new', 2, '2026-10-20T05:25:00.000Z', {
+ readInstances: () => {
+ reads += 1
+ if (reads === 1) throw new Error('RESOURCE_EXHAUSTED')
+ return new Set(['a', 'b'])
+ },
+ pollIntervalMs: 0,
+ timeoutMs: 60_000
+ })
+ assert.equal(reads, 2)
+})
+
+test('the spread wait fails at its deadline', async () => {
+ await assert.rejects(
+ waitForServingInstances({}, 'relay-00003-new', 5, '2026-10-20T05:25:00.000Z', {
+ readInstances: () => new Set(['a']),
+ pollIntervalMs: 0,
+ timeoutMs: 0
+ }),
+ /served from 1 of 5 instances before the deadline/
+ )
+})
+
test('director deploy rejects a serving maximum above the checked budget', async () => {
const harness = directorHarness({ servingMaximum: 6 })
await assert.rejects(
diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs
index 6a0928041c99..0c324dda5ea3 100644
--- a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs
+++ b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs
@@ -6,6 +6,7 @@ import {
selectorCellState,
transitionAdmissionSelector
} from './relay-admission-selector.mjs'
+import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs'
const DEFAULT_POLL_INTERVAL_MS = 5_000
const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000
@@ -304,8 +305,9 @@ export function defaultIdentityToken(audience) {
return result.stdout.trim()
}
-async function responseJson(response, label) {
+async function responseJson(response, label, cellId) {
const body = await response.json().catch(() => ({ error: `http_${response.status}` }))
+ if (isReserveCellRefusal(response.status, body)) throw reserveCellRollRefusal(cellId)
if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`)
return body
}
@@ -349,7 +351,10 @@ export function createAdminPost(config, deps, token) {
await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1))
continue
}
- return await responseJson(response, path)
+ // The director names no cell, so a two-cell request names both.
+ const cells =
+ body?.cellId ?? [body?.sourceCellId, body?.targetCellId].filter(Boolean).join(' or ')
+ return await responseJson(response, path, cells || null)
}
throw new Error(`${path} retry attempts exhausted`)
}
diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs
index 3eccd1cf1912..3dc33d659532 100644
--- a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs
+++ b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs
@@ -62,6 +62,39 @@ test('retries evacuation-status HTTP 500 without widening other admin retries',
assert.equal(calls, 1)
})
+test('a reserve-cell refusal names the flip back to db; other 404s keep their error', async () => {
+ const post = (body) => createAdminPost({}, {
+ fetch: async () => Response.json(body, { status: 404 }),
+ emit: () => {},
+ wait: async () => {},
+ random: () => 0
+ }, 'token')
+ await assert.rejects(
+ post({ error: 'cell_in_reserve_mode' })(
+ 'https://relay.example',
+ '/v1/admin/cell-status',
+ { v: 1, cellId: 'production-gce-c7' }
+ ),
+ /production-gce-c7 is in reserve mode .*flip it to admitMode=db/
+ )
+ await assert.rejects(
+ post({ error: 'cell_not_found' })(
+ 'https://relay.example',
+ '/v1/admin/cell-status',
+ { v: 1, cellId: 'production-gce-c7' }
+ ),
+ /cell-status failed: cell_not_found/
+ )
+ await assert.rejects(
+ post({ error: 'cell_in_reserve_mode' })(
+ 'https://relay.example',
+ '/v1/admin/evacuation-status',
+ { v: 1, sourceCellId: 'production-gce-c7', targetCellId: 'production-gce-c8' }
+ ),
+ /production-gce-c7 or production-gce-c8 is in reserve mode/
+ )
+})
+
test('verifies a 1,000-cap cell against both runtime and director telemetry', () => {
const expected = deployment(
{
diff --git a/cloud/dev/scripts/load-relay-controls.mjs b/cloud/dev/scripts/load-relay-controls.mjs
index 409f83d8f6e2..8b736e2f25d0 100644
--- a/cloud/dev/scripts/load-relay-controls.mjs
+++ b/cloud/dev/scripts/load-relay-controls.mjs
@@ -32,10 +32,16 @@ import {
} from './relay-load-run-lifecycle.mjs'
import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs'
import { holdRelayLoadSplice, relayLoadSpliceReopenSummary } from './relay-load-splice-retry.mjs'
+import {
+ installRelayLoadResumeDevice,
+ relayLoadResumeSummary,
+ resumeRelayLoadPhoneOnce
+} from './relay-load-resume-phone.mjs'
import {
parseRelayLoadArguments,
relayLoadPrincipalIndex,
relayLoadReaderEvidenceError,
+ relayLoadResumePhoneIndexes,
relayLoadSpliceIndexes,
relayLoadSpliceProfile,
relayLoadSpliceStartDelayMs
@@ -119,6 +125,7 @@ function report(state, final = false) {
...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes)
),
readerClosesByCode: state.readerClosesByCode,
+ resumePhones: relayLoadResumeSummary(state.resumeAttempts),
// Every cell a control connected to, so a canary can prove its load reached the target.
assignedCellOrigins: [...state.assignedCellOrigins].sort(),
controlHeadroom: Math.max(0, state.controls - state.active.size),
@@ -251,6 +258,7 @@ const state = {
spliceRetries: 0,
spliceFailuresByReason: {},
spliceFailedAt: new Map(),
+ resumeAttempts: [],
spliceReopenGapsMs: [],
slowReaderSplicesCompleted: 0,
wedgedReaderSplicesClosed: 0,
@@ -411,6 +419,42 @@ for (let localIndex = 0; localIndex < config.controls; localIndex++) {
const offset = config.rampStartDelayMs + rampOffset
initialConnections.push(delay(offset).then(() => connect(peer)))
}
+function recordResumeAttempt(peer, attempt) {
+ state.resumeAttempts.push(attempt)
+ console.log(JSON.stringify({ event: 'relay_load_resume_attempt', index: peer.index, ...attempt }))
+}
+// A paired phone that reconnects every interval until the run stops; every attempt is a line.
+async function runResumePhone(peer, localIndex, endsAt) {
+ const relayDeviceId = `load-resume-device-${peer.index}`
+ // Spread the phones across one interval.
+ await delay(Math.floor((localIndex / Math.max(1, config.resumePhones)) * config.resumeIntervalMs))
+ let token
+ while (!state.stopping && Date.now() < endsAt) {
+ const startedAt = Date.now()
+ if (!token) {
+ try {
+ token = await installRelayLoadResumeDevice(peer, relayDeviceId)
+ } catch (error) {
+ recordResumeAttempt(peer, {
+ at: startedAt,
+ ok: false,
+ failure: 'install',
+ error: String(error.message).slice(0, 160)
+ })
+ }
+ } else {
+ recordResumeAttempt(peer, await resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, {
+ holdMs: config.resumeHoldMs,
+ stopping: () => state.stopping,
+ delay
+ }))
+ }
+ await delay(Math.min(
+ Math.max(1_000, config.resumeIntervalMs - (Date.now() - startedAt)),
+ Math.max(0, endsAt - Date.now())
+ ))
+ }
+}
const progressTimer = setInterval(() => report(state), 10_000)
progressTimer.unref()
await runRelayLoadWithShutdown(async () => {
@@ -593,7 +637,11 @@ await runRelayLoadWithShutdown(async () => {
})
)
)
- await Promise.all([...splicePromises, delay(config.durationMs)])
+ const resumeEndsAt = Date.now() + config.durationMs
+ const resumePromises = relayLoadResumePhoneIndexes(config).map((index, localIndex) =>
+ runResumePhone(peers.get(index), localIndex, resumeEndsAt)
+ )
+ await Promise.all([...splicePromises, ...resumePromises, delay(config.durationMs)])
}, async () => {
state.stopping = true
clearInterval(progressTimer)
diff --git a/cloud/dev/scripts/operate-relay-cell-flags.mjs b/cloud/dev/scripts/operate-relay-cell-flags.mjs
index 6eebd7cd2ed7..5456cc59fca6 100644
--- a/cloud/dev/scripts/operate-relay-cell-flags.mjs
+++ b/cloud/dev/scripts/operate-relay-cell-flags.mjs
@@ -43,7 +43,9 @@ export const CELL_FLAG_SPECS = {
// Absent keeps the cell's boot value.
readTimeoutMarginMs: { parse: numberIn(1_000, 60_000, true) },
// Flip-back leases in flight; absent is a third of the cell's pool.
- reregisterInFlight: { parse: numberIn(1, 16, true) }
+ reregisterInFlight: { parse: numberIn(1, 16, true) },
+ // Step 6: acts only while the cell is reserve.
+ phonePath: { parse: enumOf('db', 'memory'), default: 'db' }
}
export const RESERVE_CONFIRMATION = 'RESERVE'
@@ -136,6 +138,24 @@ export function assertReserveKeepsLocalReadiness(flags, changes) {
)
}
+// The cell ignores phonePath=memory outside reserve, so writing it there only arms a later
+// admitMode=reserve flip to turn phones over unnamed. Rolling back with admitMode=db stays allowed.
+export function assertPhonePathNeedsReserve(flags, changes) {
+ if ((flags.phonePath ?? 'db') !== 'memory') return
+ if ((flags.admitMode ?? 'db') === 'reserve') {
+ if (changes.admitMode === 'reserve' && !('phonePath' in changes)) {
+ throw new Error(
+ 'the object still says phonePath=memory: name phonePath in the same write ' +
+ '(e.g. --set admitMode=reserve,phonePath=default), so a reserve flip never turns phones over unnamed'
+ )
+ }
+ return
+ }
+ if ('phonePath' in changes) {
+ throw new Error("phonePath=memory requires admitMode=reserve on the cell, already written or in the same write")
+ }
+}
+
// `key=value,key=value`; `key=default` removes the key so the cell uses its default.
export function parseFlagChanges(text) {
const changes = {}
@@ -152,7 +172,8 @@ export function parseFlagChanges(text) {
}
const value = spec.parse(raw)
if (value === undefined) throw new Error(`switch ${key} cannot be ${JSON.stringify(raw)}`)
- changes[key] = value
+ // phonePath=db is the default, stored as no key: an image without the switch refuses the key.
+ changes[key] = key === 'phonePath' && value === 'db' ? null : value
}
if (Object.keys(changes).length === 0) throw new Error('set names no switch to change')
return changes
@@ -418,7 +439,8 @@ async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sle
if (current.generation !== request.expectedGeneration) {
throw generationConflict(current.generation, request.expectedGeneration)
}
- const object = desiredObject(request, current.object)
+ // phonePath goes too: a dead cell comes back with phones on the database path as well.
+ const object = desiredObject({ ...request, changes: { ...request.changes, phonePath: null } }, current.object)
const generation = await writeObject(fetchImpl, request, object, accessToken, audit)
await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db')
const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit }
@@ -453,6 +475,7 @@ export async function operateCellFlags(request, dependencies) {
)
}
assertReserveKeepsLocalReadiness(object.flags, request.changes)
+ assertPhonePathNeedsReserve(object.flags, request.changes)
const plan = {
event: 'orca_relay_cell_flags_plan',
mode: request.mode,
diff --git a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs
index 1b72cb6aebf7..5d30e7e9483f 100644
--- a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs
+++ b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs
@@ -36,7 +36,8 @@ const SUPPORTED = {
reserveDryRun: { type: 'boolean' },
rejectionFence: { type: 'boolean' },
readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true },
- reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }
+ reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true },
+ phonePath: { type: 'enum', values: ['db', 'memory'] }
}
function fakeGoogleAndCell({
@@ -499,6 +500,54 @@ test('a reserve cell refuses readinessLocal=false, but takes it together with th
assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: false, admitMode: 'db' })
})
+test('phonePath=memory needs reserve, and a reserve flip never turns phones over unnamed', async () => {
+ const onDatabase = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } }
+ const refused = fakeGoogleAndCell({ stored: onDatabase })
+ await assert.rejects(
+ run(parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })), refused).result,
+ /phonePath=memory requires admitMode=reserve/
+ )
+ assert.deepEqual(refused.state.writes, [])
+
+ const together = fakeGoogleAndCell({ stored: onDatabase })
+ const both = parseCellFlagsRequest(
+ values({ set: 'admitMode=reserve,phonePath=memory', 'expected-generation': '7', ...RESERVE })
+ )
+ assert.equal((await run(both, together).result).written, true)
+ assert.deepEqual(together.state.writes[0].object.flags, {
+ readinessLocal: true,
+ admitMode: 'reserve',
+ phonePath: 'memory'
+ })
+
+ const reserve = { readinessLocal: true, admitMode: 'reserve' }
+ const onReserve = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: reserve } } })
+ const alone = parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' }))
+ assert.equal((await run(alone, onReserve).result).written, true)
+
+ // Rolling back with admitMode=db leaves phonePath=memory in the object, which the cell ignores.
+ const memory = { ...reserve, phonePath: 'memory' }
+ const rollback = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } })
+ rollback.state.directorAdmitMode = 'reserve'
+ const toDatabase = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=db', 'expected-generation': '7' }))
+ assert.equal((await run(toDatabase, rollback).result).written, true)
+ assert.deepEqual(rollback.state.writes[0].object.flags, { ...memory, admitMode: 'db' })
+
+ // phonePath=db removes the key, so an image without the switch still takes later writes.
+ const off = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } })
+ assert.equal((await run(parseCellFlagsRequest(values({ set: 'phonePath=db', 'expected-generation': '7' })), off).result).written, true)
+ assert.deepEqual(off.state.writes[0].object.flags, reserve)
+
+ const lingering = { readinessLocal: true, admitMode: 'db', phonePath: 'memory' }
+ const unnamed = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: lingering } } })
+ await assert.rejects(
+ run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })), unnamed)
+ .result,
+ /still says phonePath=memory/
+ )
+ assert.deepEqual(unnamed.state.writes, [])
+})
+
test('a migration opened during the cache wait puts Postgres back to db', async () => {
const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } })
const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'readinessLocal=true,admitMode=reserve' }))
@@ -537,7 +586,7 @@ test('a flip back that outlasts its wait keeps Postgres on reserve and says how
assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667)
})
-test('break glass records db for a cell that cannot answer, typed for that cell only', async () => {
+test('break glass records db (and drops phonePath) for a cell that cannot answer, typed for that cell only', async () => {
const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' }
assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/)
assert.throws(
@@ -545,7 +594,7 @@ test('break glass records db for a cell that cannot answer, typed for that cell
/on its own/
)
const fake = fakeGoogleAndCell({
- stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true } } }
+ stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } }
})
fake.state.directorAdmitMode = 'reserve'
const inner = fake.fetchImpl
diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs
index d4338494b5f1..96c6ccf2f22d 100644
--- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs
+++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs
@@ -6,6 +6,8 @@ import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
const PRODUCTION_CELL =
/^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
+const SOURCE_TRANSIENT_RETRIES = 12
+const SOURCE_TRANSIENT_RETRY_DELAY_MS = 10_000
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
const values = {}
@@ -56,11 +58,19 @@ export async function probeRehomeTrust(config, dependencies = {}) {
},
{ wait: dependencies.wait }
)
+ const wait = dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
let response = await request()
let body = await response.json().catch(() => ({}))
- // The director wraps source HTTP failures in 409; retry only explicit transient statuses.
- if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) {
- await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000)
+ // The director wraps source HTTP failures in 409; retry only explicit transient statuses. The
+ // load balancer can answer 503 for a minute after a fresh instance is ready (c10, 2026-10-20).
+ for (
+ let retry = 0;
+ retry < SOURCE_TRANSIENT_RETRIES &&
+ response.status === 409 &&
+ /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '');
+ retry++
+ ) {
+ await wait(SOURCE_TRANSIENT_RETRY_DELAY_MS)
response = await request()
body = await response.json().catch(() => ({}))
}
diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
index f70087a4aeca..9e31864956df 100644
--- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
+++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
@@ -165,13 +165,27 @@ test('reports safe trust reasons, keeps rejection final, and redacts arbitrary e
}
})
-test('stops after the second wrapped transient failure', async () => {
+test('outlasts a load-balancer warm-up of consecutive wrapped source 503s', async () => {
+ let calls = 0
+ const waits = []
+ const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
+ wait: async (ms) => { waits.push(ms) },
+ fetch: async () => ++calls <= 6
+ ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
+ : Response.json(provenProbe)
+ })
+ assert.equal(calls, 7)
+ assert.deepEqual(waits, Array(6).fill(10_000))
+ assert.equal(result.proven, true)
+})
+
+test('stops after two minutes of wrapped transient failures', async () => {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
}), /returned 409.*source_503/)
- assert.equal(calls, 2)
+ assert.equal(calls, 13)
})
// The probe allowlist is the committed rehome source list, so a declared source can always probe.
diff --git a/cloud/dev/scripts/relay-evidence-code-provenance.mjs b/cloud/dev/scripts/relay-evidence-code-provenance.mjs
index 3e6ccb913103..dba11559e6fa 100644
--- a/cloud/dev/scripts/relay-evidence-code-provenance.mjs
+++ b/cloud/dev/scripts/relay-evidence-code-provenance.mjs
@@ -33,6 +33,8 @@ export const TRUSTED_EVIDENCE_CODE_PATHS = [
relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'),
relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'),
relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'),
+ relayTreePath('dev/scripts/relay-admin-transient-retry.mjs'),
+ relayTreePath('dev/scripts/relay-reserve-cell-refusal.mjs'),
// The exit metric filter the same-cap pre-drain sample's crash rule reads.
relayTreePath('infra/terraform/relay-observability.tf'),
// The monitor, the live preflight recheck, and the same-cap pre-drain sample, plus anything that
diff --git a/cloud/dev/scripts/relay-load-control-peer.mjs b/cloud/dev/scripts/relay-load-control-peer.mjs
index e250c0f65447..d58fc0c10b64 100644
--- a/cloud/dev/scripts/relay-load-control-peer.mjs
+++ b/cloud/dev/scripts/relay-load-control-peer.mjs
@@ -20,7 +20,7 @@ const REPORTABLE_ASSIGNMENT_ERRORS = [
'relay_home_cell_unavailable'
]
-function waitForOpen(socket, timeoutMs = 10_000) {
+export function waitForOpen(socket, timeoutMs = 10_000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs)
const finish = (error) => {
@@ -40,7 +40,7 @@ function waitForOpen(socket, timeoutMs = 10_000) {
})
}
-function nextJson(socket, timeoutMs = 10_000) {
+export function nextJson(socket, timeoutMs = 10_000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs)
const finish = (error, value) => {
@@ -66,7 +66,7 @@ function nextJson(socket, timeoutMs = 10_000) {
})
}
-function nextFrame(socket, timeoutMs = 10_000) {
+export function nextFrame(socket, timeoutMs = 10_000) {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs)
const finish = (error, value) => {
diff --git a/cloud/dev/scripts/relay-load-profile.mjs b/cloud/dev/scripts/relay-load-profile.mjs
index 076a7407b0f2..7d01a0501d9e 100644
--- a/cloud/dev/scripts/relay-load-profile.mjs
+++ b/cloud/dev/scripts/relay-load-profile.mjs
@@ -81,6 +81,9 @@ export function parseRelayLoadArguments(argv) {
values.get('--duration-seconds') ?? 900
) * 1000
const splices = integer('--splices', 0)
+ const resumePhones = integer('--resume-phones', 0)
+ const resumeIntervalMs = integer('--resume-interval-seconds', 20) * 1000
+ const resumeHoldMs = integer('--resume-hold-seconds', 10) * 1000
const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000
const slowReaderSplices = integer('--slow-reader-splices', 0)
const wedgedReaderSplices = integer('--wedged-reader-splices', 0)
@@ -112,6 +115,12 @@ export function parseRelayLoadArguments(argv) {
if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000')
if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls')
if (splices > controls) throw new Error('--splices cannot exceed --controls')
+ if (splices + resumePhones > controls) {
+ throw new Error('--splices plus --resume-phones cannot exceed --controls')
+ }
+ if (resumePhones > 0 && resumeIntervalMs < 1_000) {
+ throw new Error('--resume-interval-seconds must be at least 1')
+ }
if (shardCount > 1 && capacityHardCap !== undefined) {
if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) {
throw new Error('capacity-bound sharding requires explicit aggregate controls and splices')
@@ -320,6 +329,9 @@ export function parseRelayLoadArguments(argv) {
aggregateReaderBytes,
signingKeyFile: values.get('--signing-key-file'),
splices,
+ resumePhones,
+ resumeIntervalMs,
+ resumeHoldMs,
spliceRampMs,
splicePayloadBytes,
slowReaderSplices,
@@ -348,6 +360,14 @@ export function relayLoadSpliceIndexes(config) {
)
}
+// Resume phones ride the controls after the splice controls, so the two never share a peer.
+export function relayLoadResumePhoneIndexes(config) {
+ return Array.from(
+ { length: config.resumePhones },
+ (_, localIndex) => (config.splices + localIndex) * config.shardCount + config.shardIndex
+ )
+}
+
export function relayLoadSpliceStartDelayMs(config, localIndex) {
if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) {
throw new Error('invalid local splice index')
diff --git a/cloud/dev/scripts/relay-load-profile.test.mjs b/cloud/dev/scripts/relay-load-profile.test.mjs
index 9763d6a699f3..c5508453e765 100644
--- a/cloud/dev/scripts/relay-load-profile.test.mjs
+++ b/cloud/dev/scripts/relay-load-profile.test.mjs
@@ -4,6 +4,7 @@ import {
parseRelayLoadArguments,
relayLoadPrincipalIndex,
relayLoadReaderEvidenceError,
+ relayLoadResumePhoneIndexes,
relayLoadSpliceIndexes,
relayLoadSpliceProfile,
relayLoadSpliceStartDelayMs
@@ -214,6 +215,20 @@ test('maps splice ownership deterministically within a shard', () => {
assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10])
})
+test('puts resume phones on the controls after the splice controls', () => {
+ const config = parseRelayLoadArguments([
+ ...required, '--controls', '4', '--splices', '1', '--resume-phones', '2',
+ '--shard-count', '4', '--shard-index', '2'
+ ])
+
+ assert.deepEqual(relayLoadSpliceIndexes(config), [2])
+ assert.deepEqual(relayLoadResumePhoneIndexes(config), [6, 10])
+ assert.throws(
+ () => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '1', '--resume-phones', '2']),
+ /--splices plus --resume-phones cannot exceed --controls/
+ )
+})
+
test('staggered shards form one deterministic splice ramp', () => {
const config = parseRelayLoadArguments([
...required,
diff --git a/cloud/dev/scripts/relay-load-resume-phone.mjs b/cloud/dev/scripts/relay-load-resume-phone.mjs
new file mode 100644
index 000000000000..c010f27b9e31
--- /dev/null
+++ b/cloud/dev/scripts/relay-load-resume-phone.mjs
@@ -0,0 +1,220 @@
+import { createHash, randomBytes } from 'node:crypto'
+import { nextFrame, nextJson, waitForOpen } from './relay-load-control-peer.mjs'
+
+// A paired phone's reconnects: one device install over an invite splice, then real device
+// resumes (accept, then the desktop's device-resume-confirm), timed from the phone's side.
+
+const ECHO_INTERVAL_MS = 5_000
+
+const resumeTokenHash = (token) => createHash('sha256').update(token).digest('base64url')
+
+function controlAnswer(peer, reqId, type) {
+ return peer.waitForControlMessage(
+ (message) =>
+ message.reqId === reqId && (message.type === type || message.type === 'control-error')
+ )
+}
+
+function closeCode(socket, timeoutMs = 2_000) {
+ if (socket.readyState === socket.CLOSED) return Promise.resolve(undefined)
+ return new Promise((resolve) => {
+ const timer = setTimeout(() => resolve(undefined), timeoutMs)
+ socket.once('close', (code) => {
+ clearTimeout(timer)
+ resolve(code)
+ })
+ })
+}
+
+// The phone dials, the desktop's control answers conn-open, and the desktop attaches its data leg.
+async function dialPhone(peer, credential, relayDeviceId) {
+ if (!peer.socket || peer.socket.readyState !== peer.socket.OPEN || peer.generation === undefined) {
+ throw Object.assign(new Error('active control required for a phone'), { reason: 'no-control' })
+ }
+ const assignment = peer.lastAssignment
+ const phone = peer.createClientSocket(assignment)
+ peer.trackSpliceSocket(phone)
+ let data
+ try {
+ const setData = (socket) => {
+ data = socket
+ }
+ return await attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData })
+ } catch (error) {
+ // A close before any hello names its code, e.g. 4503 while the database is down.
+ const closed = /closed: (\d+)/.exec(String(error.message))
+ throw Object.assign(error, {
+ reason: error.reason ?? (closed ? `hello-${closed[1]}` : undefined),
+ phone,
+ data
+ })
+ }
+}
+
+async function attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData }) {
+ await waitForOpen(phone)
+ const connectionPromise = peer.waitForControlMessage(
+ (message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId
+ )
+ connectionPromise.catch(() => undefined)
+ const hello = nextJson(phone)
+ const startedAt = performance.now()
+ phone.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential }))
+ // A refused phone gets a failed hello and no conn-open.
+ const first = await Promise.race([
+ connectionPromise.then((connection) => ({ connection })),
+ hello.then((answer) => ({ answer }))
+ ])
+ if (first.answer) {
+ const code = first.answer.code ?? (await closeCode(phone))
+ throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` })
+ }
+ const { connection } = first
+ const data = peer.createHostDataSocket(assignment, connection.connId)
+ setData(data)
+ peer.trackSpliceSocket(data)
+ await waitForOpen(data)
+ data.send(
+ JSON.stringify({
+ type: 'host-data-auth',
+ v: 1,
+ connTicket: connection.connTicket,
+ generation: peer.generation
+ })
+ )
+ const answer = await hello
+ const acceptMs = performance.now() - startedAt
+ if (answer.ok !== true) {
+ const code = answer.code ?? (await closeCode(phone))
+ throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` })
+ }
+ return { phone, data, connId: connection.connId, acceptMs }
+}
+
+async function echo(phone, data, payload) {
+ const toHost = nextFrame(data)
+ phone.send(payload)
+ if ((await toHost).bytes.toString() !== payload) throw new Error('relay changed phone payload')
+ const toPhone = nextFrame(phone)
+ data.send(Buffer.from(payload))
+ if ((await toPhone).bytes.toString() !== payload) throw new Error('relay changed host payload')
+}
+
+function rttMs(socket, timeoutMs = 5_000) {
+ return new Promise((resolve) => {
+ const startedAt = performance.now()
+ const timer = setTimeout(() => finish(undefined), timeoutMs)
+ const finish = (value) => {
+ clearTimeout(timer)
+ socket.off('pong', onPong)
+ resolve(value)
+ }
+ const onPong = () => finish(performance.now() - startedAt)
+ socket.on('pong', onPong)
+ socket.ping()
+ })
+}
+
+export async function installRelayLoadResumeDevice(peer, relayDeviceId) {
+ const inviteReqId = `load-resume-invite-${peer.index}-${peer.spliceSequence++}`
+ const invite = controlAnswer(peer, inviteReqId, 'invite-created')
+ peer.socket.send(JSON.stringify({ type: 'invite-create', reqId: inviteReqId, relayDeviceId }))
+ const created = await invite
+ if (created.type !== 'invite-created') throw new Error(`invite failed: ${created.code}`)
+ const splice = await dialPhone(peer, created.inviteToken, relayDeviceId)
+ try {
+ const token = randomBytes(32).toString('base64url')
+ const reqId = `load-resume-install-${peer.index}-${peer.spliceSequence++}`
+ const installed = controlAnswer(peer, reqId, 'device-credential-installed')
+ peer.socket.send(
+ JSON.stringify({
+ type: 'device-credential-install',
+ v: 1,
+ reqId,
+ relayDeviceId,
+ newResumeTokenHash: resumeTokenHash(token),
+ authorization: { mode: 'relay-basis', basisConnId: splice.connId }
+ })
+ )
+ const result = await installed
+ if (result.type !== 'device-credential-installed') {
+ throw new Error(`device install failed: ${result.code}`)
+ }
+ return token
+ } finally {
+ await Promise.all([peer.closeSpliceSocket(splice.phone), peer.closeSpliceSocket(splice.data)])
+ }
+}
+
+// One resume: accept, confirm, then hold the splice with periodic echoes. Never throws.
+export async function resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, { holdMs, stopping, delay }) {
+ const attempt = { at: Date.now(), cellUrl: peer.lastAssignment?.cellUrl }
+ let splice
+ try {
+ splice = await dialPhone(peer, token, relayDeviceId)
+ attempt.acceptMs = Math.round(splice.acceptMs)
+ const reqId = `load-resume-confirm-${peer.index}-${peer.spliceSequence++}`
+ const confirmed = controlAnswer(peer, reqId, 'device-resume-confirmed')
+ const confirmStartedAt = performance.now()
+ peer.socket.send(
+ JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId: splice.connId })
+ )
+ const result = await confirmed
+ attempt.confirmMs = Math.round(performance.now() - confirmStartedAt)
+ if (result.type !== 'device-resume-confirmed') {
+ attempt.confirmFailure = String(result.code)
+ } else {
+ attempt.renewed = result.renewed
+ }
+ const rtt = await rttMs(splice.phone)
+ if (rtt !== undefined) attempt.rttMs = Math.round(rtt)
+ const holdUntil = Date.now() + holdMs
+ attempt.echoes = 0
+ while (!stopping() && Date.now() < holdUntil) {
+ await echo(splice.phone, splice.data, `orca-relay-resume:${peer.index}:${attempt.echoes}`)
+ attempt.echoes++
+ await delay(Math.min(ECHO_INTERVAL_MS, Math.max(0, holdUntil - Date.now())))
+ }
+ attempt.ok = attempt.confirmFailure === undefined
+ } catch (error) {
+ attempt.ok = false
+ attempt.failure = error.reason ?? (splice ? 'held-splice' : 'accept')
+ attempt.error = String(error.message).slice(0, 160)
+ if (splice) attempt.heldCloseCode = await closeCode(splice.phone, 100)
+ splice ??= { phone: error.phone, data: error.data }
+ } finally {
+ await Promise.all([peer.closeSpliceSocket(splice?.phone), peer.closeSpliceSocket(splice?.data)])
+ }
+ return attempt
+}
+
+const percentile = (values, fraction) => {
+ if (values.length === 0) return undefined
+ const sorted = [...values].sort((left, right) => left - right)
+ return sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))]
+}
+
+export function relayLoadResumeSummary(attempts) {
+ const accepts = attempts.flatMap((attempt) =>
+ attempt.acceptMs === undefined ? [] : [attempt.acceptMs]
+ )
+ const confirms = attempts.flatMap((attempt) =>
+ attempt.confirmMs === undefined || attempt.confirmFailure ? [] : [attempt.confirmMs]
+ )
+ const rtts = attempts.flatMap((attempt) => (attempt.rttMs === undefined ? [] : [attempt.rttMs]))
+ const failures = {}
+ for (const attempt of attempts) {
+ const reason = attempt.confirmFailure ? `confirm-${attempt.confirmFailure}` : attempt.failure
+ if (reason) failures[reason] = (failures[reason] ?? 0) + 1
+ }
+ return {
+ attempts: attempts.length,
+ ok: attempts.filter((attempt) => attempt.ok).length,
+ acceptMsP50: percentile(accepts, 0.5),
+ acceptMsP95: percentile(accepts, 0.95),
+ confirmMsP50: percentile(confirms, 0.5),
+ confirmMsP95: percentile(confirms, 0.95),
+ rttMsP50: percentile(rtts, 0.5),
+ failures
+ }
+}
diff --git a/cloud/dev/scripts/relay-load-resume-phone.test.mjs b/cloud/dev/scripts/relay-load-resume-phone.test.mjs
new file mode 100644
index 000000000000..26fd3a1938b0
--- /dev/null
+++ b/cloud/dev/scripts/relay-load-resume-phone.test.mjs
@@ -0,0 +1,24 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { relayLoadResumeSummary } from './relay-load-resume-phone.mjs'
+
+test('summarizes resume latency from completed steps and failures by reason', () => {
+ const summary = relayLoadResumeSummary([
+ { ok: true, acceptMs: 100, confirmMs: 20, rttMs: 10 },
+ { ok: true, acceptMs: 300, confirmMs: 40, rttMs: 10 },
+ { ok: false, acceptMs: 200, confirmMs: 900, confirmFailure: 'database_unavailable' },
+ { ok: false, failure: 'hello-4503' },
+ { ok: false, failure: 'hello-4503' }
+ ])
+
+ assert.deepEqual(summary, {
+ attempts: 5,
+ ok: 2,
+ acceptMsP50: 200,
+ acceptMsP95: 300,
+ confirmMsP50: 40,
+ confirmMsP95: 40,
+ rttMsP50: 10,
+ failures: { 'confirm-database_unavailable': 1, 'hello-4503': 2 }
+ })
+})
diff --git a/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json
new file mode 100644
index 000000000000..0593c68ab634
--- /dev/null
+++ b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json
@@ -0,0 +1 @@
+{"source":"production cell runtime metrics, read-only, 2026-10-18T06:00Z to 2026-10-19T08:00Z; per-minute sums of authFailuresDelta and clientAcceptCompletedDelta by the minute each 30 s line ends in","cells":{"production-gce-c7":{"authFailuresPerMinute":[0,1,0,1,2,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,1,1,0,2,1,1,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,2,1,0,1,1,0,2,2,1,0,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,2,1,1,0,3,2,2,1,3,5,1,0,1,2,1,2,1,2,0,5,4,1,1,4,0,1,0,1,3,2,4,2,1,3,2,5,1,3,1,2,1,1,2,2,1,1,3,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,4,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,2,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,2,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,1,0,2,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,1,0,2,2,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,2,0,1,0,1,0,0,0,0,0,1,0,3,0,1,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,4,1,1,1,3,1,1,1,0,1,1,0,1,2,2,2,0,0,1,6,0,1,1,1,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,1,0,4,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0],"completedAcceptsPerMinute":[2,0,1,2,0,2,1,5,1,1,2,3,1,4,0,0,2,4,1,0,0,0,1,1,2,2,2,1,2,2,1,1,0,0,2,0,0,2,1,1,1,2,0,0,1,2,1,0,2,1,0,1,1,0,1,0,1,0,1,0,0,0,1,1,4,0,1,0,1,0,2,0,0,0,0,0,2,2,0,1,1,1,1,1,0,0,0,1,2,1,0,0,0,1,0,1,3,0,0,3,0,0,1,1,0,2,1,1,2,1,2,0,1,1,2,1,3,2,1,1,0,1,0,1,2,4,2,3,4,0,1,1,3,2,0,1,2,1,2,2,2,3,4,2,3,1,3,1,0,2,2,0,0,3,1,1,2,1,3,2,2,2,2,3,1,3,1,3,3,1,1,2,0,0,3,0,4,2,3,3,1,2,2,4,3,3,2,3,4,4,0,4,4,3,2,1,0,2,1,2,3,2,3,3,1,4,2,2,2,3,3,1,4,2,1,4,2,3,4,2,0,1,4,2,1,0,0,2,0,1,4,2,1,4,3,3,3,0,2,3,3,3,2,3,5,2,5,3,1,3,0,5,2,1,4,1,6,1,3,7,4,4,1,4,3,1,1,2,2,3,3,4,0,3,0,2,2,3,2,1,2,3,0,2,0,2,2,1,6,2,4,1,2,4,1,1,3,1,1,3,3,5,3,1,2,2,2,3,2,2,3,2,5,6,2,3,3,2,5,3,2,1,5,3,2,3,6,8,6,1,2,2,1,5,6,8,1,2,7,6,4,3,5,4,6,2,5,5,2,3,2,1,5,1,2,3,6,4,1,4,1,1,5,2,1,3,2,4,3,2,1,1,4,4,2,3,4,2,3,3,1,2,0,4,1,4,3,2,0,4,2,2,1,2,2,1,1,6,2,4,3,1,0,1,1,4,0,1,1,4,1,4,4,1,1,3,2,2,3,3,1,0,1,2,0,2,2,1,2,1,2,2,1,7,4,1,3,0,0,2,2,1,1,1,0,3,7,2,2,0,0,3,0,2,2,1,1,1,4,2,1,3,2,4,0,0,1,5,3,4,2,5,4,2,1,1,2,7,4,6,1,4,2,1,5,0,2,5,2,0,2,3,4,4,1,0,1,0,4,1,4,3,2,2,3,3,2,3,3,3,2,0,1,3,4,3,2,5,3,1,6,2,4,2,3,2,2,2,3,1,2,2,0,6,2,0,2,4,5,1,1,0,2,5,1,2,4,3,4,2,0,2,1,3,4,3,3,3,2,3,5,2,6,2,3,1,3,4,3,1,2,1,3,5,4,0,2,1,1,0,1,1,3,3,3,2,3,2,2,0,2,2,1,4,1,2,4,2,3,1,2,2,3,1,5,0,4,6,1,0,2,5,3,3,5,0,3,1,3,3,7,3,3,1,5,4,4,6,1,6,1,5,2,1,1,1,1,1,4,2,3,5,2,2,1,4,1,1,2,2,1,2,0,0,3,4,2,0,2,4,5,3,3,1,1,1,2,3,1,2,1,4,2,1,3,0,0,2,2,0,0,0,1,1,0,0,0,0,4,4,3,1,1,1,2,1,0,1,1,1,2,4,2,3,3,1,1,2,4,0,1,1,1,2,2,4,2,2,3,2,2,3,1,1,1,5,4,3,4,1,3,3,1,1,1,0,4,3,5,3,1,4,2,3,1,2,3,5,2,3,2,2,4,1,2,5,2,1,4,1,2,3,4,3,2,3,2,2,3,1,0,0,0,1,2,2,0,3,1,2,1,0,0,4,1,0,4,0,0,1,3,0,0,1,1,1,2,0,3,2,0,2,1,3,3,0,2,0,1,2,4,4,3,1,3,5,1,3,0,1,1,3,4,1,0,3,1,3,4,1,3,2,3,3,1,3,3,3,4,7,2,4,3,4,1,2,0,2,3,1,2,3,2,3,0,2,2,2,7,2,3,1,1,3,2,1,3,3,1,1,1,2,4,1,2,6,1,0,0,3,0,4,0,0,2,2,3,3,1,2,0,3,1,1,3,1,5,1,2,4,2,1,1,3,6,5,2,0,4,2,5,0,2,1,4,2,2,3,2,1,7,3,5,0,4,3,0,2,1,3,2,2,1,1,0,2,1,4,2,2,1,1,2,4,3,1,0,1,2,1,2,0,1,5,0,3,3,2,2,1,0,2,1,5,0,5,1,3,2,3,2,1,3,2,3,0,0,2,2,1,1,2,0,0,3,3,2,2,0,5,2,5,2,2,4,1,2,5,0,3,4,2,1,5,3,1,5,3,0,1,3,4,5,2,2,2,4,1,1,1,3,0,2,1,5,1,1,1,1,1,4,3,1,1,1,1,4,3,0,2,5,2,4,3,1,0,3,2,4,5,1,4,2,2,4,2,3,3,4,3,1,3,1,1,4,1,3,1,2,0,4,4,3,3,2,1,2,1,1,2,0,3,1,2,3,2,1,1,0,1,1,0,1,2,3,1,3,3,1,0,2,4,1,2,3,2,1,1,2,2,1,0,1,1,3,0,0,0,3,3,3,1,0,4,0,3,1,3,1,1,4,2,3,0,1,2,2,1,0,0,4,1,3,0,3,2,2,2,0,1,1,1,1,2,3,2,0,1,2,0,2,2,1,0,1,0,2,3,2,2,1,3,3,0,2,3,2,2,6,3,1,1,0,2,2,1,0,2,3,1,0,0,1,2,1,1,0,0,1,0,1,0,2,2,3,1,2,0,1,0,0,3,0,0,2,1,2,0,2,0,2,1,3,0,0,2,3,0,1,0,0,0,0,2,1,0,4,2,0,1,0,3,3,2,0,1,2,2,1,1,1,0,1,2,2,1,3,2,1,3,0,1,1,2,2,0,2,0,2,2,1,1,2,1,0,0,1,1,2,1,0,2,1,3,2,0,2,4,2,1,0,2,1,1,1,0,2,2,1,2,2,3,0,1,2,2,3,3,2,1,2,2,5,0,0,3,1,3,0,1,1,3,3,2,2,1,0,2,3,4,2,4,2,1,1,2,1,0,2,1,0,1,2,1,3,1,4,2,4,1,3,3,2,1,3,0,3,1,3,0,1,3,4,1,5,0,4,2,4,6,3,1,1,1,4,2,2,5,3,0,0,2,1,1,0,1,2,3,0,3,0,1,2,0,4,1,2,2,2,4,1,1,0,4,1,1,1,3,0,1,2,3,2,2,4,1,2,1,1,2,3,2,2,2,1,0,0,0,0,3,0,1,2,0,1,2,1,2,1,0,2,4,3,4,4,3,4,3,3,3,2,0,0,0,1,3,1,2,4,0,4,4,1,1,3,1,1,1,0,1,1,2,2,3,0,1,0,3,0,0,1,2,1,2,2,4,1,1,3,1,0,0,3,1,2,1,1,2,3,2,3,0,0,7,2,2,0,3,1,0,3,1,1,1,2,1,3,0,4,1,1,0,2,4,1,2,1,0,4,2,2,0,1,1,2,1,1,0,1,1,1,3,0,1,1,1,4,2,0,2,2,2,2,4,0,2,1,1,3,0]},"production-gce-c28":{"authFailuresPerMinute":[2,0,0,0,0,0,6,4,3,3,5,3,2,5,5,3,2,1,0,1,1,0,0,1,1,3,2,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,0,1,0,0,0,1,0,0,0,1,1,0,2,0,1,0,0,0,0,1,1,3,0,1,1,2,1,1,0,0,0,1,7,3,5,4,5,6,3,0,1,3,2,2,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,2,0,0,0,0,0,3,0,0,1,1,0,0,2,0,0,1,3,1,4,5,1,5,6,3,5,2,0,0,0,0,0,8,4,4,8,8,10,7,7,4,7,0,1,1,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,1,1,5,12,8,5,4,3,5,5,3,3,0,1,3,3,0,0,1,1,2,2,1,0,0,0,1,0,2,0,1,0,1,0,0,0,0,0,0,1,1,0,0,4,2,0,2,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,1,0,0,2,5,3,0,0,0,0,1,0,0,0,1,1,2,2,2,1,0,0,0,3,0,0,1,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,1,0,1,0,2,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,1,0,1,0,0,0,0,0,0,0,0,4,3,2,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,2,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,2,8,4,3,2,1,1,0,1,0,0,2,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,7,2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,2,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,1,1,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,2,1,0,0,2,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,2,1,0,0,0,0,1,0,0,0,0,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,7,3,2,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,5,0,0,0,2,0,0,1,1,0,0,0,0,0,0,2,4,2,6,2,3,2,3,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,1,0,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,6,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1,2,0,0,1,1,1,0,2,1,0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,2,0,1,1,2,0,0,0,2,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,3,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,2,0,0,0,0,1,1,0,0,0,0,1,0,1,1,2,1,1,1,0,1,0,0,1,0,0,1,0,0,1,0,1,1,0,2,1,0,0,0,1,0,0,2,0,0,1,0,0,0,0,2,2,0,0,0,1,0,0,1,2,0,0,0,2,0,0,1,0,2,0,1,3,0,0,0,1,0,1,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,1,0,0,6,2,0,5,2,0,0,0,4,5,3,1,0,5,4,0,0,3,0,1,0,1,0,5,0,2,3,6,3,6,2,0,5,3,9,0,0,3,0,1,6,3,0,2,0,5,1,1,1,4,1,1,1,0,0,0,0,1,0,1,2,1,0,5,4,5,2,0,1,0,0,0,1,1,6,5,3,6,8,6,4,9,3,6,3,3,0,3,14,5,11,6,5,11,8,5,6,10,4,1,9,14,7,4,14,6,5,4,4,1,9,9,11,6,1,0,0,2,0,0,0,1,1,0,0,0,1,0,0,1,3,3,1,0,2,2,1,1,0,0,0,0,3,4,0,3,0,2,0,3,1,2,0,0,0,1,1,0,0,1,2,1,2,1,11,5,3,0,0,1,8,8,1,4,4,0,4,0,1,6,7,6,5,9,5,3,3,5,2,7,1,2,3,1,0,1,0,1,0,2,1,5,4,5,2,1,0,0,4,2,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0,0,5,6,0,1,5,2,1,0,0,0,1,2,2,1,0,0,0,2,5,2,0,0,0,0,0,0,0,0,1,2,0,0,0,1,1,0,0,0,2,0,0,0,1,0,4,2,0,1,0,2,1,1,9,2,0,0,0,1,1,0,0,1,3,2,4,3,6,2,2,1,1,0,0,2,1,0,3,6,0,2,1,0,0,0,2,1,2,5,1,2,0,3,0,0,1,0,0,0,1,0,2,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1],"completedAcceptsPerMinute":[17,10,8,8,13,10,8,10,16,16,31,20,15,17,14,14,17,19,22,20,20,12,15,23,12,16,11,20,21,19,23,17,19,25,18,19,13,12,16,8,11,10,15,15,19,6,9,7,7,11,14,16,15,14,9,10,11,7,10,5,12,12,11,12,6,7,10,6,10,4,10,7,10,7,7,13,14,5,11,4,9,7,9,9,10,17,11,14,16,9,15,12,12,1,8,8,9,7,10,17,14,13,7,10,16,13,8,21,18,13,14,7,8,14,10,14,12,14,16,19,16,22,21,15,19,18,16,16,9,8,14,9,10,14,10,12,14,8,7,13,9,9,10,8,9,7,14,16,15,11,17,6,8,18,6,12,10,5,11,11,8,10,8,7,13,16,8,9,13,9,15,8,18,14,13,9,9,5,7,9,3,8,8,6,7,13,15,17,12,8,9,7,10,11,15,16,12,15,16,16,14,9,6,6,16,11,10,12,16,14,18,11,11,16,11,8,8,2,12,7,10,8,8,13,3,8,8,15,11,6,9,7,11,14,7,15,7,6,6,10,15,8,10,10,10,10,8,7,5,5,11,9,18,12,10,19,16,12,14,8,9,6,10,20,11,10,12,22,11,20,10,17,6,14,19,12,14,7,18,12,9,8,8,11,10,9,7,1,6,6,11,6,6,9,8,13,5,8,16,17,10,8,12,11,8,16,8,9,12,10,7,5,6,8,13,12,8,7,13,6,5,12,11,8,10,14,6,13,11,11,7,10,8,2,2,6,4,9,9,9,7,8,13,6,6,10,6,6,6,5,2,9,7,4,9,10,5,9,7,9,7,4,6,7,13,10,8,6,10,7,8,14,7,9,6,5,7,13,9,14,9,5,12,17,13,19,21,8,15,12,10,18,13,12,11,13,9,8,9,13,8,12,7,6,8,3,4,4,4,9,5,10,8,4,12,13,19,10,8,9,13,4,5,4,4,1,8,9,2,5,7,7,12,10,6,7,4,9,5,9,8,13,11,10,6,11,10,8,6,15,14,9,20,24,17,12,10,6,10,8,10,6,8,5,4,3,12,9,3,7,9,5,5,4,11,6,8,7,6,7,13,4,5,5,9,6,8,4,14,9,11,12,9,5,10,6,6,7,9,7,8,11,5,6,6,9,5,8,11,7,3,5,6,7,7,9,5,12,8,8,7,5,8,5,5,10,4,10,6,6,6,2,3,7,5,12,4,3,7,6,5,5,7,10,2,7,8,12,5,7,11,5,6,8,6,12,10,4,10,7,9,7,12,14,11,14,7,11,4,9,5,7,5,4,6,13,9,3,5,6,6,12,3,4,5,9,11,5,5,4,9,11,16,9,9,13,7,4,5,9,4,5,5,7,4,4,4,1,3,4,11,7,4,3,4,3,5,2,4,3,4,4,3,3,5,4,5,6,7,3,3,2,5,2,6,4,4,3,2,2,4,6,4,1,6,7,2,2,3,4,4,1,2,3,3,4,3,3,6,2,2,4,3,3,1,6,3,1,3,0,7,3,3,2,1,1,1,3,4,1,0,1,1,4,1,1,2,3,4,3,2,2,3,1,4,4,2,0,2,1,0,2,3,1,3,1,1,3,5,3,3,3,2,1,2,6,1,0,1,4,0,5,6,3,2,0,5,1,4,1,0,0,1,4,2,2,4,0,4,0,1,1,1,1,0,1,2,2,2,1,0,1,2,2,5,3,0,1,3,0,3,2,2,4,1,2,1,1,1,2,1,1,2,1,2,4,1,2,0,1,1,0,1,0,1,3,1,1,1,0,2,0,1,0,1,0,0,0,0,3,0,3,3,0,2,1,0,1,0,2,4,6,1,0,3,1,0,0,2,0,2,1,1,2,2,1,0,0,4,1,0,4,2,2,3,1,0,3,1,2,4,2,3,3,1,1,3,5,0,2,3,4,2,0,0,1,1,0,2,1,2,1,0,1,0,1,0,1,1,0,0,2,1,2,2,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,3,0,1,4,0,1,2,0,0,2,0,0,0,1,0,0,0,1,1,0,1,5,1,0,0,0,0,1,2,1,1,1,1,1,1,0,1,1,3,2,1,1,2,0,2,0,3,0,2,1,0,1,1,5,4,3,5,1,0,2,4,1,1,3,5,0,2,0,2,0,2,3,1,1,3,2,2,3,4,3,2,1,4,2,1,2,2,2,2,1,4,1,3,4,2,2,4,2,3,3,3,0,3,2,2,3,3,6,5,7,8,3,2,3,3,2,2,4,2,4,5,1,1,2,2,4,3,4,2,2,4,6,7,5,4,2,3,2,3,6,4,5,7,5,2,9,4,5,7,7,6,5,8,9,7,6,6,6,5,9,9,7,8,4,6,7,4,2,2,5,5,9,9,4,4,3,6,2,8,4,6,8,3,2,3,6,6,11,6,8,1,5,2,3,11,7,0,6,5,3,5,8,6,6,8,6,5,5,9,11,6,7,6,5,5,5,6,7,6,2,5,7,6,7,5,9,2,6,19,4,2,4,6,7,8,6,11,5,11,7,5,7,10,7,7,1,10,10,5,7,12,8,9,12,11,9,7,11,8,8,5,12,10,8,5,8,5,11,3,7,7,5,5,5,7,3,8,5,7,11,9,9,5,8,10,8,11,20,16,16,20,15,14,13,16,13,14,15,12,16,13,13,12,8,6,6,6,7,6,8,6,12,11,5,5,7,10,3,5,9,8,4,8,5,9,4,11,7,10,11,12,7,7,14,9,14,8,6,10,7,7,10,6,11,11,6,5,11,5,9,11,10,9,8,10,10,10,7,8,10,11,10,10,7,12,9,13,11,12,9,13,8,16,10,10,5,5,7,13,10,7,10,6,10,3,9,9,14,6,9,11,4,14,12,11,8,7,15,6,16,12,9,13,14,10,21,13,8,16,11,10,13,4,8,12,7,16,14,9,12,11,10,7,13,13,11,11,13,6,7,9,13,8,9,7,11,12,7,9,9,10,13,9,12,6,8,8,8,12,8,12,2,6,1,10,9,13,13,10,2,11,12,11,7,13,10,3,8,11,7,12,8,10,7,16,17,11,7,14,7,5,8,3,8,7,11,8,6,9,11,11,7,7,5,10,9,14,4,8,10,8,17,9,6,6,5,9,11,5,10,11,10,11,10,12,16,8,8,2,9,4,9,6,13,6,7,4,11,10,4,9,9,11,6,13,10,6,10,12,12,13,16,6,8,5,7,6,4,7,13,5,10,6,5,5,9,7,6,7,11,10,5,6,7,9,3,8,8,14,7,10,11,5,8,10,14,15,16,17,9,9,9,14,14,20,13,14,8,9,5,8,14,13,19,24,13,14,15,14,11,14,13,15,14,7,7,10,9,8,17,16,14,20,12,20,14,15,15,9,5,10,5,10,11,7,9,8,4,5,6,5,9,11,5,6,2,5,6,8,13,4,9,4,8,4,4,8,5,10,9,9,5,14,14,11,11,8,8,4,12,10,13,13,9,10,10,9,5,4,7]},"production-gce-c29":{"authFailuresPerMinute":[0,1,0,1,0,0,4,3,2,5,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,9,6,0,0,4,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,3,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,4,0,0,4,5,0,0,0,0,1,5,1,5,2,4,4,5,2,3,1,1,6,3,4,4,1,0,0,0,0,3,7,1,0,0,0,0,0,6,1,1,7,0,6,5,2,0,0,1,0,0,0,0,0,0,3,4,0,1,4,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,1,1,0,2,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,3,0,1,0,0,0,2,0,0,3,4,5,3,1,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,4,3,0,0,2,2,0,0,2,2,0,0,0,1,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,2,0,2,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,2,0,0,0,0,1,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,4,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,2,0,6,4,2,0,0,1,0,2,1,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,1,4,1,0,0,3,1,0,1,0,0,0,6,2,6,5,8,4,0,0,0,6,4,8,4,3,0,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,2,3,0,0,0,0,0,0,2,0,1,0,1,0,1,0,0,1,0,2,0,0,2,0,0,1,0,0,0,0,0,0,2,6,3,7,0,1,0,0,0,1,0,2,2,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,6,4,0,0,0,0,0,3,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,2,0,0,0,2,0,0,1,1,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,2,1,1,3,0,2,0,0,0,1,2,0,0,0,1,0,0,1,2,0,0,0,0,3,0,0,0,0,3,0,0,0,2,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,3,2,0,0,2,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,2,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,0,0,1,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,2,1,0,0,0,0,1,0,0,1,0,0,2,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,2,0,0,0,2,0,1,0,1,0,0,1,0,0,0,2,0,0,0],"completedAcceptsPerMinute":[3,7,7,8,8,7,4,8,12,4,9,8,10,8,6,5,11,8,10,12,6,7,11,7,12,7,6,11,10,7,9,8,3,10,10,9,6,4,11,9,9,10,16,4,11,11,7,7,9,8,7,11,10,9,5,6,6,7,12,12,7,6,12,6,11,18,3,9,7,8,8,4,10,5,13,10,8,12,8,13,14,3,7,5,7,8,8,4,4,8,11,3,9,9,8,3,7,4,3,7,5,5,9,8,6,7,6,3,9,9,6,10,5,7,15,5,9,11,10,14,2,7,8,10,12,7,19,8,9,9,5,7,5,5,8,11,9,10,8,5,4,11,7,8,13,10,15,12,5,11,15,9,12,8,9,3,11,9,11,9,7,15,11,8,5,8,6,8,8,1,7,5,10,9,7,7,7,9,11,7,13,10,7,9,9,4,6,7,9,7,7,5,10,13,8,6,4,11,5,9,9,7,2,9,1,8,5,8,10,12,10,12,4,8,4,8,7,9,7,3,4,4,3,9,3,8,12,5,6,12,15,8,16,14,9,9,12,5,15,7,8,6,7,11,9,12,3,6,6,7,6,7,3,2,7,10,9,16,2,14,21,11,15,12,10,6,13,1,12,7,15,5,7,11,15,6,2,10,9,10,15,13,14,12,17,12,11,13,8,4,14,10,7,3,4,6,3,5,9,8,10,10,3,7,8,6,8,6,8,12,7,4,9,7,9,5,8,6,10,5,3,6,6,5,3,8,12,5,8,7,12,4,14,13,12,9,14,16,13,14,11,7,8,4,3,8,9,3,5,5,12,7,8,6,11,10,12,13,7,14,17,16,13,15,17,9,14,7,6,9,9,9,11,8,8,7,8,2,12,7,10,6,7,5,5,7,11,7,11,7,3,5,4,9,8,7,9,5,7,8,7,9,5,9,16,15,23,20,14,11,10,5,4,13,14,13,10,16,15,20,16,21,16,17,20,15,16,21,19,13,12,9,11,10,7,12,12,4,9,7,4,14,9,14,15,16,14,9,7,3,6,8,9,6,12,7,12,5,12,10,13,6,10,8,10,6,5,4,8,6,13,6,10,9,6,8,7,7,12,6,5,6,10,12,15,14,9,12,13,17,13,4,3,4,6,8,6,8,9,8,13,3,2,3,6,6,10,6,3,10,4,8,13,5,2,4,2,12,5,6,5,9,8,7,5,7,4,4,5,9,12,14,15,9,8,2,4,6,6,2,6,4,6,1,6,5,4,3,3,6,7,1,3,2,6,1,5,9,5,3,1,4,4,5,6,11,5,4,4,4,4,4,6,4,3,6,5,6,4,4,2,7,6,2,7,4,7,6,4,3,3,4,8,5,0,3,5,4,1,5,1,2,2,5,1,8,3,2,6,1,2,3,5,3,2,4,6,5,3,2,3,2,1,3,2,5,4,3,1,3,2,5,4,2,5,1,2,3,3,2,0,3,0,1,3,5,3,1,4,1,6,3,4,2,2,4,2,2,4,2,3,2,5,4,3,2,2,1,6,10,9,10,7,4,1,2,6,8,18,14,15,16,15,19,15,17,13,18,14,8,3,1,1,0,5,1,3,1,0,0,1,0,3,2,0,1,1,6,6,3,3,3,1,2,2,2,4,2,0,1,4,2,0,1,1,0,1,1,2,1,4,1,4,1,6,3,1,1,2,3,1,0,1,2,0,0,1,0,4,2,1,2,1,1,1,1,3,2,0,1,4,0,0,5,0,1,1,2,1,1,3,2,2,3,3,3,2,0,3,2,1,3,0,2,4,3,3,5,5,3,1,0,2,1,2,0,0,1,4,2,0,1,2,2,2,1,1,1,1,2,2,4,2,3,1,5,1,2,3,3,1,0,1,1,1,0,0,2,2,0,1,3,3,4,2,0,0,2,1,2,2,0,1,1,0,0,0,0,1,1,1,2,0,2,0,2,0,1,0,1,3,1,2,1,0,0,2,1,0,1,1,2,0,1,1,1,0,0,0,1,2,1,2,2,0,0,1,1,1,4,0,1,0,1,5,2,0,2,0,1,1,2,3,1,1,1,2,2,2,3,2,0,2,1,1,0,0,2,4,2,3,0,0,0,0,2,8,2,1,1,1,1,1,1,2,0,2,0,1,1,0,0,0,1,2,1,3,2,3,1,1,3,4,5,0,3,1,2,0,3,2,4,1,3,3,3,2,4,1,5,5,6,1,3,1,2,2,0,4,13,5,8,3,2,2,1,4,2,3,2,4,0,4,5,2,7,5,0,6,5,6,4,3,8,7,4,2,5,3,12,11,12,7,5,11,17,15,12,11,7,6,4,3,8,3,3,4,2,2,2,5,8,4,7,5,4,1,5,3,2,6,2,5,7,16,3,5,4,11,6,12,7,8,6,8,3,7,2,3,6,8,2,5,11,6,6,7,7,4,7,7,11,4,5,3,7,7,0,3,8,5,9,11,4,7,4,15,5,4,2,6,5,7,4,5,10,3,5,10,6,10,6,8,5,8,6,10,9,12,5,7,12,6,8,5,7,9,8,6,6,15,12,17,13,8,14,5,5,9,7,4,8,9,7,10,6,8,7,10,9,10,6,8,10,4,3,9,4,11,4,5,6,9,9,10,8,6,8,5,8,4,4,5,11,3,2,5,3,4,3,5,7,4,8,1,5,6,7,6,14,11,9,11,7,6,8,12,4,10,6,9,13,12,11,2,7,11,7,10,7,8,9,8,9,8,12,18,12,9,14,13,14,16,12,13,8,8,14,14,3,12,5,15,9,12,9,6,8,15,11,9,5,8,11,12,8,8,6,4,10,6,7,17,12,8,9,11,11,8,10,11,6,13,9,11,15,11,12,6,8,9,11,14,7,10,7,10,6,7,13,5,10,14,16,15,20,9,7,7,10,9,13,10,6,10,10,15,16,18,16,12,6,7,10,9,6,7,15,10,6,13,14,25,10,10,8,9,10,8,7,12,13,6,5,6,8,7,12,8,12,11,14,6,10,10,13,11,14,14,9,16,14,10,5,6,8,6,12,8,8,7,8,9,7,7,14,7,7,8,8,4,7,5,6,8,9,24,10,12,14,10,12,9,6,10,14,6,13,7,16,8,9,13,10,15,11,14,14,11,10,16,9,12,9,9,11,6,10,13,4,13,11,10,6,9,11,10,6,7,8,9,7,7,6,7,7,8,10,12,6,12,10,10,6,7,9,11,5,3,6,6,6,10,9,8,9,7,7,14,5,7,7,3,4,11,8,5,10,3,6,12,4,6,4,9,7,7,4,13,6,7,10,7,9,5,8,2,9,8,9,7,6,6,10,10,13,15,8,7,7,11,4,6,9,4,17,12,17,15,12,13,5,13,12,6,18,10,12,8,8,3,6,12,7,4,9,6,8,5,5,18,9,13,15,8,6,7,6,6,8,9,9,11,6,7,3,8,5,4,2,2,9,7,4,6,8,6,12,3,7,11,7,4,10,11,8,4,9,12,15,10,8,8,5,6,7,5,3,7,6,3,3,10,8,3,7,12,9,6,4]}}}
diff --git a/cloud/dev/scripts/relay-reserve-cell-refusal.mjs b/cloud/dev/scripts/relay-reserve-cell-refusal.mjs
new file mode 100644
index 000000000000..b91dcb5203d0
--- /dev/null
+++ b/cloud/dev/scripts/relay-reserve-cell-refusal.mjs
@@ -0,0 +1,16 @@
+// Step 5: the director refuses per-cell admin reads for a reserve cell with exactly this body;
+// any other 404 stays a failure.
+export const RESERVE_CELL_REFUSAL_ERROR = 'cell_in_reserve_mode'
+
+export function isReserveCellRefusal(status, body) {
+ return status === 404 && body?.error === RESERVE_CELL_REFUSAL_ERROR
+}
+
+// Step-5 design default 4: a reserve cell is flipped back to db before a roll.
+export function reserveCellRollRefusal(cellId) {
+ return new Error(
+ `${cellId ?? 'a cell this step reads'} is in reserve mode ` +
+ `(${RESERVE_CELL_REFUSAL_ERROR}): flip it to admitMode=db with the cell-flags ` +
+ 'workflow and wait for it to record db before rolling it, then flip it back to reserve'
+ )
+}
diff --git a/cloud/dev/scripts/relay-reserve-watch-rules.mjs b/cloud/dev/scripts/relay-reserve-watch-rules.mjs
index 7fd9aae89f25..9da9d1ce40b5 100644
--- a/cloud/dev/scripts/relay-reserve-watch-rules.mjs
+++ b/cloud/dev/scripts/relay-reserve-watch-rules.mjs
@@ -6,9 +6,9 @@ import { backgroundOf, SHADOW_GATE_THRESHOLDS } from './relay-same-cap-shadow-ga
const MINUTE_MS = 60_000
export const RESERVE_WATCH_RULES = {
- // The roll ladder's warn line: max(1.5x, background + 20), so a zero background is not tripped
- // by one 503. Three complete minutes in a row, not two. The read spans more minutes than that so
- // a late publishing front still leaves three published minutes to judge.
+ // The roll ladder's warn line on non-drain 503s: max(1.5x, background + 20), so a zero background
+ // is not tripped by one 503. Three complete minutes in a row, not two. The read spans more minutes
+ // than that so a late publishing front still leaves three published minutes to judge.
director503: {
multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMultiple,
marginPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMarginPerMinute,
@@ -33,6 +33,80 @@ export const RESERVE_WATCH_RULES = {
// sample line's `rules` keys follow this order too.
export const RULE_NAMES = ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift']
+// Step 6 design §7: judged only while the cell's switch file says phonePath=memory, against the
+// cell's own runtime metrics for the 10 minutes before phonePath went memory. A trip turns the
+// phone path off (phonePath=db) and the watch goes on with the reserve rules.
+export const PHONE_WATCH_RULES = {
+ // Each rule trips on three complete minutes in a row over its line.
+ consecutiveMinutes: 3,
+ readMinutes: 10,
+ // Judged as a share of the minute's phone attempts (the signal plus completed accepts; for
+ // confirm failures, the accepts), not a count: a count line false-tripped 33/121 5-hour watches
+ // on busy c28 (10-18/19), where bursts of refusals arrive with the evening rise in accepts. A
+ // minute with fewer attempts is not judged, so low-volume c7 (~2 accepts/min) is mostly unjudged.
+ minAttemptsPerMinute: 4,
+ // Over half of the minute's attempts, and 0.25 above the share before the flip, capped so two
+ // failures per accept always trips. Exactly half is not enough: c28 (04:08Z) and c7 (09:13Z)
+ // both show 3 minutes at 50% of host-and-phone auth failures.
+ shareFloor: 0.5,
+ shareOverBackground: 0.25,
+ shareCeiling: 0.6,
+ // Page only: confirms still work, only their audit rows or renewals are lost.
+ pageAfterMs: 5 * MINUTE_MS
+}
+export const PHONE_RULE_NAMES = [
+ 'phoneCredentialRefusals',
+ 'phoneConfirmFailures',
+ 'phoneAttachDeadlines',
+ 'phoneAcceptDatabaseErrors'
+]
+const PHONE_RULE_SIGNALS = {
+ phoneCredentialRefusals: 'credentialRefusals',
+ phoneConfirmFailures: 'confirmFailures',
+ phoneAttachDeadlines: 'attachDeadlines',
+ phoneAcceptDatabaseErrors: 'acceptDatabaseErrors'
+}
+
+const sum = (values) => values.reduce((total, value) => total + value, 0)
+
+// The share line for a signal whose share of attempts before the flip was `backgroundShare`.
+export function phoneShareLine(backgroundShare) {
+ const { shareFloor, shareOverBackground, shareCeiling } = PHONE_WATCH_RULES
+ return Math.min(shareCeiling, Math.max(shareFloor, backgroundShare + shareOverBackground))
+}
+
+// A completed accept makes one confirm, so confirm failures are a share of accepts (capped at
+// all of them); every other signal is an attempt that did not complete.
+function attemptsAndShare(signal, count, completedAccepts) {
+ if (signal === 'confirmFailures') {
+ return { attempts: completedAccepts, share: completedAccepts === 0 ? 0 : Math.min(count, completedAccepts) / completedAccepts }
+ }
+ const attempts = count + completedAccepts
+ return { attempts, share: attempts === 0 ? 0 : count / attempts }
+}
+
+// Over the line: enough attempts that minute, and strictly more than the line refused or failed.
+export function phoneMinuteOver(signal, count, completedAccepts, line) {
+ const { attempts, share } = attemptsAndShare(signal, count, completedAccepts)
+ return attempts >= PHONE_WATCH_RULES.minAttemptsPerMinute && share > line
+}
+
+// The 10 minutes before phonePath went memory, from `readCellPhoneMinutes`: each signal's share of
+// attempts. Null until every minute is metered.
+export function measuredPhoneBackground(read) {
+ if (!read.ok || read.unmetered.length > 0 || read.minutes.length === 0) return null
+ const accepted = sum(read.series.completedAccepts)
+ return {
+ share: Object.fromEntries(
+ Object.values(PHONE_RULE_SIGNALS).map((signal) => {
+ const count = sum(read.series[signal])
+ return [signal, attemptsAndShare(signal, count, accepted).share]
+ })
+ ),
+ minutes: read.minutes.length
+ }
+}
+
export function initialWatchState({ cellId, startedAt, durationMinutes, background503PerMinute, flippedAt }) {
return {
v: 1,
@@ -50,16 +124,34 @@ export function initialWatchState({ cellId, startedAt, durationMinutes, backgrou
hosts: [],
reserveSeenAt: null,
ledgerAboveSince: null,
- seatBehindRowFirstSeen: {}
+ seatBehindRowFirstSeen: {},
+ // Step 6: what a trip writes back, and the phone rules' state. `phoneSince` is when the switch
+ // file went phonePath=memory (flippedAt if it already said so); `phoneOffAt` when the watch
+ // turned it off; `phoneTrip` a phone verdict the segment has yet to write.
+ flip: null,
+ lastSampleAt: null,
+ phoneSince: null,
+ phoneBackground: null,
+ phoneOffAt: null,
+ phoneTrip: null,
+ phoneOffFailures: 0,
+ phoneUnverifiableStreak: 0,
+ auditDropsLast: null,
+ auditDropsSince: null,
+ renewalFailuresSince: null,
+ paged: []
}
}
-// The background from the per-minute 503 counts before the flip. `publishedThrough` is the newest
-// minute any response code has a point for: a 503 minute at or before it with no point had none,
-// and the background waits (null) until it covers the last minute.
-export function measuredBackground(perMinute, minutes, publishedThrough) {
+// The background from the per-minute non-drain 503s before the flip, the live samples' measure.
+// `publishedThrough` is the newest minute any response code has a point for: a 503 minute at or
+// before it with no point had none, and the background waits (null) until it covers the last
+// minute. Unmetered minutes are left out; under half the minutes metered is no background.
+export function measuredBackground({ perMinute, minutes, publishedThrough, unmetered }) {
if (minutes.length === 0 || publishedThrough === null || publishedThrough < minutes.at(-1)) return null
- const background = backgroundOf({ minutes, series: minutes.map((minute) => perMinute[minute] ?? 0), unverified: false })
+ const metered = minutes.filter((minute) => !unmetered.includes(minute))
+ if (metered.length * 2 < minutes.length) return null
+ const background = backgroundOf({ minutes: metered, series: metered.map((minute) => perMinute[minute] ?? 0), unverified: false })
return { medianPerMinute: background.medianPerMinute, minutes: background.minutes, source: 'measured' }
}
@@ -68,6 +160,8 @@ const trip = (reason, detail = {}) => ({ status: 'trip', reason, ...detail })
const unverifiable = (reason) => ({ status: 'unverifiable', reason })
// Not readable yet by design, so it does not count toward the unverifiable streak.
const pending = (reason) => ({ status: 'pending', reason })
+const failedPhoneRead = { ok: false, error: 'no cell phone metrics read' }
+const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS
function judgeDeadMan(cell) {
if (!cell.ok) return unverifiable(cell.error)
@@ -96,8 +190,8 @@ function judgeOverrun(cell) {
return pass({ enforcedUnits: cell.enforcedUnits, hardCap: cell.hardCap })
}
-// Fleet-wide director 503s: no existing signal attributes a 503 to the cell a host was bound for,
-// so a problem local to one cell can stay under the fleet's line. This proxy can miss.
+// Fleet-wide non-drain director 503s: no existing signal attributes a 503 to the cell a host was
+// bound for, so a problem local to one cell can stay under the fleet's line. This proxy can miss.
function judgeDirector503(read, state, at) {
const { multiple, marginPerMinute, consecutiveMinutes, publishDelayMs } = RESERVE_WATCH_RULES.director503
const { background } = state
@@ -116,6 +210,8 @@ function judgeDirector503(read, state, at) {
if (recent.length < consecutiveMinutes) {
return unverifiable(`director requests are published only through ${read.publishedThrough}`)
}
+ const unmetered = recent.filter((minute) => read.unmetered.includes(minute))
+ if (unmetered.length > 0) return unverifiable(`no director runtime metrics for ${unmetered.join(', ')}`)
const counts = recent.map((minute) => read.perMinute[minute] ?? 0)
const detail = { backgroundPerMinute: perMinute, above, recent: Object.fromEntries(recent.map((m, i) => [m, counts[i]])) }
if (counts.every((count) => count > above)) {
@@ -181,9 +277,78 @@ function judgeLedgerDrift(cell, ledger, aboveSince, at) {
return { verdict: pass({ ...detail, aboveForMs: at - since }), aboveSince: since }
}
+// A counter still rising (or failing) every sample for 5 minutes pages once; a single drop does not.
+function judgeSustained(rising, since, at) {
+ const next = rising ? (since ?? at) : null
+ return { since: next, sustained: next !== null && at - next >= PHONE_WATCH_RULES.pageAfterMs }
+}
+
+export function phoneSinceOf(cell, state) {
+ if (state.phoneSince !== null || !cell.ok || cell.phonePathConfigured !== 'memory') return state.phoneSince
+ // Memory from the first sample: the operator's flipped-at. Later: the sample before this one.
+ return state.lastSampleAt === null ? state.flippedAt : state.lastSampleAt
+}
+
+// The phone rules for one sample, or none while the switch file does not say phonePath=memory.
+function judgePhoneRules(cell, phone, state, at) {
+ if (!cell.ok || cell.phonePathConfigured !== 'memory' || state.phoneOffAt !== null) return {}
+ const background = state.phoneBackground
+ if (!background) {
+ const verdict =
+ state.phoneSince === null
+ ? unverifiable('the phone rules need --flipped-at: their background is the 10 minutes before it')
+ : at < minuteFloor(state.phoneSince) + RESERVE_WATCH_RULES.director503.publishDelayMs
+ ? pending('the phone background before phonePath went memory is not published yet')
+ : unverifiable('no phone background yet')
+ return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, verdict]))
+ }
+ if (!phone.ok) return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, unverifiable(phone.error)]))
+ const { consecutiveMinutes } = PHONE_WATCH_RULES
+ // Only the judged minutes need metrics; a gap earlier in the read changes nothing.
+ const judged = phone.minutes.slice(-consecutiveMinutes)
+ const gaps = phone.unmetered.filter((minute) => judged.includes(minute))
+ if (gaps.length > 0) {
+ const gap = unverifiable(`no cell runtime metrics for ${gaps.join(', ')}`)
+ return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, gap]))
+ }
+ const accepts = phone.series.completedAccepts.slice(-consecutiveMinutes)
+ return Object.fromEntries(
+ PHONE_RULE_NAMES.map((name) => {
+ const signal = PHONE_RULE_SIGNALS[name]
+ const line = phoneShareLine(background.share[signal])
+ const recent = phone.series[signal].slice(-consecutiveMinutes)
+ const detail = { backgroundShare: background.share[signal], line, recent, accepts }
+ const over =
+ recent.length === consecutiveMinutes && recent.every((count, index) => phoneMinuteOver(signal, count, accepts[index], line))
+ return [
+ name,
+ over ? trip(`${signal} over ${Math.round(line * 100)}% of attempts for ${consecutiveMinutes} min (${recent.join(', ')} against ${accepts.join(', ')} accepts)`, detail) : pass(detail)
+ ]
+ })
+ )
+}
+
+// Page-only signals: audit rows dropped since the previous sample, and renewal writes failing in
+// every one of the last 5 published minutes.
+function judgePhonePages(cell, phone, state, at) {
+ const dropped = cell.ok && Number.isInteger(cell.resumeAuditDropped) ? cell.resumeAuditDropped : null
+ const audit = judgeSustained(
+ dropped !== null && state.auditDropsLast !== null && dropped > state.auditDropsLast,
+ state.auditDropsSince,
+ at
+ )
+ const renewals = phone?.ok ? (phone.series.renewalFailures ?? []).slice(-PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS) : []
+ const renewal = judgeSustained(renewals.length > 0 && renewals.every((count) => count > 0), state.renewalFailuresSince, at)
+ const pages = []
+ if (audit.sustained && !state.paged.includes('audit-drops')) pages.push('audit-drops')
+ if (renewal.sustained && !state.paged.includes('renewal-failures')) pages.push('renewal-failures')
+ return { auditDropsLast: dropped ?? state.auditDropsLast, auditDropsSince: audit.since, renewalFailuresSince: renewal.since, pages }
+}
+
/**
* One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of
- * director 503s with the request series' publishing front (`publishedThrough`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this
+ * non-drain director 503s with the request series' publishing front (`publishedThrough`) and the
+ * minutes no runtime metrics cover (`unmetered`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this
* cell's reconcile reports in the lookback. Each is `{ ok: false, error }` when it could not be read.
*/
export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
@@ -195,14 +360,14 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) {
// Not judged, and the streak holds: the reserve write may not have applied yet.
const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) }
- return { state: next, rules, line: sampleLine(next, sample, rules) }
+ return { state: next, rules, line: sampleLine(next, sample, rules), pages: [] }
}
// Someone else flipped it back (or it never became reserve): nothing left to watch.
next.state = seen ? 'stopped' : 'refused'
next.verdict = seen
? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})`
: `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})`
- return { state: next, rules: {}, line: sampleLine(next, sample, {}) }
+ return { state: next, rules: {}, line: sampleLine(next, sample, {}), pages: [] }
}
if (cell.ok) {
next.reserveSeenAt = state.reserveSeenAt ?? at
@@ -214,34 +379,63 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) {
next.seatBehindRowFirstSeen = duplicate.firstSeen
const ledger = judgeLedgerDrift(cell, sample.ledger, state.ledgerAboveSince, at)
next.ledgerAboveSince = ledger.aboveSince
+ next.lastSampleAt = at
+ next.phoneSince = phoneSinceOf(cell, state)
+ const phoneState = { ...state, phoneSince: next.phoneSince }
+ const pageCheck = judgePhonePages(cell, sample.phone, state, at)
+ next.auditDropsLast = pageCheck.auditDropsLast
+ next.auditDropsSince = pageCheck.auditDropsSince
+ next.renewalFailuresSince = pageCheck.renewalFailuresSince
+ next.paged = [...state.paged, ...pageCheck.pages]
+ const pages = pageCheck.pages
const rules = {
deadMan: judgeDeadMan(cell),
overrun: judgeOverrun(cell),
director503: judgeDirector503(sample.director503, state, at),
duplicateSeat: duplicate.verdict,
hostDrop: cell.ok ? judgeHostDrop(next.hosts) : unverifiable(cell.error),
- ledgerDrift: ledger.verdict
+ ledgerDrift: ledger.verdict,
+ ...judgePhoneRules(cell, sample.phone ?? failedPhoneRead, phoneState, at)
}
const tripped = RULE_NAMES.find((name) => rules[name].status === 'trip')
if (tripped) {
next.state = 'tripped'
+ next.flip = 'admitMode'
next.verdict = `flip back ${state.cellId}: ${tripped}: ${rules[tripped].reason}`
- return { state: next, rules, line: sampleLine(next, sample, rules) }
+ return { state: next, rules, line: sampleLine(next, sample, rules), pages }
+ }
+ // A phone trip turns off only the phone path; the segment writes it and the watch goes on.
+ const phoneNames = PHONE_RULE_NAMES.filter((name) => name in rules)
+ const phoneTripped = phoneNames.find((name) => rules[name].status === 'trip')
+ const phoneMissing = phoneNames.filter((name) => rules[name].status === 'unverifiable')
+ next.phoneUnverifiableStreak = phoneMissing.length > 0 ? state.phoneUnverifiableStreak + 1 : 0
+ if (phoneTripped) {
+ next.phoneTrip = `turn off the phone path on ${state.cellId}: ${phoneTripped}: ${rules[phoneTripped].reason}`
+ } else if (next.phoneUnverifiableStreak >= unverifiableLimit) {
+ next.phoneTrip =
+ `turn off the phone path on ${state.cellId}: unverifiable: ${next.phoneUnverifiableStreak} samples ` +
+ `in a row could not read ${phoneMissing.join(', ')}`
}
const missing = RULE_NAMES.filter((name) => rules[name].status === 'unverifiable')
- const waiting = RULE_NAMES.some((name) => rules[name].status === 'pending')
+ const waiting = [...RULE_NAMES, ...phoneNames].some((name) => rules[name].status === 'pending')
// A pending rule holds the streak where it was: neither evidence of health nor a gap.
next.unverifiableStreak = missing.length > 0 ? state.unverifiableStreak + 1 : waiting ? state.unverifiableStreak : 0
if (next.unverifiableStreak >= unverifiableLimit) {
next.state = 'tripped'
+ next.flip = 'admitMode'
next.verdict =
`flip back ${state.cellId}: unverifiable: ${next.unverifiableStreak} samples in a row could not ` +
`read ${missing.join(', ')}`
+ } else if (at >= state.deadlineAt && next.phoneTrip) {
+ // Still unwritten at the end: the workflow's flip back writes it, and pages if that fails too.
+ next.state = 'tripped'
+ next.flip = 'phonePath'
+ next.verdict = `${next.phoneTrip}; phonePath=db did not land before the watch ended`
} else if (at >= state.deadlineAt) {
next.state = 'completed'
next.verdict = `${state.cellId} stayed healthy in reserve mode for the whole watch`
}
- return { state: next, rules, line: sampleLine(next, sample, rules) }
+ return { state: next, rules, line: sampleLine(next, sample, rules), pages }
}
function sampleLine(state, sample, rules) {
@@ -253,6 +447,7 @@ function sampleLine(state, sample, rules) {
state: state.state,
unverifiableStreak: state.unverifiableStreak,
rules,
+ ...(state.phoneTrip ? { phoneTrip: state.phoneTrip } : {}),
...(state.verdict ? { verdict: state.verdict } : {})
})
}
diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs
index 2914b2820953..3815a6b965ff 100644
--- a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs
+++ b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs
@@ -232,9 +232,9 @@ const DIRECTOR_DRAIN_FIELDS = [
]
// Five instances at one sample per 30 s is ~100 per 10-min sub-window; this many is truncation.
-const DIRECTOR_METRICS_LIMIT = 1000
+export const DIRECTOR_METRICS_LIMIT = 1000
-async function readDirectorDrainReturn(reader, { config, window }) {
+export async function readDirectorDrainReturn(reader, { config, window }) {
const projection = `json(timestamp,${DIRECTOR_DRAIN_FIELDS
.map((field) => `jsonPayload.${field}`)
.join(',')})`
diff --git a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs
index f5e183b47486..f1313bcafecb 100644
--- a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs
+++ b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs
@@ -42,9 +42,9 @@ const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c
const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0'
// C31 launches on the digest the director served when it was declared.
const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269'
-// Staging C4 is declared on image G (main 2de73812, E-pre with the t_preauth metric); the staging
+// Staging C4 is declared on image J (main 30f5bef9, step-6 phonePath staging test); the staging
// director deploy binds its digest to this line.
-const stagingC4Digest = '07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d'
+const stagingC4Digest = '21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4'
function cellBlock(tfvars, cellId) {
const start = tfvars.indexOf(`"${cellId}"`)
diff --git a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs
index 1f4d946557c0..66168db1309a 100644
--- a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs
+++ b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs
@@ -150,6 +150,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () =>
'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer',
'google_iam_workload_identity_pool_provider.github_staging_relay_deploy',
'google_project_iam_member.github_staging_relay_deploy_compute_viewer',
+ 'google_project_iam_member.github_staging_relay_deploy_logging_viewer',
'google_service_account.github_staging_relay_deploy',
'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user',
'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user',
@@ -189,7 +190,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () =>
const projectRoles = declaredFamilies()
.filter((family) => family.startsWith('google_project_iam_member.'))
.map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1])
- assert.deepEqual(projectRoles, ['roles/compute.viewer'])
+ assert.deepEqual(projectRoles, ['roles/compute.viewer', 'roles/logging.viewer'])
assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/)
assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/)
})
diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.mjs
index 6f5cc38e279f..4e685ae38a26 100644
--- a/cloud/dev/scripts/watch-relay-reserve-cell.mjs
+++ b/cloud/dev/scripts/watch-relay-reserve-cell.mjs
@@ -21,14 +21,28 @@ import {
readRuntime,
WRITE_CONFIRMATION
} from './operate-relay-cell-flags.mjs'
-import { readDirectorRequestsByCodePerMinute, readLogEntries } from './relay-same-cap-shadow-gate.mjs'
-import { minutesOf } from './relay-same-cap-shadow-gate-verdict.mjs'
+import {
+ DIRECTOR_METRICS_LIMIT,
+ readDirectorDrainReturn,
+ readDirectorRequestsByCodePerMinute,
+ readLogEntries
+} from './relay-same-cap-shadow-gate.mjs'
+import {
+ DIRECTOR_METRICS_INTERVAL_MS,
+ drainReturnByMinute,
+ minuteKey,
+ minutesOf,
+ withoutDrainDeferrals
+} from './relay-same-cap-shadow-gate-verdict.mjs'
import { PRODUCTION_LEASE } from './cloud-sql-rollout-lock-census.mjs'
import { RELAY_REPOSITORY_ROOT } from './relay-repository.mjs'
import {
evaluateWatchSample,
initialWatchState,
measuredBackground,
+ measuredPhoneBackground,
+ phoneSinceOf,
+ PHONE_WATCH_RULES,
RESERVE_WATCH_RULES
} from './relay-reserve-watch-rules.mjs'
@@ -58,8 +72,11 @@ function integerIn(name, raw, min, max) {
}
// The flip back's request. Its generation is a placeholder: the flip back re-reads the current one.
+// A phone-rule trip turns off only the phone path; anything else flips the whole cell back. Both
+// remove the phonePath key, so a cell rolled back to an image without the switch takes the write.
export function dbFlipRequest(values) {
- return parseCellFlagsRequest({ ...values, set: 'admitMode=db', 'expected-generation': '0' })
+ const set = values.flip === 'phonePath' ? 'phonePath=default' : 'admitMode=db,phonePath=default'
+ return parseCellFlagsRequest({ ...values, set, 'expected-generation': '0' })
}
// `now` is the watch's start: a later segment of a long watch must not age flipped-at out.
@@ -93,6 +110,8 @@ export function parseWatchArguments(values, now = Date.now()) {
if (!values['state-file']) throw new Error('--state-file is required')
return {
request,
+ // The flag tool's own inputs, for turning the phone path off mid-watch.
+ flipValues: values,
directorService: values['director-service'],
durationMinutes: integerIn('duration-minutes', values['duration-minutes'], 5, MAX_WATCH_MINUTES),
segmentMinutes: integerIn('segment-minutes', values['segment-minutes'], 1, MAX_SEGMENT_MINUTES),
@@ -119,6 +138,8 @@ export async function readCellSample(fetchImpl, request, idToken, wait) {
ok: true,
admitModeConfigured: runtime.flagsApplied.flags?.admitMode ?? 'db',
admitModeEffective: runtime.admitModeEffective,
+ phonePathConfigured: runtime.flagsApplied.flags?.phonePath ?? 'db',
+ resumeAuditDropped: runtime.runtime?.resumeAuditDropped,
draining: runtime.draining === true,
controls,
enforcedUnits: runtime.runtime?.enforcedConnectionUnits,
@@ -130,31 +151,97 @@ export async function readCellSample(fetchImpl, request, idToken, wait) {
}
const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS
-const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null })
+const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null, unmetered: [] })
-// The `minutes` whole minutes that end at the minute boundary `endedAt`: the 503 counts, and the
-// newest minute any response code has a point for. The director always serves something, so that
-// front is how far the 503 series is published; no front at all is no evidence. A code publishing
-// late only delays a trip: a later read sees its points.
+// The `minutes` whole minutes that end at the minute boundary `endedAt`: the director's non-drain
+// 503s, the roll ladder's measure its line comes from. Cloud Run's 503 count less what the
+// director's runtime metrics attribute to drain-return deferrals and to a host re-dialling inside
+// its own retry interval or while its last dial is in flight (10-18: ~98% of 503s), since those
+// grow with traffic, not trouble. `publishedThrough` is the newest minute any response code has a
+// point for (the director always serves something, so no front at all is no evidence); a minute
+// with no metrics line from any instance is `unmetered`, since subtracting nothing there is a guess.
+// Coverage is any instance, not each: one instance's missing lines under-subtract, biased to a trip.
export async function readDirector503Minutes(reader, { projectId, directorService, endedAt, minutes }) {
const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) }
+ const config = { projectId, directorService }
let read
try {
- read = await readDirectorRequestsByCodePerMinute(reader, { config: { projectId, directorService }, span })
+ read = await readDirectorRequestsByCodePerMinute(reader, { config, span })
} catch (error) {
// A point with an unparseable end time has no minute.
return failed503Read(`director 503 metric unreadable: ${error instanceof Error ? error.message : error}`)
}
if (read.failed) return failed503Read('director 503 metric read failed')
+ // A line covers the 30 s before it, so the last minute's tail is in the next minute's lines.
+ const metrics = await readDirectorDrainReturn(reader, {
+ config,
+ window: { startedAt: span.startedAt, endedAt: new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS) }
+ })
+ const lineTimes = metrics.flatMap((part) => part.samples.map((sample) => Date.parse(sample.timestamp)))
+ if (lineTimes.some((at) => !Number.isFinite(at))) return failed503Read('director runtime metrics line with an unparseable timestamp')
+ const scheduled = drainReturnByMinute(metrics, DIRECTOR_METRICS_LIMIT)
+ if (scheduled.truncated) return failed503Read('director runtime metrics read failed, truncated, or short')
+ const keys = minutesOf(span)
+ const metered = new Set(lineTimes.flatMap((at) => [minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS), minuteKey(at - 1)]))
+ const nonDrain = withoutDrainDeferrals({ perMinute: read.byCode['503'] ?? {} }, scheduled, keys)
const published = Object.values(read.byCode).flatMap((perMinute) => Object.keys(perMinute)).sort()
return {
ok: true,
- perMinute: read.byCode['503'] ?? {},
- minutes: minutesOf(span),
- publishedThrough: published.at(-1) ?? null
+ perMinute: Object.fromEntries(keys.map((minute, index) => [minute, nonDrain.series[index]])),
+ minutes: keys,
+ publishedThrough: published.at(-1) ?? null,
+ unmetered: keys.filter((minute) => !metered.has(minute))
}
}
+const CELL_METRICS_EVENT = 'orca_relay_runtime_metrics'
+const PHONE_SIGNALS = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors', 'renewalFailures', 'completedAccepts']
+const sumOf = (counts) => Object.values(counts ?? {}).reduce((sum, count) => sum + Number(count), 0)
+
+// The cell's own runtime metrics lines (one per 30 s) for the `minutes` whole minutes ending at
+// `endedAt`, per minute: what the phone rules judge. A line counts toward the minute it ends in;
+// a minute no line covers is `unmetered`, never zero.
+export async function readCellPhoneMinutes(reader, { cellId, endedAt, minutes }) {
+ const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) }
+ const read = await readLogEntries(reader, {
+ filter: [
+ 'resource.type="gce_instance"',
+ `jsonPayload.event="${CELL_METRICS_EVENT}"`,
+ `jsonPayload.cellId="${cellId}"`,
+ `timestamp>="${span.startedAt.toISOString()}"`,
+ `timestamp<"${new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS).toISOString()}"`
+ ].join(' AND '),
+ projection:
+ 'json(timestamp,jsonPayload.phoneRejectionsByCodeDelta,jsonPayload.resumeConfirmFailuresByCodeDelta,' +
+ 'jsonPayload.phoneAttachDeadlinesDelta,jsonPayload.phoneAcceptDatabaseErrorsDelta,jsonPayload.resumeRenewalFailuresDelta,' +
+ 'jsonPayload.clientAcceptCompletedDelta)',
+ limit: LOG_ENTRY_LIMIT
+ })
+ if (read.failed) return failedRead(read.error)
+ if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('cell runtime metrics read truncated')
+ const keys = minutesOf(span)
+ const series = Object.fromEntries(PHONE_SIGNALS.map((signal) => [signal, keys.map(() => 0)]))
+ const metered = new Set()
+ for (const entry of read.entries) {
+ const at = Date.parse(entry.timestamp)
+ if (!Number.isFinite(at)) return failedRead('cell runtime metrics line with an unparseable timestamp')
+ const payload = entry.jsonPayload ?? {}
+ if (payload.phoneAttachDeadlinesDelta === undefined) {
+ return { ...failedRead('cell runtime metrics carry no phone counters (an image before step 6)'), permanent: true }
+ }
+ metered.add(minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS)).add(minuteKey(at - 1))
+ const index = keys.indexOf(minuteKey(at - 1))
+ if (index < 0) continue
+ series.credentialRefusals[index] += Number(payload.phoneRejectionsByCodeDelta?.['4401'] ?? 0)
+ series.confirmFailures[index] += sumOf(payload.resumeConfirmFailuresByCodeDelta)
+ series.attachDeadlines[index] += Number(payload.phoneAttachDeadlinesDelta)
+ series.acceptDatabaseErrors[index] += Number(payload.phoneAcceptDatabaseErrorsDelta ?? 0)
+ series.renewalFailures[index] += Number(payload.resumeRenewalFailuresDelta ?? 0)
+ series.completedAccepts[index] += Number(payload.clientAcceptCompletedDelta ?? 0)
+ }
+ return { ok: true, minutes: keys, series, unmetered: keys.filter((minute) => !metered.has(minute)) }
+}
+
const LEDGER_SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary'
const SEAT_BEHIND_ROW_EVENT = 'orca_relay_reserve_ledger_seat_behind_row'
@@ -342,13 +429,48 @@ export async function watchSegment(config, dependencies) {
endedAt: baselineEndsAt,
minutes: BACKGROUND_MINUTES
})
- state = { ...state, background: read.ok ? measuredBackground(read.perMinute, read.minutes, read.publishedThrough) : null }
+ state = { ...state, background: read.ok ? measuredBackground(read) : null }
}
const reader = createReader(deps, request.projectId)
// Serialised: concurrent Logging reads are what earn a 429.
+ const cell = await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep)
+ const phoneWatched = cell.ok && cell.phonePathConfigured === 'memory' && state.phoneOffAt === null
+ const phoneSince = phoneSinceOf(cell, state)
+ const phoneBaselineEndsAt = phoneSince === null ? null : minuteFloor(phoneSince)
+ if (phoneWatched && !state.phoneBackground && phoneBaselineEndsAt !== null && at >= phoneBaselineEndsAt + METRIC_DELAY_MS) {
+ const read = await readCellPhoneMinutes(reader, {
+ cellId: request.cellId,
+ endedAt: phoneBaselineEndsAt,
+ minutes: BACKGROUND_MINUTES
+ })
+ // A restart or roll in those minutes leaves no background to judge against: no phone
+ // rule could ever pass, so refuse rather than turn the phone path off for certain.
+ const incomplete = read.ok ? read.unmetered.length > 0 && `no metrics for ${read.unmetered.join(', ')}` : read.permanent && read.error
+ if (incomplete) {
+ state = {
+ ...state,
+ state: 'refused',
+ verdict:
+ `${request.cellId}: the phone background, the 10 minutes before ${new Date(phoneBaselineEndsAt).toISOString()}, ` +
+ `is incomplete (${incomplete}), as after a cell restart or roll; start a new watch with a later --flipped-at`
+ }
+ await writeFile(config.stateFile, JSON.stringify(state))
+ break
+ }
+ state = { ...state, phoneBackground: measuredPhoneBackground(read) }
+ }
const sample = {
at,
- cell: await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep),
+ cell,
+ ...(phoneWatched
+ ? {
+ phone: await readCellPhoneMinutes(reader, {
+ cellId: request.cellId,
+ endedAt: minuteFloor(at - METRIC_DELAY_MS),
+ minutes: PHONE_WATCH_RULES.readMinutes
+ })
+ }
+ : {}),
director503: await readDirector503Minutes(reader, {
projectId: request.projectId,
directorService,
@@ -362,6 +484,13 @@ export async function watchSegment(config, dependencies) {
const result = evaluateWatchSample(state, sample, { unverifiableLimit: config.unverifiableLimit })
state = result.state
log(result.line)
+ for (const kind of result.pages) {
+ await raiseAlarm(
+ { mode: request.mode, cellId: request.cellId, projectId: request.projectId, kind, reason: `sustained for ${PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS} min` },
+ { runGcloud: dependencies.runGcloud, log }
+ )
+ }
+ if (state.state === 'watching' && state.phoneTrip) state = await turnPhonePathOff(config, state, at, { ...deps, log })
await writeFile(config.stateFile, JSON.stringify(state))
if (state.state !== 'watching') break
await sleep(Math.max(0, at + SAMPLE_INTERVAL_MS - now()))
@@ -370,8 +499,41 @@ export async function watchSegment(config, dependencies) {
return state
}
-function writeOutcome(githubOutput, { state, verdict }) {
- return appendFile(githubOutput, `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\n`)
+// Writes phonePath=db (the flag tool's CAS write, a plan in dry-run) and keeps watching the
+// reserve rules. A write that fails keeps the trip, so the next sample retries it, and pages once
+// after PHONE_OFF_PAGE_AFTER_FAILURES in a row: the cell stays watched either way.
+export const PHONE_OFF_PAGE_AFTER_FAILURES = 3
+async function turnPhonePathOff(config, state, at, dependencies) {
+ try {
+ const result = await flipBack({ ...config.flipValues, flip: 'phonePath' }, dependencies)
+ dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off', cellId: state.cellId, verdict: state.phoneTrip, written: result.written }))
+ return { ...state, phoneOffAt: at, phoneTrip: null, phoneOffFailures: 0 }
+ } catch (error) {
+ const reason = error instanceof Error ? error.message : String(error)
+ const failures = (state.phoneOffFailures ?? 0) + 1
+ dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off_failed', cellId: state.cellId, failures, reason }))
+ if (failures === PHONE_OFF_PAGE_AFTER_FAILURES) {
+ await raiseAlarm(
+ {
+ mode: config.request.mode,
+ cellId: state.cellId,
+ projectId: config.request.projectId,
+ kind: 'flip-back-failed',
+ flip: 'phonePath',
+ reason: `${state.phoneTrip}; phonePath=db failed ${failures} times: ${reason}`
+ },
+ { runGcloud: dependencies.runGcloud, log: dependencies.log }
+ )
+ }
+ return { ...state, phoneOffFailures: failures }
+ }
+}
+
+function writeOutcome(githubOutput, { state, verdict, flip }) {
+ return appendFile(
+ githubOutput,
+ `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\nflip=${flip ?? 'admitMode'}\n`
+ )
}
// The job's outcome. A watch that ended mid-segment (a crash, a timeout) records failed, which the
@@ -431,7 +593,11 @@ export async function flipBack(values, dependencies) {
)
for (let attempt = 1; ; attempt += 1) {
const current = await readCurrentObject(fetchImpl, base, dependencies.accessToken)
- if ((current.object?.flags?.admitMode ?? 'db') !== 'reserve') {
+ const stillOn =
+ values.flip === 'phonePath'
+ ? (current.object?.flags?.phonePath ?? 'db') === 'memory'
+ : (current.object?.flags?.admitMode ?? 'db') === 'reserve'
+ if (!stillOn) {
const result = { written: false, alreadyDb: true, generation: current.generation }
record(result)
return result
@@ -464,7 +630,7 @@ export async function flipBack(values, dependencies) {
export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm'
export const ALARM_LOG_NAME = 'orca-relay-reserve-watch'
-export const ALARM_KINDS = ['refused', 'flip-back-failed']
+export const ALARM_KINDS = ['refused', 'flip-back-failed', 'audit-drops', 'renewal-failures']
// GitHub's workflow-command escaping, so a verdict cannot end the annotation or start another.
export const escapeWorkflowCommand = (text) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
@@ -474,12 +640,18 @@ const escapeWorkflowProperty = (text) => escapeWorkflowCommand(text).replace(/:/
* Both kinds can leave the cell in reserve with nothing watching it. Only write mode pages: a
* dry-run watch never flips back, so its refusal or failure changes nothing for the cell.
*/
-export async function raiseAlarm({ mode, cellId, projectId, kind, reason }, { runGcloud, log = console.log }) {
+export async function raiseAlarm({ mode, cellId, projectId, kind, reason, flip }, { runGcloud, log = console.log }) {
if (!ALARM_KINDS.includes(kind)) throw new Error(`--kind must be one of ${ALARM_KINDS.join(', ')}`)
const message =
kind === 'refused'
? `${cellId} may be in reserve mode and unwatched: the watch was refused (${reason}). Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.`
- : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.`
+ : kind === 'audit-drops'
+ ? `${cellId} is losing resume audit rows (${reason}); phones still connect. Check its database writes, or set phonePath=db.`
+ : kind === 'renewal-failures'
+ ? `${cellId} cannot renew phone credentials (${reason}); phones still connect but age toward re-pairing. Check its database writes, or set phonePath=db.`
+ : flip === 'phonePath'
+ ? `${cellId} may still have phonePath=memory with no watch: turning it off did not land (${reason}). Set phonePath=default by hand with Operate Relay Production Cell Flags.`
+ : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.`
log(`::error title=${escapeWorkflowProperty(`Reserve watch ${kind} for ${cellId}`)}::${escapeWorkflowCommand(message)}`)
if (mode !== 'write') return { paged: false }
const entry = JSON.stringify({ event: ALARM_EVENT, cellId, kind, reason })
@@ -518,7 +690,8 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
'state-file',
'github-output',
'verdict',
- 'kind'
+ 'kind',
+ 'flip'
].map((name) => [name, { type: 'string' }])
)
})
@@ -528,7 +701,7 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
try {
if (command === 'alarm') {
await raiseAlarm(
- { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '' },
+ { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '', flip: values.flip },
{ runGcloud }
)
// The alarm step is red, whatever the page did.
@@ -542,6 +715,13 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
const config = await parseWatchSegment(values)
const state = await watchSegment(config, {
idToken,
+ // For turning the phone path off mid-watch; without it that write fails and the workflow's flip back retries.
+ accessToken: process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN,
+ audit: {
+ writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`,
+ runId: process.env.GITHUB_RUN_ID ?? 'local',
+ runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0'
+ },
readRolloutLease: readProductionRolloutLease,
runGcloud
})
diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs
index 28d34ab47235..0f5b1467c0cd 100644
--- a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs
+++ b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs
@@ -7,7 +7,12 @@ import { test } from 'node:test'
import {
evaluateWatchSample,
initialWatchState,
- measuredBackground
+ measuredBackground,
+ measuredPhoneBackground,
+ phoneMinuteOver,
+ phoneShareLine,
+ PHONE_RULE_NAMES,
+ PHONE_WATCH_RULES
} from './relay-reserve-watch-rules.mjs'
import {
ALARM_EVENT,
@@ -18,6 +23,7 @@ import {
parseWatchArguments,
parseWatchSegment,
raiseAlarm,
+ readCellPhoneMinutes,
readDirector503Minutes,
readSeatBehindRowSample,
recordOutcome,
@@ -47,7 +53,7 @@ const minutesBefore = (at, count) =>
)
const quiet503 = (at, perMinute = 10) => {
const minutes = minutesBefore(at, 3)
- return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1) }
+ return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1), unmetered: [] }
}
const sampleAt = (at, overrides = {}) => ({
at,
@@ -93,7 +99,8 @@ test('503s trip only after three complete minutes over max(1.5x background, back
ok: true,
minutes,
perMinute: Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])),
- publishedThrough: minutes.at(-1)
+ publishedThrough: minutes.at(-1),
+ unmetered: []
}
}
const twoOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 140, 200]) }), LIMIT)
@@ -115,12 +122,12 @@ test('503s trip only after three complete minutes over max(1.5x background, back
test('a quiet fleet with no 503 series at all reads zero, not unverifiable', () => {
const minutes = minutesBefore(START, 10)
- const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1) }
+ const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1), unmetered: [] }
const [result] = run([sampleAt(START, { director503: quiet })], freshState({ background: { medianPerMinute: 0, source: 'measured' } }))
assert.equal(result.rules.director503.status, 'pass')
assert.deepEqual(Object.values(result.rules.director503.recent), [0, 0, 0])
assert.equal(result.state.unverifiableStreak, 0)
- assert.deepEqual(measuredBackground({}, minutes, minutes.at(-1)), { medianPerMinute: 0, minutes: 10, source: 'measured' })
+ assert.deepEqual(measuredBackground({ perMinute: {}, minutes, publishedThrough: minutes.at(-1), unmetered: [] }), { medianPerMinute: 0, minutes: 10, source: 'measured' })
})
test('503 minutes past the publishing front are left out, and only no front at all is unverifiable', () => {
@@ -129,20 +136,20 @@ test('503 minutes past the publishing front are left out, and only no front at a
const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT)
// The front is two minutes behind: the three minutes before it are judged, the two after are not.
const over = Object.fromEntries(minutes.map((minute) => [minute, 500]))
- const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7] })
+ const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7], unmetered: [] })
assert.equal(behind.rules.director503.status, 'trip')
assert.deepEqual(Object.keys(behind.rules.director503.recent), minutes.slice(5, 8))
// A sparse 503 series: minutes after its last point but within the front are zero, not late.
- const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1) })
+ const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1), unmetered: [] })
assert.equal(sparse.rules.director503.status, 'pass')
assert.equal(sparse.state.unverifiableStreak, 0)
// No request series at all.
- const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null })
+ const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null, unmetered: [] })
assert.equal(empty.rules.director503.status, 'unverifiable')
assert.match(empty.rules.director503.reason, /request series is empty/)
assert.equal(empty.state.unverifiableStreak, 1)
// A front too far behind to leave three published minutes in the read.
- const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1] })
+ const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1], unmetered: [] })
assert.equal(stale.rules.director503.status, 'unverifiable')
assert.match(stale.rules.director503.reason, new RegExp(`published only through ${minutes[1]}`))
})
@@ -215,16 +222,21 @@ test('the duplicate read carries a positive control: no fresh ledger summary in
assert.equal(gap.state.unverifiableStreak, 1)
})
-test('the measured background is the median of the minutes before the flip, published gaps as zero', () => {
+test('the measured background is the median of the metered minutes before the flip, published gaps as zero', () => {
const minutes = ['2026-10-14T11:47', '2026-10-14T11:48', '2026-10-14T11:49']
const front = minutes[2]
- assert.equal(measuredBackground({ '2026-10-14T11:48': 30 }, minutes, front).medianPerMinute, 0)
- assert.equal(measuredBackground({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, minutes, front).medianPerMinute, 20)
- assert.equal(measuredBackground({}, minutes, '2026-10-14T11:52').medianPerMinute, 0)
+ const background = (perMinute, publishedThrough, readMinutes = minutes, unmetered = []) =>
+ measuredBackground({ perMinute, minutes: readMinutes, publishedThrough, unmetered })
+ assert.equal(background({ '2026-10-14T11:48': 30 }, front).medianPerMinute, 0)
+ assert.equal(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front).medianPerMinute, 20)
+ assert.equal(background({}, '2026-10-14T11:52').medianPerMinute, 0)
// Not published through the last minute yet, or no request series: no background yet.
- assert.equal(measuredBackground({ [minutes[0]]: 10 }, minutes, minutes[1]), null)
- assert.equal(measuredBackground({}, minutes, null), null)
- assert.equal(measuredBackground({}, [], front), null)
+ assert.equal(background({ [minutes[0]]: 10 }, minutes[1]), null)
+ assert.equal(background({}, null), null)
+ assert.equal(background({}, front, []), null)
+ // An unmetered minute is left out, not read as zero; under half metered is no background.
+ assert.deepEqual(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front, minutes, [minutes[0]]), { medianPerMinute: 25, minutes: 2, source: 'measured' })
+ assert.equal(background({}, front, minutes, minutes.slice(1)), null)
})
test('any overrun trips at once', () => {
@@ -391,6 +403,17 @@ test('the 503 background needs flipped-at or an explicit value; flipped-at must
})
// A cell, Cloud Monitoring and Cloud Logging, answering from `script` per sample.
+const METRICS_EVENT = 'orca_relay_runtime_metrics'
+// Each instance's line every 30 s inside the read's timestamp bounds, newest first.
+function metricsLines(filter, metrics) {
+ const [, from, to] = filter.match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/)
+ const lines = []
+ for (let at = Math.ceil(Date.parse(from) / 30_000) * 30_000; at < Date.parse(to); at += 30_000) {
+ for (const deltas of metrics(at)) lines.push({ timestamp: new Date(at).toISOString(), jsonPayload: deltas })
+ }
+ return lines.reverse()
+}
+
function fakeSources({
cell = () => ({}),
// Logged at each sample's minute, so it stays fresh for the positive control.
@@ -399,7 +422,9 @@ function fakeSources({
// Requests of every other code per minute; zero is a minute with no traffic at all.
perOther = () => 1_000,
seatControl = true,
- rolloutLease = null
+ rolloutLease = null,
+ // Each director instance's runtime-metrics deltas for the 30 s ending at `endedAt`.
+ metrics = () => [{}]
} = {}) {
let samples = 0
const monitoringSpans = []
@@ -447,6 +472,7 @@ function fakeSources({
const summaries = ledger(samples).map((entry) => ({ ...entry, jsonPayload: { ...entry.jsonPayload, event: SUMMARY_EVENT } }))
if (filter.includes(SEAT_EVENT)) return { stdout: JSON.stringify(seatControl ? summaries : []) }
if (filter.includes(SUMMARY_EVENT)) return { stdout: JSON.stringify(ledger(samples)) }
+ if (filter.includes(METRICS_EVENT)) return { stdout: JSON.stringify(metricsLines(filter, metrics)) }
return { stdout: '[]' }
}
return { fetchImpl, runGcloud, monitoringSpans, readRolloutLease: async () => rolloutLease }
@@ -473,7 +499,7 @@ test('a segment samples once a minute, carries state to the next segment, and st
assert.equal(first.state, 'watching')
assert.equal(lines.length, 3)
assert.equal(first.background.source, 'measured')
- assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\n')
+ assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\nflip=admitMode\n')
const second = await watchSegment(config, dependencies)
assert.equal(second.state, 'tripped')
assert.equal(second.samples, 5)
@@ -517,7 +543,7 @@ test('a watch that ended without a verdict records failed, which the workflow fl
assert.equal((await recordOutcome(join(directory, 'watching.json'), output)).state, 'failed')
await writeFile(join(directory, 'tripped.json'), JSON.stringify(freshState({ state: 'tripped', verdict: 'flip back x:\ny' })))
assert.equal((await recordOutcome(join(directory, 'tripped.json'), output)).state, 'tripped')
- assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\n$/)
+ assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\nflip=admitMode\n$/)
} finally {
await rm(directory, { recursive: true, force: true })
}
@@ -525,10 +551,20 @@ test('a watch that ended without a verdict records failed, which the workflow fl
// Google Storage and the director, as far as the flag tool's flip back reaches them. `onUpload`
// returns 'conflict' to play another writer landing first, optionally with its own admitMode.
-function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = true } = {}) {
+function fakeFlipBack({
+ generation = '7',
+ onUpload = () => 'ok',
+ cellAnswers = true,
+ flags = { admitMode: 'reserve', readinessLocal: true },
+ supportedFlags = {
+ admitMode: { type: 'enum', values: ['db', 'reserve'] },
+ readinessLocal: { type: 'boolean' },
+ phonePath: { type: 'enum', values: ['db', 'memory'] }
+ }
+} = {}) {
const calls = []
let uploads = 0
- let object = { generation: Number(generation), flags: { admitMode: 'reserve', readinessLocal: true } }
+ let object = { generation: Number(generation), flags }
let applied = object
const fetchImpl = async (url, init = {}) => {
const text = String(url)
@@ -557,10 +593,7 @@ function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = t
role: 'cell',
cellId: CELL,
flagsApplied: applied,
- supportedFlags: {
- admitMode: { type: 'enum', values: ['db', 'reserve'] },
- readinessLocal: { type: 'boolean' }
- },
+ supportedFlags,
admitModeEffective: applied.flags.admitMode,
runtime: { controls: 10 }
})
@@ -755,6 +788,111 @@ test('a quiet fleet with zero director 503s watches on and never trips', async (
}
})
+// 10-20 c29: a hand flip back to db mid-watch crashed the segment ("result.pages is not iterable").
+test('a segment that waits for reserve, then sees a hand flip back to db, stops without crashing', async () => {
+ const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+ try {
+ let clock = START
+ const config = parseAt(
+ watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'flipped-at': new Date(START - 2 * MINUTE).toISOString() })
+ )
+ const db = { flagsApplied: { generation: 5, flags: { admitMode: 'db', readinessLocal: true } }, admitModeEffective: 'db' }
+ const lines = []
+ const state = await watchSegment(config, {
+ ...fakeSources({ cell: (sample) => (sample === 1 || sample >= 4 ? db : {}) }),
+ idToken: 'id',
+ retryDelayMs: 0,
+ now: () => clock,
+ sleep: async (ms) => { clock += ms },
+ log: (line) => lines.push(JSON.parse(line))
+ })
+ assert.equal(lines[0].rules.reserve.status, 'pending')
+ assert.equal(state.state, 'stopped')
+ assert.match(state.verdict, /is not in reserve mode \(switch file says db\)/)
+ assert.equal(lines.length, 4)
+ } finally {
+ await rm(directory, { recursive: true, force: true })
+ }
+})
+
+// 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early.
+const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } })
+
+async function watchFor(minutes, sources, flippedAt = START - 2 * MINUTE) {
+ const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+ try {
+ let clock = START
+ const config = parseAt(
+ watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes), 'flipped-at': new Date(flippedAt).toISOString() })
+ )
+ const lines = []
+ const state = await watchSegment(config, {
+ ...sources,
+ idToken: 'id',
+ retryDelayMs: 0,
+ now: () => clock,
+ sleep: async (ms) => { clock += ms },
+ log: (line) => lines.push(JSON.parse(line))
+ })
+ return { state, lines }
+ } finally {
+ await rm(directory, { recursive: true, force: true })
+ }
+}
+
+test('a host re-dialling inside its own interval never trips the 503 rule, however much it grows', async () => {
+ // Background 20/min, all re-dials; then 50/min, 48 of them re-dials. All 503s would trip at 40.
+ const flippedAt = START - 2 * MINUTE
+ const { state, lines } = await watchFor(20, fakeSources({
+ per503: (minute) => (minute < flippedAt ? 20 : 50),
+ metrics: (at) => [rateLimited(at <= flippedAt ? 10 : 24)]
+ }), flippedAt)
+ assert.equal(state.state, 'watching')
+ assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' })
+ assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass'))
+ assert.ok(lines.slice(1).every((line) => Object.values(line.rules.director503.recent).every((count) => count <= 2)))
+})
+
+test('503s the director could not serve trip after three minutes over the line, beside the re-dials', async () => {
+ const flippedAt = START - 2 * MINUTE
+ const { state } = await watchFor(10, fakeSources({
+ per503: (minute) => (minute < flippedAt ? 20 : 60),
+ // 10 re-dials a minute throughout; the 40 more after the flip are queue waits that timed out.
+ metrics: (at) => [{
+ ...rateLimited(5),
+ stickyRejectionsByReasonDelta: at <= flippedAt ? {} : { 'wait-timeout': 20 }
+ }]
+ }), flippedAt)
+ assert.equal(state.state, 'tripped')
+ assert.deepEqual(state.background, { medianPerMinute: 10, minutes: 10, source: 'measured' })
+ assert.match(state.verdict, /director503: director 503s above 30\/min for 3 min \(50, 50, 50\)/)
+})
+
+test('a minute no director instance logged runtime metrics for is unverifiable, not zero', async () => {
+ // Two instances, both silent for three minutes in the middle of the read.
+ const endedAt = START + 2 * MINUTE
+ const silent = (at) => at > endedAt - 5 * MINUTE && at <= endedAt - 2 * MINUTE
+ const reader = fakeSources({ per503: () => 50, metrics: (at) => (silent(at) ? [] : [rateLimited(12), rateLimited(12)]) })
+ const read = await readDirector503Minutes(
+ { ...fakeReader(), fetch: reader.fetchImpl, runGcloud: reader.runGcloud },
+ { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 }
+ )
+ assert.equal(read.ok, true)
+ assert.deepEqual(read.unmetered, read.minutes.slice(5, 8))
+ assert.equal(read.perMinute[read.minutes[0]], 2)
+ const judged = evaluateWatchSample(freshState({ background: { medianPerMinute: 0, source: 'measured' } }), sampleAt(START, { director503: read }), LIMIT)
+ assert.equal(judged.rules.director503.status, 'unverifiable')
+ assert.match(judged.rules.director503.reason, /no director runtime metrics for/)
+ // No line at all in the read is short, not a quiet fleet.
+ const unlogged = fakeSources({ per503: () => 50, metrics: () => [] })
+ const none = await readDirector503Minutes(
+ { ...fakeReader(), fetch: unlogged.fetchImpl, runGcloud: unlogged.runGcloud },
+ { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 }
+ )
+ assert.equal(none.ok, false)
+ assert.match(none.error, /runtime metrics read failed, truncated, or short/)
+})
+
test('no request series at all is unverifiable and trips at the limit', async () => {
const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
try {
@@ -956,3 +1094,346 @@ test('an alarm is an escaped error annotation everywhere, and in write mode a pa
assert.match(lines.at(-2), /^::error title=Reserve watch refused for /)
assert.match(lines.at(-1), /nothing pages: --project-id is not a GCP project id/)
})
+
+// Step 6 phone rules. A quiet window, minute by minute: ordinary 4401s from phones whose pairing
+// was removed, an occasional failed confirm or half-open control. Synthetic: replaying a recorded
+// production window needs read access this suite does not have.
+const QUIET_PHONE_MINUTE = (index) => ({
+ credentialRefusals: [0, 1, 0, 2, 1][index % 5],
+ confirmFailures: [0, 0, 1, 0][index % 4],
+ attachDeadlines: [1, 0, 2, 0, 1, 3][index % 6],
+ acceptDatabaseErrors: 0,
+ renewalFailures: 0,
+ completedAccepts: [10, 12, 8][index % 3]
+})
+const phoneRead = (minuteCount, perMinute) => {
+ const minutes = Array.from({ length: minuteCount }, (_, index) => `2026-10-19T12:${String(index).padStart(2, '0')}`)
+ const rows = minutes.map((_, index) => perMinute(index))
+ const series = Object.fromEntries(Object.keys(rows[0]).map((signal) => [signal, rows.map((row) => row[signal])]))
+ return { ok: true, minutes, series, unmetered: [] }
+}
+const memoryCell = (overrides = {}) => healthyCell({ phonePathConfigured: 'memory', resumeAuditDropped: 0, ...overrides })
+const phoneState = (overrides = {}) =>
+ freshState({
+ flippedAt: START - 10 * MINUTE,
+ phoneSince: START - 10 * MINUTE,
+ phoneBackground: measuredPhoneBackground(phoneRead(10, QUIET_PHONE_MINUTE)),
+ ...overrides
+ })
+// The last 10 minutes as of watch minute `offset`, the newest minutes shaped by `spike`.
+const phoneSampleAt = (offset, spike = () => ({}), cell = memoryCell()) =>
+ sampleAt(START + offset * MINUTE, {
+ cell,
+ phone: phoneRead(10, (index) => ({ ...QUIET_PHONE_MINUTE(offset + index), ...spike(index) }))
+ })
+
+test('phone rules: a quiet hour replays without a trip', () => {
+ const results = run(Array.from({ length: 60 }, (_, offset) => phoneSampleAt(offset)), phoneState({ deadlineAt: START + 120 * MINUTE }))
+ assert.equal(results.length, 60)
+ assert.ok(results.every((result) => result.state.state === 'watching' && result.state.phoneTrip === null))
+ assert.deepEqual(Object.keys(results[0].rules).filter((name) => name.startsWith('phone')), PHONE_RULE_NAMES)
+})
+
+test('phone rules: judged as a share of attempts, over half and over the background', () => {
+ assert.equal(phoneShareLine(0), 0.5)
+ assert.equal(phoneShareLine(0.3), 0.55)
+ assert.equal(phoneShareLine(0.5), 0.6)
+ // Exactly half is not over; nor is a minute with under 4 attempts.
+ assert.equal(phoneMinuteOver('credentialRefusals', 9, 9, 0.5), false)
+ assert.equal(phoneMinuteOver('credentialRefusals', 11, 10, 0.5), true)
+ assert.equal(phoneMinuteOver('credentialRefusals', 3, 0, 0.5), false)
+ assert.equal(phoneMinuteOver('credentialRefusals', 4, 0, 0.5), true)
+ // Each accept makes one confirm: every confirm failing is all of them, not half the attempts.
+ assert.equal(phoneMinuteOver('confirmFailures', 4, 4, 0.6), true)
+ assert.equal(phoneMinuteOver('confirmFailures', 2, 4, 0.5), false)
+ assert.equal(phoneMinuteOver('confirmFailures', 9, 3, 0.5), false)
+ const background = measuredPhoneBackground(phoneRead(10, () => ({ ...QUIET_PHONE_MINUTE(0), credentialRefusals: 1, completedAccepts: 9 })))
+ assert.equal(background.share.credentialRefusals, 0.1)
+ assert.equal(measuredPhoneBackground({ ...phoneRead(10, QUIET_PHONE_MINUTE), unmetered: ['2026-10-19T12:03'] }), null)
+})
+
+test('phone rules: each fires on its own spike, and the watch goes on with the reserve rules', () => {
+ const spikes = {
+ phoneCredentialRefusals: { credentialRefusals: 25 },
+ phoneConfirmFailures: { confirmFailures: 25 },
+ phoneAttachDeadlines: { attachDeadlines: 25 },
+ phoneAcceptDatabaseErrors: { acceptDatabaseErrors: 25 }
+ }
+ for (const [rule, counts] of Object.entries(spikes)) {
+ const [result] = run([phoneSampleAt(0, (index) => (index >= 7 ? counts : {}))], phoneState())
+ assert.equal(result.state.state, 'watching', rule)
+ assert.match(result.state.phoneTrip, new RegExp(`turn off the phone path on ${CELL}: ${rule}: `), rule)
+ }
+ // Two minutes over the line is not three.
+ const [twoMinutes] = run([phoneSampleAt(0, (index) => (index >= 8 ? { credentialRefusals: 25 } : {}))], phoneState())
+ assert.equal(twoMinutes.state.phoneTrip, null)
+ // Once the watch turned the phone path off, the phone rules stand down.
+ const [off] = run([phoneSampleAt(0, () => ({ credentialRefusals: 50 }))], phoneState({ phoneOffAt: START }))
+ assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), [])
+})
+
+test('phone rules: judged only while the switch file says memory; reserve rules still flip the cell back', () => {
+ const [off] = run([sampleAt(START, { cell: healthyCell({ phonePathConfigured: 'db' }) })], freshState())
+ assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), [])
+ const [overrun] = run([phoneSampleAt(0, () => ({}), memoryCell({ enforcedUnits: 2_001 }))], phoneState())
+ assert.equal(overrun.state.state, 'tripped')
+ assert.equal(overrun.state.flip, 'admitMode')
+})
+
+test('phone rules: missing phone data turns off only the phone path, and needs a flipped-at background', () => {
+ const gap = (offset) => ({ ...phoneSampleAt(offset), phone: { ...phoneSampleAt(offset).phone, unmetered: ['2026-10-19T12:09'] } })
+ const results = run([gap(0), gap(1), gap(2)], phoneState())
+ assert.equal(results.at(-1).state.state, 'watching')
+ assert.equal(results.at(-1).state.unverifiableStreak, 0)
+ assert.match(results.at(-1).state.phoneTrip, /unverifiable: 3 samples/)
+ const [noBackground] = run([phoneSampleAt(0)], freshState({ flippedAt: null }))
+ assert.match(noBackground.rules.phoneCredentialRefusals.reason, /--flipped-at/)
+ // A gap before the judged 3 minutes changes nothing.
+ const early = { ...phoneSampleAt(0), phone: { ...phoneSampleAt(0).phone, unmetered: ['2026-10-19T12:02'] } }
+ const [earlyGap] = run([early], phoneState())
+ assert.equal(earlyGap.rules.phoneCredentialRefusals.status, 'pass')
+})
+
+test('phone rules: memory set partway through a watch measures from then, not from the reserve flip', () => {
+ const state = freshState({ flippedAt: START - 60 * MINUTE })
+ const [before, after] = run([sampleAt(START), phoneSampleAt(1)], state)
+ assert.equal(before.state.phoneSince, null)
+ assert.equal(after.state.phoneSince, START)
+ assert.equal(after.rules.phoneCredentialRefusals.status, 'pending')
+})
+
+test('phone rules: audit drops and failing renewals page once when sustained, and flip nothing', () => {
+ const dropping = (offset, dropped) => phoneSampleAt(offset, () => ({}), memoryCell({ resumeAuditDropped: dropped }))
+ // One drop, then none: no page.
+ const single = run([dropping(0, 0), dropping(1, 1), ...Array.from({ length: 8 }, (_, index) => dropping(index + 2, 1))], phoneState())
+ assert.ok(single.every((result) => result.pages.length === 0))
+ const rising = run(Array.from({ length: 9 }, (_, offset) => dropping(offset, offset * 3)), phoneState())
+ assert.deepEqual(rising.map((result) => result.pages.join()), ['', '', '', '', '', '', 'audit-drops', '', ''])
+ const renewing = run(Array.from({ length: 7 }, (_, offset) => phoneSampleAt(offset, () => ({ renewalFailures: 1 }))), phoneState())
+ assert.deepEqual(renewing.map((result) => result.pages.join()).filter(Boolean), ['renewal-failures'])
+ assert.ok([...rising, ...renewing].every((result) => result.state.state === 'watching' && result.state.phoneTrip === null))
+})
+
+test('the cell phone read sums each minute from the cell runtime metrics lines, and refuses an old image', async () => {
+ const endedAt = Date.parse('2026-10-19T12:03:00Z')
+ const line = (at, payload) => ({
+ timestamp: new Date(at).toISOString(),
+ jsonPayload: {
+ phoneRejectionsByCodeDelta: {},
+ resumeConfirmFailuresByCodeDelta: {},
+ phoneAttachDeadlinesDelta: 0,
+ phoneAcceptDatabaseErrorsDelta: 0,
+ resumeRenewalFailuresDelta: 0,
+ clientAcceptCompletedDelta: 0,
+ ...payload
+ }
+ })
+ let filter
+ const read = await readCellPhoneMinutes(
+ fakeReader({
+ runGcloud: async (args) => {
+ filter = args[2]
+ return {
+ stdout: JSON.stringify([
+ line(endedAt - 150_000, { phoneRejectionsByCodeDelta: { 4401: 2, 4404: 9 } }),
+ line(endedAt - 120_000, { resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1, error: 2 } }),
+ line(endedAt - 90_000, { phoneAttachDeadlinesDelta: 3, phoneAcceptDatabaseErrorsDelta: 1, resumeRenewalFailuresDelta: 4, clientAcceptCompletedDelta: 7 })
+ ])
+ }
+ }
+ }),
+ { cellId: CELL, endedAt, minutes: 3 }
+ )
+ assert.match(filter, /resource\.type="gce_instance"/)
+ assert.match(filter, new RegExp(`jsonPayload\\.cellId="${CELL}"`))
+ assert.deepEqual(read.series, {
+ credentialRefusals: [2, 0, 0],
+ confirmFailures: [3, 0, 0],
+ attachDeadlines: [0, 3, 0],
+ acceptDatabaseErrors: [0, 1, 0],
+ renewalFailures: [0, 4, 0],
+ completedAccepts: [0, 7, 0]
+ })
+ assert.deepEqual(read.unmetered, ['2026-10-19T12:02'])
+ const old = await readCellPhoneMinutes(
+ fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([{ timestamp: new Date(endedAt - 1).toISOString(), jsonPayload: {} }]) }) }),
+ { cellId: CELL, endedAt, minutes: 3 }
+ )
+ assert.match(old.error, /no phone counters/)
+ assert.equal(old.permanent, true)
+})
+
+// The watch over a memory cell whose phone metrics come from `phoneMetrics(at)`.
+async function phoneWatch(phoneMetrics, minutes) {
+ const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-'))
+ try {
+ let clock = START
+ const lines = []
+ const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes) }))
+ const sources = fakeSources({
+ cell: () => ({ flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } })
+ })
+ const runGcloud = async (args) =>
+ args[2]?.includes('gce_instance') ? { stdout: JSON.stringify(metricsLines(args[2], (at) => [phoneMetrics(at)])) } : sources.runGcloud(args)
+ const state = await watchSegment(config, {
+ ...sources,
+ runGcloud,
+ idToken: 'id',
+ accessToken: 'access',
+ audit: {},
+ retryDelayMs: 0,
+ now: () => clock,
+ sleep: async (ms) => { clock += ms },
+ log: (line) => lines.push(line)
+ })
+ return { state, lines }
+ } finally {
+ await rm(directory, { recursive: true, force: true })
+ }
+}
+const phoneCounters = (overrides = {}) => ({
+ phoneRejectionsByCodeDelta: {},
+ resumeConfirmFailuresByCodeDelta: {},
+ phoneAttachDeadlinesDelta: 0,
+ phoneAcceptDatabaseErrorsDelta: 0,
+ resumeRenewalFailuresDelta: 0,
+ clientAcceptCompletedDelta: 5,
+ ...overrides
+})
+
+test('a phone trip whose phonePath=db write fails keeps watching, retries each sample, and pages after 3 failures', async () => {
+ // No bucket here, so every write fails.
+ const { state, lines } = await phoneWatch((at) => phoneCounters(at > START ? { phoneAttachDeadlinesDelta: 20 } : {}), 15)
+ assert.equal(state.state, 'watching')
+ assert.equal(state.phoneOffAt, null)
+ assert.match(state.phoneTrip, /phoneAttachDeadlines: /)
+ const failures = lines.filter((line) => line.includes('orca_relay_reserve_watch_phone_path_off_failed'))
+ assert.ok(failures.length > 3, `${failures.length} failed writes`)
+ assert.equal(state.phoneOffFailures, failures.length)
+ const alarms = lines.filter((line) => line.includes('may still have phonePath=memory'))
+ assert.equal(alarms.length, 1)
+})
+
+test('a phone trip still unwritten when the watch ends leaves it to the flip back', () => {
+ const [last] = run([phoneSampleAt(0)], phoneState({ deadlineAt: START, phoneTrip: 'turn off the phone path on x: phoneAttachDeadlines: y' }))
+ assert.equal(last.state.state, 'tripped')
+ assert.equal(last.state.flip, 'phonePath')
+ assert.match(last.state.verdict, /phoneAttachDeadlines: y; phonePath=db did not land before the watch ended/)
+})
+
+test('a phone background broken by a restart or roll refuses the watch instead of turning phones off for certain', async () => {
+ const { state } = await phoneWatch((at) => (at < START - 8 * MINUTE ? {} : phoneCounters()), 5)
+ assert.equal(state.state, 'refused')
+ assert.match(state.verdict, /phone background.*incomplete.*later --flipped-at/)
+})
+
+test('a phone-rule flip back removes phonePath, and does nothing once the file no longer says memory', async () => {
+ const { fetchImpl, calls } = fakeFlipBack({ flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } })
+ const result = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fetchImpl, []))
+ assert.equal(result.written, true)
+ const upload = calls.find((call) => call.url.includes('/upload/storage/v1/'))
+ assert.match(upload.body, /"flags":\{"admitMode":"reserve","readinessLocal":true\}/)
+ assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode))
+ const done = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fakeFlipBack().fetchImpl, []))
+ assert.equal(done.alreadyDb, true)
+})
+
+test('both flip backs land on a cell rolled back to an image without the phonePath switch', async () => {
+ const flags = { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' }
+ for (const flip of ['admitMode', 'phonePath']) {
+ const { fetchImpl, calls } = fakeFlipBack({ flags, supportedFlags: { admitMode: { type: 'enum', values: ['db', 'reserve'] }, readinessLocal: { type: 'boolean' } } })
+ const result = await flipBack(flipValues({ flip }), flipDependencies(fetchImpl, []))
+ assert.equal(result.written, true, flip)
+ const upload = calls.find((call) => call.url.includes('/upload/storage/v1/'))
+ assert.doesNotMatch(upload.body, /phonePath/, flip)
+ }
+})
+
+test('workflow: the flip back writes what the watch says tripped, and the whole cell when it says nothing', () => {
+ const workflow = watchWorkflow
+ assert.match(workflow, /flip: \$\{\{ steps\.outcome\.outputs\.flip \}\}/)
+ assert.match(workflow, /FLIP: \$\{\{ needs\.watch\.outputs\.flip \|\| 'admitMode' \}\}/)
+ assert.match(workflow, /--flip "\$\{FLIP\}"/)
+ // A failed phone flip back pages with the phone wording.
+ assert.match(workflow, /--verdict "\$\{VERDICT\}" --flip "\$\{FLIP\}"\n/)
+ assert.match(workflow, /ORCA_RELAY_CONTROL_ACCESS_TOKEN="\$\(gcloud auth print-access-token\)"\n\s+echo "::add-mask::/)
+})
+
+
+test('a failed phone flip back and the phone pages say what is still on', async () => {
+ const lines = []
+ const runGcloud = async () => ({ stdout: '' })
+ await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x', flip: 'phonePath' }, { runGcloud, log: (line) => lines.push(line) })
+ await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'renewal-failures', reason: 'y' }, { runGcloud, log: (line) => lines.push(line) })
+ assert.match(lines[0], /may still have phonePath=memory/)
+ assert.doesNotMatch(lines[0], /reserve mode/)
+ assert.match(lines[1], /cannot renew phone credentials/)
+})
+
+// Recorded, not synthetic: 26 h of c7, c28 and c29 runtime metrics (read-only, 10-18/19). The phone
+// counters are not on those images yet, so auth failures (host and phone 4401s, an over-count)
+// stand in for the refused or failed attempts, against completed accepts. They cannot stand in for
+// confirm failures: c28's 04:08Z minutes (9/9, 9/9, 11/10) read as confirms are a total outage.
+const PHONE_RULE_SIGNAL_NAMES = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors']
+const recordedPhoneMinutes = JSON.parse(readFileSync(new URL('./relay-phone-watch-recorded-quiet-minutes.json', import.meta.url), 'utf8')).cells
+// The minute index a recorded watch of `signal` from `start` trips at, or null; `minute(i)` may rewrite a minute.
+function recordedTrip({ authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts }, signal, start, watchMinutes, minute = (i) => [failures[i], accepts[i]]) {
+ const { readMinutes, consecutiveMinutes } = PHONE_WATCH_RULES
+ const refused = failures.slice(start - readMinutes, start)
+ const background = measuredPhoneBackground({
+ ok: true,
+ minutes: refused,
+ unmetered: [],
+ series: { ...Object.fromEntries(PHONE_RULE_SIGNAL_NAMES.map((name) => [name, refused])), completedAccepts: accepts.slice(start - readMinutes, start) }
+ })
+ const line = phoneShareLine(background.share[signal])
+ let run = 0
+ for (let index = start; index < start + watchMinutes; index += 1) {
+ run = phoneMinuteOver(signal, ...minute(index), line) ? run + 1 : 0
+ if (run >= consecutiveMinutes) return index
+ }
+ return null
+}
+
+test('phone rules: 26 recorded hours of c7, c28 and c29 trip no attempt rule in any 60- or 300-minute watch', () => {
+ assert.deepEqual(Object.keys(recordedPhoneMinutes), ['production-gce-c7', 'production-gce-c28', 'production-gce-c29'])
+ for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) {
+ for (const watchMinutes of [60, 300]) {
+ for (let start = 10; start + watchMinutes <= minutes.authFailuresPerMinute.length; start += 5) {
+ assert.equal(recordedTrip(minutes, 'credentialRefusals', start, watchMinutes), null, `${cell}: a ${watchMinutes}-min watch from minute ${start} trips`)
+ }
+ }
+ }
+})
+
+// From minute 30 of each recorded hour, a broken path; it must trip by its third minute (sooner
+// when the minute before was already over) wherever those minutes carry 4 attempts each.
+const BROKEN_PATHS = {
+ 'two refusals per accept': ['credentialRefusals', (failed, accepted) => [Math.max(2 * accepted, 4), accepted]],
+ 'every accept errors on the database, phones give up after one try': ['acceptDatabaseErrors', (failed, accepted) => [accepted, 0]],
+ // Only trips are tested: the auth-failure stand-in cannot measure confirm false trips.
+ 'every confirm fails': ['confirmFailures', (failed, accepted) => [accepted, accepted]]
+}
+test('phone rules: the same recorded hours trip on each broken path wherever its minutes are judged', () => {
+ for (const [name, [signal, breakMinute]] of Object.entries(BROKEN_PATHS)) {
+ for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) {
+ const { authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts } = minutes
+ let judged = 0
+ for (let start = 10; start + 60 <= failures.length; start += 5) {
+ const broken = (index) => (index >= start + 30 ? breakMinute(failures[index], accepts[index]) : [failures[index], accepts[index]])
+ const judgedMinutes = [0, 1, 2].every((offset) => {
+ const [count, accepted] = broken(start + 30 + offset)
+ return (signal === 'confirmFailures' ? accepted : count + accepted) >= PHONE_WATCH_RULES.minAttemptsPerMinute
+ })
+ if (!judgedMinutes) continue
+ judged += 1
+ const at = recordedTrip(minutes, signal, start, 60, broken)
+ // Confirms may trip earlier: the stand-in reads as failing confirms on c28 (see above).
+ const earliest = signal === 'confirmFailures' ? start : start + 30
+ assert.ok(at !== null && at >= earliest && at <= start + 32, `${cell} ${name}: from minute ${start + 30} trips at ${at}`)
+ }
+ // Busy cells are judged most of the day; low-volume c7 rarely reaches 4 attempts a minute.
+ if (cell !== 'production-gce-c7') assert.ok(judged > 150, `${cell} ${name}: ${judged} judged`)
+ }
+ }
+})
diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md
index 5788ca6d3598..2e23041dc36f 100644
--- a/cloud/docs/orca-relay-operations.md
+++ b/cloud/docs/orca-relay-operations.md
@@ -241,7 +241,19 @@ Every later change must use the selector CAS.
Deploy the selector-compatible director before cutover. Blue/green deployment
creates a separate `selector-rollback` revision at minimum instances zero,
promotes a second compatible revision, and retains older revisions through the
-stabilization window. Cutover removes every older revision and refuses to
+stabilization window. With a floor above one, promotion first moves 20% of
+traffic and waits (up to six minutes) until every floor instance has served a
+request, because Cloud Run routes new floor instances only ~100 s after they
+start; a timeout returns all traffic to the previous serving revision.
+
+If the deploy job is cancelled or loses its runner during that wait, the 20/80
+split stays in place, and both the next deploy and the rehome workflow refuse a
+director without one 100% revision. Restore it with
+`gcloud run services update-traffic orca-cloud-relay --project onorca-cloud --region us-central1 --to-revisions==100`
+(the revision that held 100% before the deploy). A timed-out deploy restores
+that itself but removes every tag, including `selector-rollback`, so the rehome
+workflow's rollback-digest check refuses (`rollback tag missing`) until the next
+successful director deploy. Cutover removes every older revision and refuses to
proceed unless only the compatible active and rollback revisions remain.
First inspect the current generation and exact membership:
diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md
index edab96a42771..72b749b79de0 100644
--- a/cloud/docs/relay-workflows.md
+++ b/cloud/docs/relay-workflows.md
@@ -826,7 +826,7 @@ this order:
|---|---|---|
| Dead-man tripped | runtime-status `admitModeEffective=db` under a reserve switch file | Exact |
| Overrun | runtime-status `enforcedConnectionUnits` over `connectionCapacity.hardCap` | Invariant check: the ledger already refuses past its cap, so this trips only if that refusal broke. It counts seats, bookings, in-flight and phone units |
-| 503s | Director `request_count` 503s above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line |
+| 503s | Director non-drain 503s (the roll ladder's measure: `request_count` 503s less the drain-return deferrals and the `host-rate-limited` and `host-in-flight` re-dials the director's `orca_relay_runtime_metrics` count) above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line |
| Duplicate past grace | The ledger reconcile's `orca_relay_reserve_ledger_seat_behind_row` for the cell, reported again more than 90 s (60 s demotion close + 30 s map lag) after it was first seen, keyed by director instance, cell, epoch and row epoch | Proxy that can only over-count: the line names no host, so two hosts sharing that key read as one seat |
| Host drop | runtime-status `controls` fell more than 5% (and at least 10 hosts) within 5 minutes with no `draining` sample | Exact |
| Ledger drift | Queued plus dropped rows in each director's newest `orca_relay_reserve_ledger_summary`, over the cell's controls, above 0.5% for 2 minutes | Proxy that can only over-count: the summaries are fleet-wide, so exact only while this is the only reserve cell |
@@ -837,7 +837,15 @@ newest minute any code has a point for is how far the series is published: a 503
before it with no point had no 503s, so a quiet fleet reads zero. Minutes past it are left out, not
counted; fewer than 3 published minutes in the read, no request series at all, or an unparseable
timestamp is unverifiable. This assumes the codes publish together; one code lagging the others
-delays a trip by its lag but cannot hide one, because a later read sees its points. Until the
+delays a trip by its lag but cannot hide one, because a later read sees its points. A host's own
+early or duplicate re-dial says nothing about a cell and grows with traffic: on 2026-10-18 it was
+~98% of 503s and tripped c7 back twice with no fault. The subtraction is only trusted where the
+director logged: a judged minute no runtime-metrics line covers is unverifiable, a 10-minute read
+with fewer lines than a single instance logs (20) or at its entry limit is unverifiable, and the
+measured background leaves such minutes out (and is not measured with fewer than half of its 10
+metered). Coverage is any instance, not every one: an instance whose lines are missing while others
+log under-subtracts its re-dials, which can only push toward a trip, never hide one. The same measure is the
+background and `background-503-per-minute`. Until the
minutes before `flipped-at` are published (3 minutes past it) the 503 rule is pending, which does
not add to the streak. The duplicate read also takes `orca_relay_reserve_ledger_summary`, which
every placing director logs each minute, in the same query and window: with no summary from the
diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars
index d74abd85b75c..bebb51a9126a 100644
--- a/cloud/infra/terraform/environments/staging.tfvars
+++ b/cloud/infra/terraform/environments/staging.tfvars
@@ -48,8 +48,8 @@ relay_gce_cells = {
boot_disk_gb = 30
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7"
capacity_requests = 4000
- # Step-5 image G (main 2de73812, reports preAuthLatencyMs). Never 2d0f6e6: it cannot report capped telemetry.
- image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
+ # Step-6 image J (main 30f5bef9, phonePath). Never 2d0f6e6: it cannot report capped telemetry.
+ image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4"
# Step-5 staging e2e: c1 is the second US reserve cell, so it needs a cap and rehome trust.
connection_hard_cap = 600
connection_unobserved_bound = 60
@@ -86,7 +86,7 @@ relay_gce_cells = {
boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21"
capacity_requests = 6000
database_pool_max = 10
- image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d"
+ image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4"
initially_enabled = false
connection_hard_cap = 3000
connection_unobserved_bound = 60
diff --git a/cloud/infra/terraform/relay-staging-deploy-iam.tf b/cloud/infra/terraform/relay-staging-deploy-iam.tf
index a45652f2f682..74ee413f109b 100644
--- a/cloud/infra/terraform/relay-staging-deploy-iam.tf
+++ b/cloud/infra/terraform/relay-staging-deploy-iam.tf
@@ -170,3 +170,13 @@ resource "google_project_iam_member" "github_staging_relay_deploy_compute_viewer
role = "roles/compute.viewer"
member = google_service_account.github_staging_relay_deploy[0].member
}
+
+# Deploy Relay Staging step "Deploy director blue/green" with a director floor above one reads
+# the candidate's Cloud Run request log to confirm every floor instance serves before the full move.
+resource "google_project_iam_member" "github_staging_relay_deploy_logging_viewer" {
+ count = local.create_staging_relay_deploy_identity ? 1 : 0
+
+ project = var.project_id
+ role = "roles/logging.viewer"
+ member = google_service_account.github_staging_relay_deploy[0].member
+}
diff --git a/cloud/package.json b/cloud/package.json
index 6701f3859fd1..de5daabc60dc 100644
--- a/cloud/package.json
+++ b/cloud/package.json
@@ -17,7 +17,7 @@
"load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs",
"ops:relay": "pnpm --filter @orca-cloud/relay-ops dev",
"pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner",
- "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
+ "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-resume-phone.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs",
"typecheck": "pnpm -r typecheck",
"test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs"
},
@@ -30,7 +30,7 @@
},
"engines": {
"node": ">=24 <27",
- "pnpm": ">=10"
+ "pnpm": ">=12"
},
- "packageManager": "pnpm@10.34.6"
+ "packageManager": "pnpm@12.14.0+sha512.c18ba6d87993d3d164cb5e8948f32b2c06ee854487e06cd8d86781342fb3e5bb4634367034711e86802baa9c37bea906ce1929ad08d70884c97d801df13389a6"
}
diff --git a/cloud/packages/relay-contract/src/close-codes.ts b/cloud/packages/relay-contract/src/close-codes.ts
index cb2c6307b906..12c125e3fcaa 100644
--- a/cloud/packages/relay-contract/src/close-codes.ts
+++ b/cloud/packages/relay-contract/src/close-codes.ts
@@ -8,3 +8,15 @@ export const RELAY_CLOSE_CODE = {
} as const
export type RelayCloseCode = (typeof RELAY_CLOSE_CODE)[keyof typeof RELAY_CLOSE_CODE]
+
+// Mirror of MOBILE_RELAY_PHONE_CLOSE_REASON in src/shared/mobile-relay-close-codes.ts
+// in the Orca app repo half. Close reasons on a phone socket that refine its close
+// code; not RELAY_HOST_CLOSE_REASON, which names host control-socket closes.
+export const RELAY_PHONE_CLOSE_REASON = {
+ // A HOST_OFFLINE sent while the host's control looked alive but its data link
+ // missed hostAttachDeadlineMs; the phone redials the same cell once, quickly.
+ HOST_ATTACH_TIMEOUT: 'attach-timeout'
+} as const
+
+export type RelayPhoneCloseReason =
+ (typeof RELAY_PHONE_CLOSE_REASON)[keyof typeof RELAY_PHONE_CLOSE_REASON]
diff --git a/cloud/packages/relay-contract/src/contract.test.ts b/cloud/packages/relay-contract/src/contract.test.ts
index 391a0877c651..2a1a134ed6ae 100644
--- a/cloud/packages/relay-contract/src/contract.test.ts
+++ b/cloud/packages/relay-contract/src/contract.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest'
-import { RELAY_CLOSE_CODE } from './close-codes.js'
+import { RELAY_CLOSE_CODE, RELAY_PHONE_CLOSE_REASON } from './close-codes.js'
import {
AuthRefreshSchema,
DrainSchema,
@@ -52,6 +52,8 @@ describe('relay protocol contract', () => {
LIMIT_EXCEEDED: 4429,
DRAINING: 4503
})
+ // Shipped phones match this exact text; it is a wire value, not copy.
+ expect(RELAY_PHONE_CLOSE_REASON).toEqual({ HOST_ATTACH_TIMEOUT: 'attach-timeout' })
expect(RELAY_PROTOCOL_LIMITS).toMatchObject({
firstFrameDeadlineMs: 2_000,
maxHttpBodyBytes: 4_096,
diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml
index d57e00bd7312..ba788c076b27 100644
--- a/cloud/pnpm-lock.yaml
+++ b/cloud/pnpm-lock.yaml
@@ -1,3 +1,161 @@
+---
+lockfileVersion: '9.0'
+
+importers:
+
+ .:
+ configDependencies: {}
+ packageManagerDependencies:
+ pnpm:
+ specifier: 12.14.0
+ version: 12.14.0
+
+packages:
+
+ '@pnpm/exe.android-arm64@12.14.0':
+ resolution: {integrity: sha512-+iSh7x21C3A2hqc+Z0ZmbKTcn6YNlzrXM142PsnQH/dLj0DYxFzoBvc6XAnL/yfQ5nnH2ebBiRj/zVxr61fZbQ==}
+ cpu: [arm64]
+ os: [android]
+
+ '@pnpm/exe.android-x64@12.14.0':
+ resolution: {integrity: sha512-qT+0LqEr2wBVaxcG2T8cX/R8SSRz2Rvpp8nRR4fh8tnGUZRch87gFmsHUAoSQh7w6j1h0qTjrA384j4xi8frvw==}
+ cpu: [x64]
+ os: [android]
+
+ '@pnpm/exe.darwin-arm64@12.14.0':
+ resolution: {integrity: sha512-tnrdUB8ecJoehuKKqi1YHCBVl61xtJ9XVGcx5EviC6fWO4DeGI8GtXrjvJxDBM3nuVjJeWistgsUp/N142x4sw==}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@pnpm/exe.darwin-x64@12.14.0':
+ resolution: {integrity: sha512-vlBfGkruqQRsEHTW43cPwpYd7QlbIJ//cvpAdb3bVXkI8SgTPoXhCnc05m+ZNiPrrEkBsX/t7KHw9aXpkzPm3Q==}
+ cpu: [x64]
+ os: [darwin]
+
+ '@pnpm/exe.freebsd-x64@12.14.0':
+ resolution: {integrity: sha512-Qg5rxBrGYl6Tfso+EyygPf84fY1Fzy55E/0or+iuYKtnf0vc0MlHJfVIlT7pBukMRntQHRik7hUi0Z4s7AlJsw==}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@pnpm/exe.linux-arm64-musl@12.14.0':
+ resolution: {integrity: sha512-DRIe4yQkXBGA/PhVJdzlbnllxBdAtnMMvTVLLcGpsK5FvJz9kxD2Hyxtn68ovmNkVX06f6p45cQwfm3wx2jn1A==}
+ cpu: [arm64]
+ os: [linux]
+ libc: [musl]
+
+ '@pnpm/exe.linux-arm64@12.14.0':
+ resolution: {integrity: sha512-7joGShZ44Nk3WpNU+60JoOPjCXlgxhJAq11S2UiVtwCUDgTer1yTPPfxGW8pxaD6046actYLIYDbZNlubHWs0A==}
+ cpu: [arm64]
+ os: [linux]
+ libc: [glibc]
+
+ '@pnpm/exe.linux-ppc64@12.14.0':
+ resolution: {integrity: sha512-gSoryn7ySdXd//gjgf3MvvA8ZerJayaSbuMZMtAWq4vb2/wlVEgpl0Kvqt54y58TRj533Qjq2txKgnxd4JyoAQ==}
+ cpu: [ppc64]
+ os: [linux]
+ libc: [glibc]
+
+ '@pnpm/exe.linux-riscv64@12.14.0':
+ resolution: {integrity: sha512-EMjew1H9WqQ7pL+tYqCPwiuB+0PeCiJRV7fyWeuFw0wwYUajFBjcaPO3IGAVu9sdmJDU+e+HYv/Zey6aKae0Vw==}
+ cpu: [riscv64]
+ os: [linux]
+ libc: [glibc]
+
+ '@pnpm/exe.linux-s390x@12.14.0':
+ resolution: {integrity: sha512-shYOj/SFoXD35fHe0pWr2JofwSXulC0BnEIXLj2QokDi0EF//AfKsui35yKIkTEl+s2VJybc3PO4iJNhKIj5Qw==}
+ cpu: [s390x]
+ os: [linux]
+ libc: [glibc]
+
+ '@pnpm/exe.linux-x64-musl@12.14.0':
+ resolution: {integrity: sha512-btZiZ2Ot0llKK1ndSfA5i3IRX7/S0VrmzseNH/BJ4c+Cady+S81H6vvU1zByggHMyqQhC8OgxO5PiIY7D9bH0Q==}
+ cpu: [x64]
+ os: [linux]
+ libc: [musl]
+
+ '@pnpm/exe.linux-x64@12.14.0':
+ resolution: {integrity: sha512-F6K8e/jWxiuEYJaQtRRUG9ynBSTEHa+t7Lf86MKuOO5UA7WVVMbi+oB8nY6iKW++ncWS2yG6T9faXZGxqNsfoQ==}
+ cpu: [x64]
+ os: [linux]
+ libc: [glibc]
+
+ '@pnpm/exe.win32-arm64@12.14.0':
+ resolution: {integrity: sha512-DOjlkVB6TtTKeAAe0hNZXn5S7YqESmo1Z6SGbR4BZNQ4isRg31XVHjX2RFTbY8gnZq68IwzyLmPqlEjMmYxofg==}
+ cpu: [arm64]
+ os: [win32]
+
+ '@pnpm/exe.win32-x64@12.14.0':
+ resolution: {integrity: sha512-/LSX6mew/sLf08XqcjemuklNKf1K/Ftpgj4//hK/HKzI2dKbmxDB27a/2OaQ5a0O3/3OFzlssXDziZsi+yVBwQ==}
+ cpu: [x64]
+ os: [win32]
+
+ pnpm@12.14.0:
+ resolution: {integrity: sha512-wYum2HmT09Fky16JSPMrLAbuhUSH4GzY2GeBNC+z5btGNDZwNHEehoArqpw3vqkGzhkprQjXCITJfYAd8TOJpg==}
+ engines: {node: '>=18.*'}
+ hasBin: true
+
+snapshots:
+
+ '@pnpm/exe.android-arm64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.android-x64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.darwin-arm64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.darwin-x64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.freebsd-x64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-arm64-musl@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-arm64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-ppc64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-riscv64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-s390x@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-x64-musl@12.14.0':
+ optional: true
+
+ '@pnpm/exe.linux-x64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.win32-arm64@12.14.0':
+ optional: true
+
+ '@pnpm/exe.win32-x64@12.14.0':
+ optional: true
+
+ pnpm@12.14.0:
+ optionalDependencies:
+ '@pnpm/exe.android-arm64': 12.14.0
+ '@pnpm/exe.android-x64': 12.14.0
+ '@pnpm/exe.darwin-arm64': 12.14.0
+ '@pnpm/exe.darwin-x64': 12.14.0
+ '@pnpm/exe.freebsd-x64': 12.14.0
+ '@pnpm/exe.linux-arm64': 12.14.0
+ '@pnpm/exe.linux-arm64-musl': 12.14.0
+ '@pnpm/exe.linux-ppc64': 12.14.0
+ '@pnpm/exe.linux-riscv64': 12.14.0
+ '@pnpm/exe.linux-s390x': 12.14.0
+ '@pnpm/exe.linux-x64': 12.14.0
+ '@pnpm/exe.linux-x64-musl': 12.14.0
+ '@pnpm/exe.win32-arm64': 12.14.0
+ '@pnpm/exe.win32-x64': 12.14.0
+
+---
lockfileVersion: '9.0'
settings:
diff --git a/cloud/pnpm-workspace.yaml b/cloud/pnpm-workspace.yaml
index 97bf7cd8c2ef..af76ff32b474 100644
--- a/cloud/pnpm-workspace.yaml
+++ b/cloud/pnpm-workspace.yaml
@@ -3,3 +3,8 @@ packages:
- packages/*
minimumReleaseAge: 4320
+engineStrict: false
+
+# pnpm 10 skipped this unlisted build script; keep the binary-only install it produced.
+allowBuilds:
+ esbuild: false
diff --git a/config/audit-tools/package.json b/config/audit-tools/package.json
index 2d366a5484b0..52aae3d426fd 100644
--- a/config/audit-tools/package.json
+++ b/config/audit-tools/package.json
@@ -9,5 +9,5 @@
"engines": {
"node": "24"
},
- "packageManager": "pnpm@12.8.1+sha512.f64ba907507f5ceafe06c8d38e6052d0179444580ec1279ddd5bfc11cb48aa8a2644b66598e07e761da84872a9fc57d5f902b87fa49d024198d558612aabbe45"
+ "packageManager": "pnpm@12.14.0+sha512.c18ba6d87993d3d164cb5e8948f32b2c06ee854487e06cd8d86781342fb3e5bb4634367034711e86802baa9c37bea906ce1929ad08d70884c97d801df13389a6"
}
diff --git a/config/audit-tools/pnpm-lock.yaml b/config/audit-tools/pnpm-lock.yaml
index c9f420dc8035..8ad8aa84360e 100644
--- a/config/audit-tools/pnpm-lock.yaml
+++ b/config/audit-tools/pnpm-lock.yaml
@@ -7,153 +7,153 @@ importers:
configDependencies: {}
packageManagerDependencies:
pnpm:
- specifier: 12.8.1
- version: 12.8.1
+ specifier: 12.14.0
+ version: 12.14.0
packages:
- '@pnpm/exe.android-arm64@12.8.1':
- resolution: {integrity: sha512-siDxcidfz5eM6L2tJfik+i2bbUEw8UC7MZ3Yl2HYdRmqbdhHGAOQACtOOu6QVW4Jul4/w5wVKm+wM8ACtgh1Pg==}
+ '@pnpm/exe.android-arm64@12.14.0':
+ resolution: {integrity: sha512-+iSh7x21C3A2hqc+Z0ZmbKTcn6YNlzrXM142PsnQH/dLj0DYxFzoBvc6XAnL/yfQ5nnH2ebBiRj/zVxr61fZbQ==}
cpu: [arm64]
os: [android]
- '@pnpm/exe.android-x64@12.8.1':
- resolution: {integrity: sha512-0F/uqPRc3aN4vLHsa0f+6fTtmsigFQ9/yxEU43fionF//pNHEd5e97b3w/nKIgqYMCUQCJphmBRMA9Q8/wlo9Q==}
+ '@pnpm/exe.android-x64@12.14.0':
+ resolution: {integrity: sha512-qT+0LqEr2wBVaxcG2T8cX/R8SSRz2Rvpp8nRR4fh8tnGUZRch87gFmsHUAoSQh7w6j1h0qTjrA384j4xi8frvw==}
cpu: [x64]
os: [android]
- '@pnpm/exe.darwin-arm64@12.8.1':
- resolution: {integrity: sha512-/rwavvMQJsl2RIxOHBnDF+4Gk+fhQFAY4M3PQZoKskJOEnaJcygG9p1xby6wcQcbS4d9dIubv09CQhufTgfmpw==}
+ '@pnpm/exe.darwin-arm64@12.14.0':
+ resolution: {integrity: sha512-tnrdUB8ecJoehuKKqi1YHCBVl61xtJ9XVGcx5EviC6fWO4DeGI8GtXrjvJxDBM3nuVjJeWistgsUp/N142x4sw==}
cpu: [arm64]
os: [darwin]
- '@pnpm/exe.darwin-x64@12.8.1':
- resolution: {integrity: sha512-htYt2gN7zqJKLhC99Ft6EUiO7myjZAZXfmGCTJUL1v+o7vE9hZdE1+fAkk4Oj1ZHNRti6b2woOxEkRkAusi8OQ==}
+ '@pnpm/exe.darwin-x64@12.14.0':
+ resolution: {integrity: sha512-vlBfGkruqQRsEHTW43cPwpYd7QlbIJ//cvpAdb3bVXkI8SgTPoXhCnc05m+ZNiPrrEkBsX/t7KHw9aXpkzPm3Q==}
cpu: [x64]
os: [darwin]
- '@pnpm/exe.freebsd-x64@12.8.1':
- resolution: {integrity: sha512-PB5BULNgytxvExoOXUUeseS/DcpPENs+cg3iA6xHHeaLT4ctEGKSGXEbuj7USeqvo8JBpAx039hivnbzGqDwnA==}
+ '@pnpm/exe.freebsd-x64@12.14.0':
+ resolution: {integrity: sha512-Qg5rxBrGYl6Tfso+EyygPf84fY1Fzy55E/0or+iuYKtnf0vc0MlHJfVIlT7pBukMRntQHRik7hUi0Z4s7AlJsw==}
cpu: [x64]
os: [freebsd]
- '@pnpm/exe.linux-arm64-musl@12.8.1':
- resolution: {integrity: sha512-kKeyC/ArjrgdciQRBrOL0j9HtOI4gDRgoyP+eJ1CQKyadpKlf/DhDDyspXe07R3B9KagTVLUF2nlt6Dr5quFIg==}
+ '@pnpm/exe.linux-arm64-musl@12.14.0':
+ resolution: {integrity: sha512-DRIe4yQkXBGA/PhVJdzlbnllxBdAtnMMvTVLLcGpsK5FvJz9kxD2Hyxtn68ovmNkVX06f6p45cQwfm3wx2jn1A==}
cpu: [arm64]
os: [linux]
libc: [musl]
- '@pnpm/exe.linux-arm64@12.8.1':
- resolution: {integrity: sha512-q4s9a9X2O3N9aeUFooW24orNrxvu20+jyVUFR5RanPCqOKKPBrgs2iywMkBBx1aXkkzdWT54/mfz8Be8sJlWEw==}
+ '@pnpm/exe.linux-arm64@12.14.0':
+ resolution: {integrity: sha512-7joGShZ44Nk3WpNU+60JoOPjCXlgxhJAq11S2UiVtwCUDgTer1yTPPfxGW8pxaD6046actYLIYDbZNlubHWs0A==}
cpu: [arm64]
os: [linux]
libc: [glibc]
- '@pnpm/exe.linux-ppc64@12.8.1':
- resolution: {integrity: sha512-F7XSjKJthwCh/L6abZiMpAfo2Cm61SZIT3e4dfgiIjESndlWaXCnfLKngSgZ/SEMsdmHT6dud3+kWIB3tw0/xQ==}
+ '@pnpm/exe.linux-ppc64@12.14.0':
+ resolution: {integrity: sha512-gSoryn7ySdXd//gjgf3MvvA8ZerJayaSbuMZMtAWq4vb2/wlVEgpl0Kvqt54y58TRj533Qjq2txKgnxd4JyoAQ==}
cpu: [ppc64]
os: [linux]
libc: [glibc]
- '@pnpm/exe.linux-riscv64@12.8.1':
- resolution: {integrity: sha512-z1ecpIK/ZY08Unk69KWWE9867oaKUytGUjrHnBoGFAAhsK+q+0cfvBH+f5fWobJ7FX97leMlLO+gkJolddM45Q==}
+ '@pnpm/exe.linux-riscv64@12.14.0':
+ resolution: {integrity: sha512-EMjew1H9WqQ7pL+tYqCPwiuB+0PeCiJRV7fyWeuFw0wwYUajFBjcaPO3IGAVu9sdmJDU+e+HYv/Zey6aKae0Vw==}
cpu: [riscv64]
os: [linux]
libc: [glibc]
- '@pnpm/exe.linux-s390x@12.8.1':
- resolution: {integrity: sha512-KU6tnrBfu4uKXTq6N5qJwSm8FSrs9Wc8b5AUEsHK9bRXwNdsZ1/lRxG1++X29SKRW9OSNuDSjxASv5mAvGo1gA==}
+ '@pnpm/exe.linux-s390x@12.14.0':
+ resolution: {integrity: sha512-shYOj/SFoXD35fHe0pWr2JofwSXulC0BnEIXLj2QokDi0EF//AfKsui35yKIkTEl+s2VJybc3PO4iJNhKIj5Qw==}
cpu: [s390x]
os: [linux]
libc: [glibc]
- '@pnpm/exe.linux-x64-musl@12.8.1':
- resolution: {integrity: sha512-eUE8BWJkDr2tJb/yrk0yVnszlmNqFarVxUlqiZ9BAkct28Nq/5g2H0tvMDmPA7fAQW3STNj+omJPPRVT+4s5sQ==}
+ '@pnpm/exe.linux-x64-musl@12.14.0':
+ resolution: {integrity: sha512-btZiZ2Ot0llKK1ndSfA5i3IRX7/S0VrmzseNH/BJ4c+Cady+S81H6vvU1zByggHMyqQhC8OgxO5PiIY7D9bH0Q==}
cpu: [x64]
os: [linux]
libc: [musl]
- '@pnpm/exe.linux-x64@12.8.1':
- resolution: {integrity: sha512-8bDZ0lZlCdi2rvnFul7EYgcC7zKeWSe76y8JV2oXobaS8p9i9d6PSf6LgLtTM0fI7R9Oh4eLCbveXyNDMmvZ+g==}
+ '@pnpm/exe.linux-x64@12.14.0':
+ resolution: {integrity: sha512-F6K8e/jWxiuEYJaQtRRUG9ynBSTEHa+t7Lf86MKuOO5UA7WVVMbi+oB8nY6iKW++ncWS2yG6T9faXZGxqNsfoQ==}
cpu: [x64]
os: [linux]
libc: [glibc]
- '@pnpm/exe.win32-arm64@12.8.1':
- resolution: {integrity: sha512-MsT0dlrRxnCRCHCaosNbhWpEiAFnFuMEu3KuDJDU41BXgTwiINxU8goloUSUDejWZUqKaUqNC/qUTihl9nFtqg==}
+ '@pnpm/exe.win32-arm64@12.14.0':
+ resolution: {integrity: sha512-DOjlkVB6TtTKeAAe0hNZXn5S7YqESmo1Z6SGbR4BZNQ4isRg31XVHjX2RFTbY8gnZq68IwzyLmPqlEjMmYxofg==}
cpu: [arm64]
os: [win32]
- '@pnpm/exe.win32-x64@12.8.1':
- resolution: {integrity: sha512-SWtPe0PsbTTk//gJND10tMdOfCYjrkq5+qV49hzVhY7xz2iJ339Rc+xwASlcvbwTfrvH39YNqpKYibD+CRGLwA==}
+ '@pnpm/exe.win32-x64@12.14.0':
+ resolution: {integrity: sha512-/LSX6mew/sLf08XqcjemuklNKf1K/Ftpgj4//hK/HKzI2dKbmxDB27a/2OaQ5a0O3/3OFzlssXDziZsi+yVBwQ==}
cpu: [x64]
os: [win32]
- pnpm@12.8.1:
- resolution: {integrity: sha512-9kupB1B/XOr+BsjTjmBS0BeURFgOwSed3Vv8EctIqoomRLZlmOB+dh2oSHKp/FfV+QK4f6SdAkGY1VhhKqu+RQ==}
+ pnpm@12.14.0:
+ resolution: {integrity: sha512-wYum2HmT09Fky16JSPMrLAbuhUSH4GzY2GeBNC+z5btGNDZwNHEehoArqpw3vqkGzhkprQjXCITJfYAd8TOJpg==}
engines: {node: '>=18.*'}
hasBin: true
snapshots:
- '@pnpm/exe.android-arm64@12.8.1':
+ '@pnpm/exe.android-arm64@12.14.0':
optional: true
- '@pnpm/exe.android-x64@12.8.1':
+ '@pnpm/exe.android-x64@12.14.0':
optional: true
- '@pnpm/exe.darwin-arm64@12.8.1':
+ '@pnpm/exe.darwin-arm64@12.14.0':
optional: true
- '@pnpm/exe.darwin-x64@12.8.1':
+ '@pnpm/exe.darwin-x64@12.14.0':
optional: true
- '@pnpm/exe.freebsd-x64@12.8.1':
+ '@pnpm/exe.freebsd-x64@12.14.0':
optional: true
- '@pnpm/exe.linux-arm64-musl@12.8.1':
+ '@pnpm/exe.linux-arm64-musl@12.14.0':
optional: true
- '@pnpm/exe.linux-arm64@12.8.1':
+ '@pnpm/exe.linux-arm64@12.14.0':
optional: true
- '@pnpm/exe.linux-ppc64@12.8.1':
+ '@pnpm/exe.linux-ppc64@12.14.0':
optional: true
- '@pnpm/exe.linux-riscv64@12.8.1':
+ '@pnpm/exe.linux-riscv64@12.14.0':
optional: true
- '@pnpm/exe.linux-s390x@12.8.1':
+ '@pnpm/exe.linux-s390x@12.14.0':
optional: true
- '@pnpm/exe.linux-x64-musl@12.8.1':
+ '@pnpm/exe.linux-x64-musl@12.14.0':
optional: true
- '@pnpm/exe.linux-x64@12.8.1':
+ '@pnpm/exe.linux-x64@12.14.0':
optional: true
- '@pnpm/exe.win32-arm64@12.8.1':
+ '@pnpm/exe.win32-arm64@12.14.0':
optional: true
- '@pnpm/exe.win32-x64@12.8.1':
+ '@pnpm/exe.win32-x64@12.14.0':
optional: true
- pnpm@12.8.1:
+ pnpm@12.14.0:
optionalDependencies:
- '@pnpm/exe.android-arm64': 12.8.1
- '@pnpm/exe.android-x64': 12.8.1
- '@pnpm/exe.darwin-arm64': 12.8.1
- '@pnpm/exe.darwin-x64': 12.8.1
- '@pnpm/exe.freebsd-x64': 12.8.1
- '@pnpm/exe.linux-arm64': 12.8.1
- '@pnpm/exe.linux-arm64-musl': 12.8.1
- '@pnpm/exe.linux-ppc64': 12.8.1
- '@pnpm/exe.linux-riscv64': 12.8.1
- '@pnpm/exe.linux-s390x': 12.8.1
- '@pnpm/exe.linux-x64': 12.8.1
- '@pnpm/exe.linux-x64-musl': 12.8.1
- '@pnpm/exe.win32-arm64': 12.8.1
- '@pnpm/exe.win32-x64': 12.8.1
+ '@pnpm/exe.android-arm64': 12.14.0
+ '@pnpm/exe.android-x64': 12.14.0
+ '@pnpm/exe.darwin-arm64': 12.14.0
+ '@pnpm/exe.darwin-x64': 12.14.0
+ '@pnpm/exe.freebsd-x64': 12.14.0
+ '@pnpm/exe.linux-arm64': 12.14.0
+ '@pnpm/exe.linux-arm64-musl': 12.14.0
+ '@pnpm/exe.linux-ppc64': 12.14.0
+ '@pnpm/exe.linux-riscv64': 12.14.0
+ '@pnpm/exe.linux-s390x': 12.14.0
+ '@pnpm/exe.linux-x64': 12.14.0
+ '@pnpm/exe.linux-x64-musl': 12.14.0
+ '@pnpm/exe.win32-arm64': 12.14.0
+ '@pnpm/exe.win32-x64': 12.14.0
---
lockfileVersion: '9.0'
diff --git a/config/build-plugins/markdown-parser-exports.ts b/config/build-plugins/markdown-parser-exports.ts
index 2b468c0b02fe..40b545da1e4f 100644
--- a/config/build-plugins/markdown-parser-exports.ts
+++ b/config/build-plugins/markdown-parser-exports.ts
@@ -1,11 +1,16 @@
import { createRequire } from 'node:module'
const require = createRequire(import.meta.url)
-const parserRequire = createRequire(require.resolve('remark-parse'))
+// Why: walk the declared dependency chain; isolated installs don't hoist decode-named-character-reference next to remark-parse.
+const fromMarkdownRequire = createRequire(
+ createRequire(require.resolve('remark-parse')).resolve('mdast-util-from-markdown')
+)
const mathRequire = createRequire(require.resolve('rehype-katex'))
// Both preview paths use DOM-free parsing so their dependencies also run in workers.
export const markdownParserAliases = {
- 'decode-named-character-reference': parserRequire.resolve('decode-named-character-reference'),
+ 'decode-named-character-reference': fromMarkdownRequire.resolve(
+ 'decode-named-character-reference'
+ ),
'hast-util-from-html-isomorphic': mathRequire.resolve('hast-util-from-html-isomorphic')
}
diff --git a/config/dark-build-switches.cjs b/config/dark-build-switches.cjs
index 5b5cb5c02838..b46f40cb6864 100644
--- a/config/dark-build-switches.cjs
+++ b/config/dark-build-switches.cjs
@@ -2,8 +2,9 @@ const { existsSync } = require('node:fs')
const { join } = require('node:path')
// Dark build switches: a build compiles a switch's path in only when its variable is 'main'. Such a
-// main build leaves the switch's marker in out/main, and packaging refuses it; packaging and the
-// orcad template also refuse the variable itself.
+// main build leaves the switch's marker in out/main, the phone page bundle (out/mobile-web), out/orcad
+// and a --dark-test-template orcad template; packaging refuses any of them, and packaging and a
+// release orcad template also refuse the variable itself.
const DARK_BUILD_SWITCHES = [
// src/shared/layout-owner-switch.ts
{ variable: 'ORCA_LAYOUT_OWNER', off: 'window' },
@@ -37,7 +38,12 @@ function turnedOnDarkBuildDefines(env = process.env) {
)
}
-/** The marker each switched-on build leaves in out/main. */
+/** Every marker a switched-on build can leave, so a bundle check accounts for them. */
+function darkBuildMarkerFileNames() {
+ return DARK_BUILD_SWITCHES.map(markerFileName)
+}
+
+/** The marker each switched-on build leaves in its output. */
function darkBuildMarkers(env = process.env) {
return turnedOn(env).map((darkSwitch) => ({
fileName: markerFileName(darkSwitch),
@@ -45,14 +51,8 @@ function darkBuildMarkers(env = process.env) {
}))
}
-function assertNotDarkBuild(outputDirs, env = process.env) {
- const [asked] = turnedOn(env)
- if (asked) {
- throw new Error(
- `${asked.variable}=main is for dark test builds; unset it to package a release.`
- )
- }
- for (const darkSwitch of DARK_BUILD_SWITCHES) {
+function assertUnmarked(outputDirs, switches) {
+ for (const darkSwitch of switches) {
const marked = outputDirs.find((dir) => existsSync(join(dir, markerFileName(darkSwitch))))
if (marked) {
throw new Error(
@@ -62,9 +62,30 @@ function assertNotDarkBuild(outputDirs, env = process.env) {
}
}
+function assertNotDarkBuild(outputDirs, env = process.env) {
+ const [asked] = turnedOn(env)
+ if (asked) {
+ throw new Error(
+ `${asked.variable}=main is for dark test builds; unset it to package a release.`
+ )
+ }
+ assertUnmarked(outputDirs, DARK_BUILD_SWITCHES)
+}
+
+/** For a build that may itself be dark: refuses only outputs marked by a switch this build has off. */
+function assertNoDarkOutputOfOffSwitches(outputDirs, env = process.env) {
+ const on = turnedOn(env)
+ assertUnmarked(
+ outputDirs,
+ DARK_BUILD_SWITCHES.filter((darkSwitch) => !on.includes(darkSwitch))
+ )
+}
+
module.exports = {
+ assertNoDarkOutputOfOffSwitches,
assertNotDarkBuild,
darkBuildDefines,
+ darkBuildMarkerFileNames,
darkBuildMarkers,
turnedOnDarkBuildDefines
}
diff --git a/config/deprecated-runtime-owner-importers-baseline.txt b/config/deprecated-runtime-owner-importers-baseline.txt
deleted file mode 100644
index a8b94e02abf3..000000000000
--- a/config/deprecated-runtime-owner-importers-baseline.txt
+++ /dev/null
@@ -1,61 +0,0 @@
-# Renderer files that import getRuntimeEnvironmentIdForWorktree / resolveRuntimeOwnerForWorktree.
-# This is a RATCHET: the list may only SHRINK. Both resolvers read an ambiguous (two-host)
-# worktree as "not a runtime", which callers treat as local.
-# Prune after migrating a file: pnpm check:deprecated-runtime-owner-ratchet --prune
-src/renderer/src/components/browser-pane/assemble-chrome/browser-workspace-pane.tsx
-src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-lifecycle.ts
-src/renderer/src/components/browser-pane/workspace-doc/doc-preview-document-actions.ts
-src/renderer/src/components/browser-pane/workspace-doc/workspace-doc-page-pane.tsx
-src/renderer/src/components/native-chat/native-chat-file-link.ts
-src/renderer/src/components/native-chat/native-chat-http-link-source-owner.ts
-src/renderer/src/components/native-chat/native-chat-runtime-owner.ts
-src/renderer/src/components/right-sidebar/FileExplorer.tsx
-src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-controller-state.tsx
-src/renderer/src/components/right-sidebar/file-explorer-operation-owner.ts
-src/renderer/src/components/right-sidebar/file-explorer-row-context-menu.tsx
-src/renderer/src/components/right-sidebar/git-status-file-watch-refresh.ts
-src/renderer/src/components/right-sidebar/source-control/listing/entry-context-menu.tsx
-src/renderer/src/components/right-sidebar/use-file-explorer-tree-pane-state.ts
-src/renderer/src/components/right-sidebar/useFileSearchRunner.ts
-src/renderer/src/components/right-sidebar/useFileSearchScope.ts
-src/renderer/src/components/tab-bar/use-tab-bar-runtime-model.ts
-src/renderer/src/components/tab-bar/web-runtime-tab-move-mirror.ts
-src/renderer/src/components/tab-group/useTabGroupActivationCommands.ts
-src/renderer/src/components/tab-group/useTabGroupCreationCommands.ts
-src/renderer/src/components/terminal-pane/terminal-input-host-platform.ts
-src/renderer/src/components/terminal-pane/terminal-pane-menu-paste.ts
-src/renderer/src/components/terminal-pane/terminal-pane-paste-execution.ts
-src/renderer/src/components/terminal-pane/terminal-pane-paste-listeners.ts
-src/renderer/src/components/terminal-workspace-model.ts
-src/renderer/src/hooks/editor-external-watch-targets.ts
-src/renderer/src/hooks/ipc-events/browser-state-ipc-bridge.ts
-src/renderer/src/lib/activate-ai-vault-structured-session.ts
-src/renderer/src/lib/active-agent-note-send.ts
-src/renderer/src/lib/active-agent-note-target.ts
-src/renderer/src/lib/agent-hibernation-coordinator.ts
-src/renderer/src/lib/client-creation-action-policy.ts
-src/renderer/src/lib/doc-preview-grants.ts
-src/renderer/src/lib/execute-open-editor-path-move.ts
-src/renderer/src/lib/execution-host-facts.ts
-src/renderer/src/lib/file-preview.ts
-src/renderer/src/lib/launch-agent-background-session.ts
-src/renderer/src/lib/launch-agent-in-new-tab.ts
-src/renderer/src/lib/launch-ai-vault-session.ts
-src/renderer/src/lib/structured-agent-session-tab-activation.ts
-src/renderer/src/lib/tab-number-shortcuts.ts
-src/renderer/src/lib/terminal-tab-owner-activation.ts
-src/renderer/src/lib/terminal-worktree-route.ts
-src/renderer/src/lib/web-runtime-worktree-terminal-after-wake.ts
-src/renderer/src/lib/workspace-tab-palette-activation.ts
-src/renderer/src/lib/worktree-activation-pty-inventory.ts
-src/renderer/src/lib/worktree-activation.ts
-src/renderer/src/lib/worktree-creation-agent-seeds.ts
-src/renderer/src/lib/worktree-row-open-in-owner.ts
-src/renderer/src/lib/worktree-sort-order-host-split.ts
-src/renderer/src/store/slices/browser/browser-page-conversion-actions.ts
-src/renderer/src/store/slices/browser/browser-tab-actions.ts
-src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts
-src/renderer/src/store/slices/terminal-tab-retirement.ts
-src/renderer/src/store/terminals/terminal-active-workspace-creation.ts
-src/renderer/src/store/terminals/terminal-pane-hibernation.ts
-src/renderer/src/store/terminals/terminal-shutdown.ts
diff --git a/config/docker/headless-pairing/Dockerfile.build b/config/docker/headless-pairing/Dockerfile.build
index 0300d8a82d49..01ca56f235d0 100644
--- a/config/docker/headless-pairing/Dockerfile.build
+++ b/config/docker/headless-pairing/Dockerfile.build
@@ -26,7 +26,7 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
RUN corepack enable \
- && corepack prepare pnpm@12.8.1 --activate \
+ && corepack prepare pnpm@12.14.0 --activate \
&& pnpm --version
WORKDIR /workspace
diff --git a/config/docker/test-sandbox/Dockerfile b/config/docker/test-sandbox/Dockerfile
index 7e31d4d4c123..8fe1109b0520 100644
--- a/config/docker/test-sandbox/Dockerfile
+++ b/config/docker/test-sandbox/Dockerfile
@@ -86,7 +86,11 @@ RUN useradd --create-home --shell /bin/bash runner \
USER runner
WORKDIR /work
+# Why: the image is a throwaway per-shard install, so it keeps a project-local node_modules as CI does.
+ENV ORCA_PNPM_GLOBAL_VIRTUAL_STORE=0
+
COPY --chown=runner:runner package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY --chown=runner:runner config/pnpmfile.cjs ./config/pnpmfile.cjs
COPY --chown=runner:runner config/patches ./config/patches
# Why: pnpm-workspace.yaml lists these packages, so install fails without their manifests.
COPY --chown=runner:runner src/packages ./src/packages
diff --git a/config/e2e-typecheck-baseline.txt b/config/e2e-typecheck-baseline.txt
index c96721068268..e22e466ed325 100644
--- a/config/e2e-typecheck-baseline.txt
+++ b/config/e2e-typecheck-baseline.txt
@@ -27,8 +27,6 @@
1 tests/e2e/cross-version-wire/structured-agent-session-host-fixture.ts
1 tests/e2e/cross-version-wire/submission-positions-downgrade.unit.test.ts
1 tests/e2e/cross-version-wire/versioned-agent-session-wire.ts
-1 tests/e2e/daemon-generation-legacy-close-safety.spec.ts
-1 tests/e2e/daemon-generation-reconnect-safety.spec.ts
1 tests/e2e/editable-context-paste-ownership.spec.ts
1 tests/e2e/file-explorer-linked-watch-refresh.spec.ts
5 tests/e2e/fixtures/daemon-generation-legacy-close-client.ts
@@ -38,7 +36,6 @@
1 tests/e2e/github-created-issue-start-prefill.spec.ts
1 tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts
1 tests/e2e/headless-serve-desktop-activation.spec.ts
-1 tests/e2e/helpers/daemon-generation-safety-fixtures.ts
5 tests/e2e/helpers/headless-paired-runtime-host.ts
1 tests/e2e/helpers/native-chat-view.ts
2 tests/e2e/helpers/nested-runtime-ssh-client-route.ts
@@ -121,8 +118,6 @@
2 tests/e2e/terminal-tui-wheel-reports.spec.ts
1 tests/e2e/terminal-warm-reattach-split-live-output.spec.ts
1 tests/e2e/workspace-space-git-status.spec.ts
-1 tests/e2e/worktree-parent-reorder.spec.ts
-1 tests/e2e/worktree-scroll-to-current.spec.ts
1 tests/tools/omp-child-history-rendered/fixture.tsx
1 tests/tools/omp-fresh-launch-windows-smoke.ts
1 tests/tools/omp-remote-sidebar-rendered/fixture.tsx
diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs
index 5835807ed418..7f37e4f91829 100644
--- a/config/electron-builder.config.cjs
+++ b/config/electron-builder.config.cjs
@@ -66,6 +66,7 @@ const {
verifyAppImageNodeRuntimeServesPty
} = require('./scripts/appimage-node-runtime.cjs')
const { signWindowsUninstallerViaSignPath } = require('./scripts/windows-uninstaller-signing.cjs')
+const { buildMacShipItScheduling } = require('./scripts/macos-shipit-scheduling.cjs')
const isMacRelease = process.env.ORCA_MAC_RELEASE === '1'
const isLinuxArm64Release = process.env.ORCA_LINUX_ARM64_RELEASE === '1'
@@ -226,6 +227,8 @@ module.exports = {
'!out/runtimes{,/**/*}',
'!out/node-runtime-cache{,/**/*}',
'!config{,/**/*}',
+ // Cloud services deploy separately; their sources are not desktop runtime resources.
+ '!cloud{,/**/*}',
// Release archives and other build staging are not runtime resources.
'!.build{,/**/*}',
// Worktree-local Vite dependency caches are development inputs.
@@ -262,6 +265,8 @@ module.exports = {
'!out/**/*.map',
// Why: Vite's manifest is only used to project the paired web client.
'!out/renderer/.vite{,/**/*}',
+ // The installed browser serves its allowlisted assets from the unpacked renderer.
+ '!out/web{,/**/*}',
// Why: out/electron-dev caches `pnpm dev`'s per-branch Electron.app copies (~270MB each).
// CI never creates it, but packaging on a machine that has run dev would pack them all.
'!out/electron-dev{,/**/*}',
@@ -316,7 +321,7 @@ module.exports = {
// entries reached solely from the Electron main process stay packed, since
// asar redirects their app.asar paths.
// Why: `orca serve` runs the orcad selection entry under ELECTRON_RUN_AS_NODE too (#27307).
- // Why out/web and out/mobile-web: their assets are opened by path, and opening a packed file
+ // Why out/renderer and out/mobile-web: their assets are opened by path, and opening a packed file
// reads a /tmp extract Electron makes once and never remakes, so a /tmp sweep broke them (#27976).
asarUnpack: [
'out/package.json',
@@ -346,7 +351,7 @@ module.exports = {
'out/main/parcel-watcher-process-entry.js',
'out/main/orcad/orcad-local-serve-selection-entry.js',
'out/main/chunks/**',
- 'out/web/**',
+ 'out/renderer/**',
'out/mobile-web/**',
'resources/**',
'node_modules/ws/**',
@@ -364,9 +369,11 @@ module.exports = {
beforePack: (
context,
mobileWebBundleDir = MOBILE_WEB_BUNDLE_DIR,
- ensureWaylandClipboard = ensureBundledWaylandClipboard
+ ensureWaylandClipboard = ensureBundledWaylandClipboard,
+ orcadTemplateDir = join(__dirname, '..', orcadTemplateExtraResource.from)
) => {
- assertNotDarkBuild([join(__dirname, '..', 'out', 'main')])
+ // The template is the server every SSH host gets, so a dark test template must never ship.
+ assertNotDarkBuild([join(__dirname, '..', 'out', 'main'), mobileWebBundleDir, orcadTemplateDir])
assertPackagedNativeVariantsInstalled(context.electronPlatformName, context.arch)
assertBundledRipgrepInstalled()
assertProcessHostOutputBuilt()
@@ -399,6 +406,8 @@ module.exports = {
context.packager.appInfo.productFilename
)
const architecture = macSliceArch(context.arch, 'local-build compatibility')
+ const schedulingAddon = buildMacShipItScheduling(resourcesDir, architecture)
+ await signMacStandaloneHelper(schedulingAddon, 'ShipIt scheduling', context.packager)
const version = context.packager.appInfo.version
let commit = process.env.ORCA_BUILD_COMMIT || process.env.GITHUB_SHA || 'unknown'
if (commit === 'unknown') {
diff --git a/config/focus-setting-read-baseline.txt b/config/focus-setting-read-baseline.txt
index a77bfd21391f..789b4b189019 100644
--- a/config/focus-setting-read-baseline.txt
+++ b/config/focus-setting-read-baseline.txt
@@ -4,7 +4,7 @@
# the default host, through defaultCreationHost. Everything else routes by the owner.
# Prune after removing reads: pnpm check:focus-read-ratchet --prune
2 src/renderer/src/App.tsx
-5 src/renderer/src/app-shell/use-app-shell-services.ts
+3 src/renderer/src/app-shell/use-app-shell-services.ts
1 src/renderer/src/components/JiraIssueWorkspace.tsx
1 src/renderer/src/components/Landing.tsx
2 src/renderer/src/components/LinearIssueTextEditor.tsx
@@ -24,23 +24,16 @@
1 src/renderer/src/components/automations/use-automations-page-destination-form.ts
2 src/renderer/src/components/automations/use-automations-page-setup-state.ts
1 src/renderer/src/components/browser-pane/ReopenBrowserPageOnServerButton.tsx
-1 src/renderer/src/components/browser-pane/assemble-chrome/browser-egress-indicator.tsx
-1 src/renderer/src/components/browser-pane/assemble-chrome/browser-page-viewport-overlays.tsx
1 src/renderer/src/components/browser-pane/assemble-chrome/browser-workspace-pane.tsx
-1 src/renderer/src/components/browser-pane/assemble-chrome/ssh-routed-browser-page-gate.tsx
1 src/renderer/src/components/browser-pane/browser-reopen-on-server.ts
1 src/renderer/src/components/browser-pane/stream-remote/remote-browser-page-pane.tsx
1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-lifecycle.ts
1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-stream.ts
1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-stream-activation.ts
-2 src/renderer/src/components/browser-pane/use-ssh-workspace-browser-route.ts
-1 src/renderer/src/components/browser-pane/workspace-doc/HtmlDocPreview.tsx
1 src/renderer/src/components/browser-pane/workspace-doc/doc-preview-document-actions.ts
1 src/renderer/src/components/browser-pane/workspace-doc/workspace-doc-page-pane.tsx
2 src/renderer/src/components/cmd-j/quick-action-context.ts
-2 src/renderer/src/components/dashboard/dashboard-card-terminal-input.ts
-1 src/renderer/src/components/dashboard/launch-dashboard-agent.ts
-1 src/renderer/src/components/dashboard/use-agent-row-conversation-name.ts
+1 src/renderer/src/components/dashboard/dashboard-card-terminal-input.ts
2 src/renderer/src/components/editor/EditorContent.tsx
1 src/renderer/src/components/editor/EditorDiffFileSurface.tsx
1 src/renderer/src/components/editor/EditorEditFileSurface.tsx
@@ -76,21 +69,12 @@
1 src/renderer/src/components/linear-item-drawer-edit-section.tsx
2 src/renderer/src/components/native-chat-resume-modal.test-support.tsx
1 src/renderer/src/components/native-chat-resume-tree.test-support.ts
-1 src/renderer/src/components/native-chat/NativeChatAgentMessageSenders.tsx
1 src/renderer/src/components/native-chat/NativeChatResolvedView.tsx
1 src/renderer/src/components/native-chat/NativeChatVisualFrame.tsx
-1 src/renderer/src/components/native-chat/StructuredAgentSessionAttentionBridge.tsx
-1 src/renderer/src/components/native-chat/StructuredAgentSessionPaneOverlayLayer.tsx
-1 src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx
-5 src/renderer/src/components/native-chat/native-chat-agent-message-sender-label.ts
-2 src/renderer/src/components/native-chat/native-chat-attachment-upload.ts
1 src/renderer/src/components/native-chat/native-chat-file-link.ts
3 src/renderer/src/components/native-chat/native-chat-http-link-source-owner.ts
1 src/renderer/src/components/native-chat/native-chat-link-target-actions.ts
2 src/renderer/src/components/native-chat/native-chat-runtime-owner.ts
-1 src/renderer/src/components/native-chat/native-chat-skill-discovery-context.ts
-1 src/renderer/src/components/native-chat/use-native-chat-file-drops.ts
-2 src/renderer/src/components/native-chat/use-native-chat-workspace-file-drop.ts
1 src/renderer/src/components/new-workspace/use-jira-source-connection.ts
4 src/renderer/src/components/new-workspace/use-linear-source-connection.ts
1 src/renderer/src/components/new-workspace/use-smart-workspace-name-field-controller.ts
@@ -102,12 +86,9 @@
2 src/renderer/src/components/right-sidebar/FileExplorer.tsx
1 src/renderer/src/components/right-sidebar/FileExplorerFilesTreePane.tsx
1 src/renderer/src/components/right-sidebar/FolderWorkspacePrChecksRow.tsx
-1 src/renderer/src/components/right-sidebar/ai-vault-original-pane-actions.ts
-1 src/renderer/src/components/right-sidebar/ai-vault-session-resume-in-chat-workspace.ts
3 src/renderer/src/components/right-sidebar/checks-panel-hosted-review-click-routing.ts
1 src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-controller-state.tsx
1 src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-create-review.tsx
-1 src/renderer/src/components/right-sidebar/file-explorer-operation-owner.ts
1 src/renderer/src/components/right-sidebar/file-explorer-row-context-menu.tsx
1 src/renderer/src/components/right-sidebar/git-status-file-watch-refresh.ts
1 src/renderer/src/components/right-sidebar/source-control/listing/entry-context-menu.tsx
@@ -116,9 +97,9 @@
2 src/renderer/src/components/right-sidebar/use-file-explorer-tree-pane-state.ts
1 src/renderer/src/components/right-sidebar/useCreatePullRequestDialogFields.ts
4 src/renderer/src/components/right-sidebar/useFileSearchPanel.ts
-2 src/renderer/src/components/right-sidebar/useFileSearchRunner.ts
-2 src/renderer/src/components/right-sidebar/useFileSearchScope.ts
-2 src/renderer/src/components/right-sidebar/useGitStatusPolling.ts
+1 src/renderer/src/components/right-sidebar/useFileSearchRunner.ts
+1 src/renderer/src/components/right-sidebar/useFileSearchScope.ts
+1 src/renderer/src/components/right-sidebar/useGitStatusPolling.ts
1 src/renderer/src/components/settings/BaseRefPicker.tsx
2 src/renderer/src/components/settings/BrowserPane.tsx
1 src/renderer/src/components/settings/CliSkillRuntimeSetup.tsx
@@ -162,9 +143,8 @@
1 src/renderer/src/components/sidebar/SidebarTaskNavButton.tsx
1 src/renderer/src/components/sidebar/WorkspaceAttachmentSearch.tsx
1 src/renderer/src/components/sidebar/WorkspaceKanbanDrawer.tsx
-1 src/renderer/src/components/sidebar/WorktreeCardPorts.tsx
-2 src/renderer/src/components/sidebar/WorktreeList.tsx
-1 src/renderer/src/components/sidebar/WorktreeRowOpenInSubMenu.tsx
+1 src/renderer/src/components/sidebar/WorktreeList.tsx
+1 src/renderer/src/components/sidebar/WorktreeRowOpenInSubMenu.tsx # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down
1 src/renderer/src/components/sidebar/WorktreeVisibilityDialog.tsx
2 src/renderer/src/components/sidebar/delete-worktree-flow.ts
1 src/renderer/src/components/sidebar/linear-agent-skill-runtime.ts
@@ -191,15 +171,14 @@
1 src/renderer/src/components/sidebar/worktree-filter-visibility.ts
1 src/renderer/src/components/sidebar/worktree-list/listing/use-filters.ts
1 src/renderer/src/components/sidebar/worktree-list/listing/use-section-rows.ts
+1 src/renderer/src/components/sidebar/worktree-list/listing/use-window-list-model.ts
1 src/renderer/src/components/sidebar/worktree-list/rows/ProjectGroupDialogs.tsx
1 src/renderer/src/components/sidebar/worktree-visibility-host-target.ts
1 src/renderer/src/components/skills/SkillsPage.tsx
2 src/renderer/src/components/status-bar/ClaudeSwitcherMenu.tsx
1 src/renderer/src/components/status-bar/CodexSwitcherMenu.tsx
-1 src/renderer/src/components/status-bar/PortsStatusSegment.tsx
1 src/renderer/src/components/status-bar/SshStatusSegment.tsx
1 src/renderer/src/components/status-bar/StatusBarSurface.tsx
-1 src/renderer/src/components/status-bar/ports-status-popover-rows.tsx
2 src/renderer/src/components/status-bar/provider-account-sync-key.ts
1 src/renderer/src/components/status-bar/usage-provider-settings-target.ts
2 src/renderer/src/components/status-bar/use-codex-switcher-controller.ts
@@ -207,11 +186,9 @@
1 src/renderer/src/components/status-bar/use-workspace-space-git-refresh-action.ts
1 src/renderer/src/components/status-bar/use-workspace-space-manager-panel.ts
1 src/renderer/src/components/status-bar/workspace-space-decision-details.ts
-1 src/renderer/src/components/tab-bar/QuickLaunchButton.tsx
1 src/renderer/src/components/tab-bar/TabBar.tsx
-1 src/renderer/src/components/tab-bar/TabBarItemRow.tsx
3 src/renderer/src/components/tab-bar/tab-bar-windows-shell-launch-test-harness.ts
-4 src/renderer/src/components/tab-bar/use-tab-bar-runtime-model.ts
+3 src/renderer/src/components/tab-bar/use-tab-bar-runtime-model.ts
2 src/renderer/src/components/tab-bar/web-runtime-tab-move-mirror.ts
1 src/renderer/src/components/tab-bar/windows-shell-menu-visibility.ts
2 src/renderer/src/components/tab-group/useTabGroupActivationCommands.ts
@@ -221,25 +198,20 @@
9 src/renderer/src/components/task-page/TaskPage.tsx
2 src/renderer/src/components/task-page/jira/IssueAssigneeField.tsx
1 src/renderer/src/components/task-page/linear/Filters.tsx
-1 src/renderer/src/components/terminal-pane/TerminalPaneSurface.tsx
1 src/renderer/src/components/terminal-pane/pending-split-close-test-fixture.ts
1 src/renderer/src/components/terminal-pane/pty-connection/agent-idle-working-handlers.ts
-1 src/renderer/src/components/terminal-pane/pty-connection/interrupt-input-intent.ts
-1 src/renderer/src/components/terminal-pane/pty-connection/pane-agent-identity.ts
-1 src/renderer/src/components/terminal-pane/pty-connection/pane-host-capability-flags.ts
1 src/renderer/src/components/terminal-pane/retire-unbound-runtime-terminal-pane.ts
-1 src/renderer/src/components/terminal-pane/terminal-ctrl-arrow-conpty.ts
1 src/renderer/src/components/terminal-pane/terminal-fit-restore.ts
-2 src/renderer/src/components/terminal-pane/terminal-input-host-platform.ts
+1 src/renderer/src/components/terminal-pane/terminal-input-host-platform.ts
1 src/renderer/src/components/terminal-pane/terminal-keyboard-hook.ts
1 src/renderer/src/components/terminal-pane/terminal-keyboard-runtime.ts
1 src/renderer/src/components/terminal-pane/terminal-link-open-hints.ts
2 src/renderer/src/components/terminal-pane/terminal-pane-close-admission.ts
-1 src/renderer/src/components/terminal-pane/terminal-pane-manager-options.ts
2 src/renderer/src/components/terminal-pane/terminal-pane-menu-paste.ts
3 src/renderer/src/components/terminal-pane/terminal-pane-mount-preparation.ts
1 src/renderer/src/components/terminal-pane/terminal-pane-paste-execution.ts
1 src/renderer/src/components/terminal-pane/terminal-pane-paste-listeners.ts
+1 src/renderer/src/components/terminal-pane/terminal-paste-runtime.ts # the paste owner's legacy-fallback read for raw PTY ids (round 3 F5); F8 nets it down
2 src/renderer/src/components/terminal-pane/use-terminal-pane-close-actions.ts
2 src/renderer/src/components/terminal-pane/use-terminal-pane-controller.ts
1 src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts
@@ -285,9 +257,7 @@
1 src/renderer/src/hooks/ipc-events/tab-lifecycle-ipc-bridge.ts
1 src/renderer/src/hooks/ipc-events/terminal-request-ipc-bridge.ts
2 src/renderer/src/hooks/use-active-skill-discovery-runtime-target.ts
-1 src/renderer/src/hooks/use-terminal-quick-command-hosts.ts
1 src/renderer/src/hooks/useActiveProjectSkillRuntime.ts
-2 src/renderer/src/hooks/useAgentDetectionTarget.ts
1 src/renderer/src/hooks/useEditorExternalWatch.ts
1 src/renderer/src/hooks/useInstalledAgentSkills.ts
1 src/renderer/src/hooks/useIpcEvents.ts
@@ -296,29 +266,18 @@
7 src/renderer/src/hooks/useSettingsNavigationMetadata.ts
1 src/renderer/src/hooks/useUnreadDockBadge.ts
2 src/renderer/src/hooks/useWindowsTerminalCapabilityOwnerKey.ts
-4 src/renderer/src/lib/activate-ai-vault-structured-session.ts
+3 src/renderer/src/lib/activate-ai-vault-structured-session.ts
1 src/renderer/src/lib/active-agent-note-send.ts
1 src/renderer/src/lib/active-agent-note-target.ts
-1 src/renderer/src/lib/agent-account-restart-startup.ts
1 src/renderer/src/lib/agent-background-session-launch-host.ts
-1 src/renderer/src/lib/agent-detection-target-inventory.ts
1 src/renderer/src/lib/agent-hibernation-coordinator-test-fixture.ts
-1 src/renderer/src/lib/agent-hibernation-coordinator.ts
-2 src/renderer/src/lib/agent-launch-route-input.ts
-1 src/renderer/src/lib/agent-message-send.ts
-1 src/renderer/src/lib/ai-vault-resume-platform.ts
-1 src/renderer/src/lib/ai-vault-tab-title-requests.ts
-2 src/renderer/src/lib/ai-vault-tab-title-sync-inputs.ts
1 src/renderer/src/lib/client-creation-action-error.ts
3 src/renderer/src/lib/client-creation-action-policy.ts
1 src/renderer/src/lib/default-creation-host.ts # the sanctioned reader of the default host
-1 src/renderer/src/lib/doc-preview-grants.ts
-1 src/renderer/src/lib/empty-workspace-default-agent-chat.ts
-3 src/renderer/src/lib/execute-open-editor-path-move.ts
-1 src/renderer/src/lib/execution-host-display-label.ts
-2 src/renderer/src/lib/execution-host-facts.ts
-6 src/renderer/src/lib/file-preview.ts
-2 src/renderer/src/lib/folder-workspace-runtime-owner.ts
+1 src/renderer/src/lib/doc-preview-grants.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down
+1 src/renderer/src/lib/execute-open-editor-path-move.ts
+1 src/renderer/src/lib/execution-host-facts.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down
+5 src/renderer/src/lib/file-preview.ts
6 src/renderer/src/lib/http-link-destinations.ts
3 src/renderer/src/lib/http-link-routing.ts
1 src/renderer/src/lib/launch-agent-background-session.ts
@@ -326,40 +285,32 @@
2 src/renderer/src/lib/launch-agent-in-new-tab.ts
2 src/renderer/src/lib/launch-agent-web-host-tab.ts
2 src/renderer/src/lib/launch-ai-vault-session.ts
-1 src/renderer/src/lib/launch-structured-agent-session.ts
1 src/renderer/src/lib/linear-issue-url-lookup.ts
1 src/renderer/src/lib/local-preflight-context.ts
-3 src/renderer/src/lib/open-agent-message-sender.ts
2 src/renderer/src/lib/provider-runtime-context.ts
-2 src/renderer/src/lib/remap-open-editor-tabs-for-path-change.ts
1 src/renderer/src/lib/remote-workspace-drop-test-fixtures.ts
2 src/renderer/src/lib/repo-runtime-owner.ts
1 src/renderer/src/lib/runtime-session-mirror-owners.ts
1 src/renderer/src/lib/runtime-session-mirror-targets.ts
-1 src/renderer/src/lib/runtime-workspace-file-route.ts
1 src/renderer/src/lib/single-runtime-legacy-owner.ts
-1 src/renderer/src/lib/sleeping-agent-session-launch.ts
-1 src/renderer/src/lib/structured-agent-session-launch.ts
-1 src/renderer/src/lib/structured-agent-session-provisional-tab.ts
-4 src/renderer/src/lib/structured-agent-session-tab-activation.ts
+1 src/renderer/src/lib/structured-agent-session-tab-activation.ts
1 src/renderer/src/lib/tab-number-shortcuts.ts
1 src/renderer/src/lib/terminal-close-route.ts
1 src/renderer/src/lib/terminal-tab-owner-activation.ts
-5 src/renderer/src/lib/terminal-worktree-route.ts
+6 src/renderer/src/lib/terminal-worktree-route.ts # resolveTerminalPtyOwner: teardown, close and PTY identity read spawn's attribution (round 3 F8), which retired the deprecated resolver's readers
3 src/renderer/src/lib/unread-badge-count-selector.ts
2 src/renderer/src/lib/web-runtime-worktree-terminal-after-wake.ts
-5 src/renderer/src/lib/workspace-browser-tab-open.ts
-1 src/renderer/src/lib/workspace-file-drag-source.ts
+4 src/renderer/src/lib/workspace-browser-tab-open.ts
+1 src/renderer/src/lib/workspace-file-browser-open-target.ts # moved out of file-preview.ts (round 3 F7), same read
1 src/renderer/src/lib/workspace-tab-palette-activation.ts
-1 src/renderer/src/lib/worktree-activation-pty-inventory.ts
-3 src/renderer/src/lib/worktree-activation.ts
+1 src/renderer/src/lib/worktree-activation-pty-inventory.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down
+2 src/renderer/src/lib/worktree-activation.ts
1 src/renderer/src/lib/worktree-creation-agent-seeds.ts
1 src/renderer/src/lib/worktree-jump-navigation.ts
-2 src/renderer/src/lib/worktree-operation-route.ts
-1 src/renderer/src/lib/worktree-row-open-in-owner.ts
-6 src/renderer/src/lib/worktree-runtime-owner.ts # typed owner resolver plus its null-collapsing wrapper; callers net -3
+1 src/renderer/src/lib/worktree-row-open-in-owner.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down
+3 src/renderer/src/lib/worktree-runtime-owner.ts # typed owner resolver and the legacy-fallback adapter the folds moved to (round 3 F0); F8 deleted the deprecated pair
1 src/renderer/src/lib/worktree-sort-order-host-split.ts
-1 src/renderer/src/runtime/local-structured-session-tabs-sync/snapshot-apply.ts
+1 src/renderer/src/lib/worktree-sort-order-persistence.ts # its split helper calls resolveOwnerWithLegacyFallback directly (round 3 F4); F8 nets it down
17 src/renderer/src/runtime/runtime-jira-client.ts
2 src/renderer/src/runtime/runtime-jira-summary-client.ts
1 src/renderer/src/runtime/runtime-jira-target.ts
@@ -367,10 +318,7 @@
8 src/renderer/src/runtime/runtime-linear-client.ts
6 src/renderer/src/runtime/runtime-linear-issue-mutations.ts
12 src/renderer/src/runtime/runtime-linear-project-client.ts
-3 src/renderer/src/runtime/structured-agent-session-owner.ts
-1 src/renderer/src/runtime/structured-conversation-name.ts
5 src/renderer/src/runtime/use-runtime-session-mirror-environment-key.ts
-1 src/renderer/src/runtime/use-worktree-runtime-target.ts
2 src/renderer/src/runtime/web-runtime-browser-creation.ts
1 src/renderer/src/runtime/web-runtime-initial-terminal-bootstrap-dispatch.ts
1 src/renderer/src/runtime/web-runtime-session-tab-lifecycle.ts
@@ -382,33 +330,27 @@
2 src/renderer/src/runtime/web-session-tabs-sync/global-session-subscription.ts
1 src/renderer/src/runtime/web-session-tabs-sync/tracking-decisions.ts
3 src/renderer/src/runtime/web-session-tabs-sync/use-web-session-tabs-sync.ts
-1 src/renderer/src/store/folder-workspaces/folder-workspace-mutations.ts
1 src/renderer/src/store/github/repository-routing.ts
2 src/renderer/src/store/github/visible-hosted-review-refresh-targets.ts
2 src/renderer/src/store/github/work-item-aggregate-actions.ts
3 src/renderer/src/store/github/work-item-fetch-actions.ts
2 src/renderer/src/store/github/work-item-routing.ts
9 src/renderer/src/store/index.ts
-3 src/renderer/src/store/project-groups/project-group-mutations.ts
+2 src/renderer/src/store/project-groups/project-group-mutations.ts
1 src/renderer/src/store/repos/all-host-repo-catalog-actions.ts
-1 src/renderer/src/store/repos/converted-ssh-browser-pages.ts
2 src/renderer/src/store/repos/repo-add-actions.ts
1 src/renderer/src/store/repos/repo-update.ts
1 src/renderer/src/store/repos/runtime-repo-catalog-actions.ts
1 src/renderer/src/store/slices/browser-slice-test-harness.ts
-1 src/renderer/src/store/slices/browser/browser-close-actions.ts
16 src/renderer/src/store/slices/browser/browser-cookie-import-actions.ts
1 src/renderer/src/store/slices/browser/browser-host-actions.ts
-10 src/renderer/src/store/slices/browser/browser-host-state.ts
-2 src/renderer/src/store/slices/browser/browser-page-conversion-actions.ts
-1 src/renderer/src/store/slices/browser/browser-page-focus-actions.ts
+7 src/renderer/src/store/slices/browser/browser-host-state.ts
+1 src/renderer/src/store/slices/browser/browser-page-conversion-actions.ts
14 src/renderer/src/store/slices/browser/browser-profile-import-actions.ts
14 src/renderer/src/store/slices/browser/browser-profile-list-actions.ts
-7 src/renderer/src/store/slices/browser/browser-tab-actions.ts
-1 src/renderer/src/store/slices/browser/browser-tab-focus-actions.ts
+6 src/renderer/src/store/slices/browser/browser-tab-actions.ts
1 src/renderer/src/store/slices/browser/create-browser-slice.ts
1 src/renderer/src/store/slices/browser/paired-browser-tab-creator.ts
-3 src/renderer/src/store/slices/diff-comment-persistence.ts
2 src/renderer/src/store/slices/editor-slice-test-harness.ts
1 src/renderer/src/store/slices/editor/actions/markdown-link-action.ts
1 src/renderer/src/store/slices/editor/actions/markdown-preview-actions.ts
@@ -441,14 +383,13 @@
2 src/renderer/src/store/slices/preflight.ts
1 src/renderer/src/store/slices/repo-host-identity.ts
1 src/renderer/src/store/slices/repo-reorder-host-split.ts
-4 src/renderer/src/store/slices/repos.ts
+3 src/renderer/src/store/slices/repos.ts
1 src/renderer/src/store/slices/runtime-status.ts
2 src/renderer/src/store/slices/settings-owner-hydration-publication.ts
5 src/renderer/src/store/slices/settings.ts
8 src/renderer/src/store/slices/store-test-helpers.ts
-1 src/renderer/src/store/slices/tabs/tabs-close-actions.ts
2 src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts
-2 src/renderer/src/store/slices/terminal-tab-retirement.ts
+1 src/renderer/src/store/slices/terminal-tab-retirement.ts
3 src/renderer/src/store/slices/workspace-attention/workspace-unread-characterization-fixture.ts
1 src/renderer/src/store/slices/worktree-visibility-owner-settings.ts
3 src/renderer/src/store/slices/worktree-visibility-settings-write.ts
@@ -462,12 +403,8 @@
2 src/renderer/src/store/slices/worktrees/metadata/update-worktree-meta.ts
1 src/renderer/src/store/slices/worktrees/metadata/worktree-lineage-actions.ts
3 src/renderer/src/store/slices/worktrees/metadata/worktree-lineage-host-merge.ts
-1 src/renderer/src/store/slices/worktrees/metadata/worktree-lineage-refresh.ts
3 src/renderer/src/store/slices/worktrees/session/worktree-terminal-reference.ts
-1 src/renderer/src/store/slices/worktrees/teardown/purged-structured-launch-retirement.ts
2 src/renderer/src/store/terminals/terminal-active-workspace-creation.ts
-1 src/renderer/src/store/terminals/terminal-pane-hibernation.ts
-1 src/renderer/src/store/terminals/terminal-shutdown.ts
1 src/renderer/src/store/terminals/terminal-tab-appearance.ts
1 src/renderer/src/store/terminals/terminal-tab-close-providers.ts
1 src/renderer/src/web/preload-api/web-app-api.ts
diff --git a/config/fork-ownership.json b/config/fork-ownership.json
index fb20e8387de8..24ee0895da10 100644
--- a/config/fork-ownership.json
+++ b/config/fork-ownership.json
@@ -585,7 +585,7 @@
"kind": "registration",
"lines": [
"import { createHeimdallFleetSlice } from './slices/fork-heimdall/fleet'",
- " ...createHeimdallFleetSlice(...a)"
+ " ...createHeimdallFleetSlice(...a)"
]
},
{
@@ -641,6 +641,14 @@
" wireHeimdallFleetWindows(runtime, () => state.mainWindow)"
]
},
+ {
+ "path": "src/main/startup/main-process-ready-duplicate-pty-repair.test.ts",
+ "feature": "heimdall",
+ "kind": "registration",
+ "lines": [
+ " ipcMain: { handle: vi.fn(), removeHandler: vi.fn(), on: vi.fn(), removeListener: vi.fn() },"
+ ]
+ },
{
"path": "src/main/startup/main-process-state.ts",
"feature": "heimdall",
@@ -1367,7 +1375,7 @@
]
},
{
- "path": "src/renderer/src/components/native-chat/native-chat-picker-items.ts",
+ "path": "src/shared/native-chat-picker-items.ts",
"feature": "skill-plugin-attribution",
"kind": "registration",
"lines": [" pluginName?: string", " namespacePluginSkills = false"]
@@ -1519,8 +1527,8 @@
"feature": "worktree-groups",
"kind": "passthrough",
"lines": [
- " groupMembershipDrag: ReturnType",
- " trackWorktreeGroupMembershipDragFrame: groupMembershipDrag.trackPointerDragFrame,"
+ " groupMembershipDrag?: ReturnType",
+ " trackWorktreeGroupMembershipDragFrame: groupMembershipDrag?.trackPointerDragFrame,"
]
},
{
@@ -1547,7 +1555,7 @@
"kind": "passthrough",
"lines": [
" worktreeGroupMembershipDragPreview: WorktreeGroupMembershipDragPreview",
- " 'group relative flex h-6 w-full items-center gap-1.5 pr-2 text-left transition-all',",
+ " 'group relative flex h-6 w-full items-center gap-1.5 pr-2 text-left transition-all',",
" sectionSeparatorClass?: string | false",
" args.sectionSeparatorClass,"
]
@@ -2509,7 +2517,7 @@
"kind": "registration",
"lines": [
"import { ForkAiVaultLineageLine } from '@/components/agent-session-continuation/fork-session-handoff/ForkAiVaultLineageLine'",
- " "
+ " "
]
},
{
@@ -2923,13 +2931,10 @@
"kind": "registration",
"lines": [
"import { app, ipcMain } from 'electron'",
- "import { forwardAskEventsToRenderer } from '../fork-ask-question-tool/ask-ipc-forward'",
+ "import { forwardRuntimeAskEventsToRenderer } from '../fork-ask-question-tool/ask-ipc-forward'",
"import { startClaudeSuppressionVerdictPublisher } from '../fork-ask-question-tool/claude-suppression-verdict-publisher'",
"import { getDashboardPopoutWindow } from '../window/dashboard-popout-window'",
- " forwardAskEventsToRenderer(",
- " runtime.getAskServices().registry,",
- " () => state.mainWindow,",
- " getDashboardPopoutWindow",
+ " forwardRuntimeAskEventsToRenderer(runtime, () => state.mainWindow, getDashboardPopoutWindow)",
" startClaudeSuppressionVerdictPublisher({",
" store,",
" ipcMain,",
@@ -2975,7 +2980,7 @@
"kind": "registration",
"lines": [
"import { createAsksSlice } from './slices/fork-ask-question-tool/asks'",
- " ...createAsksSlice(...a),"
+ " ...createAsksSlice(...a),"
]
},
{
@@ -3257,6 +3262,27 @@
"import { WorkspaceActivityHiddenCountRow } from './fork-workspace-activity-window/WorkspaceActivityHiddenCountRow'"
]
},
+ {
+ "path": "src/renderer/src/components/sidebar/worktree-list/listing/sidebar-list-model.ts",
+ "feature": "workspace-activity-window",
+ "kind": "passthrough",
+ "lines": [" activityHiddenCount?: number"]
+ },
+ {
+ "path": "src/renderer/src/components/sidebar/worktree-list/listing/use-main-list-model.ts",
+ "feature": "workspace-activity-window",
+ "kind": "passthrough",
+ "lines": [" activityHiddenFolderCount: 0"]
+ },
+ {
+ "path": "src/renderer/src/components/sidebar/worktree-list/listing/use-window-list-model.ts",
+ "feature": "workspace-activity-window",
+ "kind": "passthrough",
+ "lines": [
+ " const { visibleWorktrees, activityHiddenCount, pairedDeviceIdsByEnvironment } =",
+ " return { visibleWorktrees, visibleScope, externalWorktreeCards, rowModel, activityHiddenCount }"
+ ]
+ },
{
"path": "src/renderer/src/components/sidebar/sidebar-filter-actions.ts",
"feature": "workspace-activity-window",
@@ -3560,7 +3586,7 @@
]
},
{
- "path": "src/renderer/src/components/tab-bar/SortableTab.tsx",
+ "path": "src/renderer/src/components/tab-bar/use-terminal-tab-activity.ts",
"feature": "ask-question-tool",
"kind": "registration",
"lines": [
@@ -4109,21 +4135,13 @@
]
},
{
- "path": "src/main/runtime/orca-runtime.ts",
+ "path": "src/main/runtime/orca-runtime-agent-transitions.ts",
"feature": "project-ledger",
"kind": "import-swap",
"lines": [
"import { OrcaRuntimeWithLedger as OrcaRuntimeWithMigrationCatalog } from './orca-runtime-ledger'"
]
},
- {
- "path": "src/main/runtime/sidebar-inputs/orca-runtime-workspace-activity-facts.ts",
- "feature": "project-ledger",
- "kind": "import-swap",
- "lines": [
- "import { OrcaRuntimeWithLedger as OrcaRuntimeWithMigrationCatalog } from '../orca-runtime-ledger'"
- ]
- },
{
"path": "src/renderer/src/components/sidebar/worktree-list/viewport/viewport-props.ts",
"feature": "project-ledger",
@@ -5571,7 +5589,7 @@
"kind": "registration",
"lines": [
"import { createLoopsSlice } from '@/fork-loops/loops-slice'",
- " ...createLoopsSlice(...a),"
+ " ...createLoopsSlice(...a),"
]
},
{
@@ -5888,11 +5906,26 @@
"reason": "Tests for the fork's release-tooling delta above.",
"status": "permanent"
},
+ {
+ "path": "config/scripts/mac-notarization-builder-contract.test.mjs",
+ "status": "permanent",
+ "reason": "Asserts the macOS release channel runs on Blacksmith through the dual-arch notarization-overlap builder; the fork's release channel builds arm64 only on a hosted runner, so its release case asserts that shape instead."
+ },
+ {
+ "path": "config/scripts/ci-managed-ssh-execution.test.mjs",
+ "status": "permanent",
+ "reason": "Contracts for the fork-owned PR job graph: this fork's PR workflow has no e2e caller, so the shard case feeds the lane the selector's output directly and asserts the caller's absence."
+ },
{
"path": "config/scripts/release-cut-token-permissions.test.mjs",
"status": "permanent",
"reason": "Asserts upstream's release-cut job matrix and github.repository == 'stablyai/orca'; the fork publishes macOS-only arm64 artifacts from zpyoung/orca, so upstream's version can never describe this pipeline."
},
+ {
+ "path": "config/scripts/release-windows-signing-gate.test.mjs",
+ "reason": "Upstream's Windows signing-gate contract, kept against the fork's release-cut.yml: the fork's Windows/Linux build job is disabled, so it asserts if: false where upstream asserts the build-mac success condition.",
+ "status": "permanent"
+ },
{
"path": "config/scripts/release-rc-history.mjs",
"reason": "Fork release-tooling delta for the fork's own RC history.",
@@ -6607,6 +6640,11 @@
"reason": "Upstream's paste-remount coverage, run against the fork composer field and actions (textarea, not upstream's rich-text editor).",
"status": "permanent"
},
+ {
+ "path": "src/renderer/src/components/native-chat/NativeChatComposer.resume-command.test.tsx",
+ "reason": "Upstream's /resume coverage, run against the fork composer field, keydown and draft hooks (upstream's mocks target components the fork composer does not render).",
+ "status": "permanent"
+ },
{
"path": "src/renderer/src/components/native-chat/native-chat-composer-prompt-recall.test.tsx",
"reason": "Exercises upstream's transcript prompt recall (useNativeChatComposerRecall), which the fork composer replaces with its own per-pane history; the fork's equivalent coverage is in fork-agent-composer.",
@@ -6701,6 +6739,10 @@
}
],
"residuals": {
+ "src/main/startup/main-process-ready-duplicate-pty-repair.test.ts": {
+ "added": 1,
+ "removed": 0
+ },
"tests/e2e/native-chat-ssh-delivery.unit.test.ts": {
"added": 10,
"removed": 9
@@ -6894,7 +6936,7 @@
"removed": 0
},
"src/main/startup/main-process-runtime-service.ts": {
- "added": 21,
+ "added": 17,
"removed": 2
},
"src/main/startup/main-process-state.ts": {
@@ -7194,7 +7236,7 @@
"removed": 1
},
"src/renderer/src/components/native-chat/native-chat-composer-state.test.ts": {
- "added": 144,
+ "added": 157,
"removed": 1
},
"src/renderer/src/components/native-chat/native-chat-composer-state.ts": {
@@ -7215,15 +7257,15 @@
},
"src/renderer/src/components/native-chat/native-chat-composer-types.ts": {
"added": 14,
- "removed": 40
+ "removed": 37
},
"src/renderer/src/components/native-chat/native-chat-pending.ts": {
"added": 1,
"removed": 1
},
- "src/renderer/src/components/native-chat/native-chat-picker-items.ts": {
- "added": 65,
- "removed": 17
+ "src/shared/native-chat-picker-items.ts": {
+ "added": 76,
+ "removed": 21
},
"src/renderer/src/components/native-chat/native-chat-pty-session-options.ts": {
"added": 25,
@@ -7626,8 +7668,8 @@
"removed": 1
},
"src/shared/native-chat-agent-profiles.ts": {
- "added": 16,
- "removed": 2
+ "added": 11,
+ "removed": 1
},
"src/shared/native-chat-session-option-state.test.ts": {
"added": 65,
@@ -7882,8 +7924,8 @@
"removed": 0
},
"src/main/runtime/orca-runtime.ts": {
- "added": 6,
- "removed": 1
+ "added": 5,
+ "removed": 0
},
"src/main/runtime/rpc/methods/terminal/terminal-multiplex-cleanup.ts": {
"added": 3,
@@ -7982,9 +8024,21 @@
"removed": 13
},
"src/renderer/src/components/sidebar/WorktreeList.tsx": {
- "added": 46,
+ "added": 49,
"removed": 5
},
+ "src/renderer/src/components/sidebar/worktree-list/listing/sidebar-list-model.ts": {
+ "added": 1,
+ "removed": 0
+ },
+ "src/renderer/src/components/sidebar/worktree-list/listing/use-main-list-model.ts": {
+ "added": 2,
+ "removed": 1
+ },
+ "src/renderer/src/components/sidebar/worktree-list/listing/use-window-list-model.ts": {
+ "added": 11,
+ "removed": 10
+ },
"src/renderer/src/components/sidebar/sidebar-filter-actions.ts": {
"added": 17,
"removed": 3
@@ -8093,7 +8147,7 @@
"added": 3,
"removed": 1
},
- "src/renderer/src/components/tab-bar/SortableTab.tsx": {
+ "src/renderer/src/components/tab-bar/use-terminal-tab-activity.ts": {
"added": 2,
"removed": 0
},
@@ -8114,16 +8168,16 @@
"removed": 2
},
"src/main/ipc/repos.ts": {
- "added": 2,
- "removed": 2
+ "added": 1,
+ "removed": 1
},
"src/main/ipc/repos/project-group-handlers.ts": {
"added": 23,
"removed": 5
},
"src/main/ipc/repos/repo-catalog-handlers.ts": {
- "added": 24,
- "removed": 4
+ "added": 15,
+ "removed": 2
},
"src/main/ipc/repos/repo-ipc-arg-schemas.ts": {
"added": 6,
@@ -8174,7 +8228,7 @@
"removed": 0
},
"src/main/runtime/runtime-project-group-controller.ts": {
- "added": 64,
+ "added": 62,
"removed": 3
},
"src/main/runtime/runtime-repository-settings-controller.ts": {
@@ -8270,8 +8324,8 @@
"removed": 2
},
"src/renderer/src/store/repos/repo-state.ts": {
- "added": 15,
- "removed": 2
+ "added": 11,
+ "removed": 1
},
"src/renderer/src/store/right-sidebar-route.test.ts": {
"added": 21,
@@ -8337,7 +8391,7 @@
"added": 20,
"removed": 0
},
- "src/main/runtime/sidebar-inputs/orca-runtime-workspace-activity-facts.ts": {
+ "src/main/runtime/orca-runtime-agent-transitions.ts": {
"added": 1,
"removed": 1
},
@@ -8626,11 +8680,11 @@
"removed": 0
},
"src/renderer/src/lib/launch-agent-background-session.ts": {
- "added": 38,
- "removed": 22
+ "added": 30,
+ "removed": 18
},
"src/renderer/src/lib/launch-agent-in-new-tab-web-runtime.test.ts": {
- "added": 27,
+ "added": 28,
"removed": 0
},
"src/renderer/src/lib/launch-agent-in-new-tab.ts": {
diff --git a/config/id-only-store-write-baseline.txt b/config/id-only-store-write-baseline.txt
new file mode 100644
index 000000000000..00835d310956
--- /dev/null
+++ b/config/id-only-store-write-baseline.txt
@@ -0,0 +1,9 @@
+# Store writes whose first parameter is a bare repo or worktree id (R4-2), as Class.method,
+# and persistence repo-row lookups by id alone (review-5 #10), as lookup:file#function.
+# This is a RATCHET: entries may only be REMOVED. Take RepoHostTarget or HostWorktreeTarget,
+# or match the row by id AND host (getRepoExecutionHostId).
+# Prune after a fix: pnpm check:id-only-store-write-ratchet --prune
+lookup:src/main/persistence/loading-store/terminal-binding-recovery.ts#getConnectionIdForWorktree
+lookup:src/main/persistence/scheduling-automations/automation-definition-operations.ts#createAutomation
+lookup:src/main/persistence/scheduling-automations/automation-definition-operations.ts#updateAutomation
+lookup:src/main/persistence/tracking-repos/project-host-operations.ts#getRepo
diff --git a/config/legacy-relay-compat-tags-baseline.txt b/config/legacy-relay-compat-tags-baseline.txt
index 31905b926410..16a3bfc3fee8 100644
--- a/config/legacy-relay-compat-tags-baseline.txt
+++ b/config/legacy-relay-compat-tags-baseline.txt
@@ -20,7 +20,7 @@ src/main/ssh/orcad-migration-terminal-gate.ts
src/main/managed-server/lifecycle/orcad-runtime-conversion.test.ts
src/main/ssh/ssh-host-server-on-connect.test.ts
src/main/ssh/ssh-host-server-on-connect.ts
-src/main/ssh/ssh-target-orcad-preflight.ts
+src/main/ssh/targets/ssh-target-orcad-preflight.ts
src/preload/api-types.ts
src/preload/api/legacy-relay-bridge-api.ts
src/renderer/src/components/terminal-pane/legacy-relay-bridge-panes.ts
diff --git a/config/main-layout-writer-baseline.txt b/config/main-layout-writer-baseline.txt
index c7b9453dfcab..4166afdb39de 100644
--- a/config/main-layout-writer-baseline.txt
+++ b/config/main-layout-writer-baseline.txt
@@ -71,5 +71,6 @@
4 src/main/runtime/workspace-session-worktree-id.ts
1 src/main/runtime/workspaces/session-hosts/get-runtime-id.ts
2 src/main/runtime/workspaces/workspace-file-session-targets.ts
-1 src/main/ssh/ssh-target-id-migration.ts
+1 src/main/ssh/orcad-migration-source-retention.ts
+1 src/main/ssh/targets/ssh-target-id-migration.ts
2 src/main/terminal-scrollback-snapshots.ts
diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt
index 5e8ac38c8903..10cb9ef36594 100644
--- a/config/max-lines-baseline.txt
+++ b/config/max-lines-baseline.txt
@@ -4,5 +4,5 @@
# Regenerate/prune: pnpm check:max-lines-ratchet --prune (removes stale entries only)
inline src/main/ipc/worktree-remote.ts
inline src/main/ssh/ssh-channel-multiplexer.ts
-inline src/main/ssh/ssh-connection.ts
+inline src/main/ssh/connection/ssh-connection.ts
inline src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts
diff --git a/config/packaged-native-build-entries.cjs b/config/packaged-native-build-entries.cjs
new file mode 100644
index 000000000000..f0c8301e35c4
--- /dev/null
+++ b/config/packaged-native-build-entries.cjs
@@ -0,0 +1,40 @@
+const { existsSync, readdirSync, renameSync, rmSync } = require('node:fs')
+const { join } = require('node:path')
+const {
+ NATIVE_BUILD_PACKAGES,
+ isNativeBuildEntryName,
+ nativeBuildEntryName
+} = require('./scripts/native-build-entry.cjs')
+
+/**
+ * Ship a dev install's per-ABI native build as build/Release.
+ *
+ * A package dir shared across worktrees (pnpm's global virtual store) holds Orca's source build
+ * in build/ rather than build/Release; see scripts/native-build-entry.cjs. Packaged
+ * consumers (daemon relocation, the macOS terminal host, the job-ownership gate) all read
+ * build/Release, so the packaged copy gets exactly the layout a CI install produces.
+ */
+function adoptPackagedNativeBuildEntries(resourcesDir, electronPlatformName, architecture) {
+ for (const moduleName of Object.keys(NATIVE_BUILD_PACKAGES)) {
+ const buildDir = join(resourcesDir, 'node_modules', ...moduleName.split('/'), 'build')
+ if (!existsSync(buildDir)) {
+ continue
+ }
+ const entryDir = join(
+ buildDir,
+ nativeBuildEntryName(moduleName, { platform: electronPlatformName, arch: architecture })
+ )
+ if (existsSync(entryDir)) {
+ rmSync(join(buildDir, 'Release'), { recursive: true, force: true })
+ renameSync(entryDir, join(buildDir, 'Release'))
+ }
+ // Other arches, plus any staging or quarantine leftovers, are dead weight in this slice.
+ for (const name of readdirSync(buildDir)) {
+ if (isNativeBuildEntryName(name)) {
+ rmSync(join(buildDir, name), { recursive: true, force: true })
+ }
+ }
+ }
+}
+
+module.exports = { adoptPackagedNativeBuildEntries }
diff --git a/config/packaged-runtime-node-modules.cjs b/config/packaged-runtime-node-modules.cjs
index 63556ad3b6dc..62ed2ff47f55 100644
--- a/config/packaged-runtime-node-modules.cjs
+++ b/config/packaged-runtime-node-modules.cjs
@@ -10,6 +10,11 @@ const {
const { dirname, join, resolve } = require('node:path')
const { builtinModules, createRequire } = require('node:module')
const { PE_MACHINE, readPeMachine } = require('./scripts/windows-pe-machine.cjs')
+const { adoptPackagedNativeBuildEntries } = require('./packaged-native-build-entries.cjs')
+const {
+ NATIVE_BUILD_PACKAGES,
+ TRANSIENT_NATIVE_BUILD_ENTRY_FILTER
+} = require('./scripts/native-build-entry.cjs')
const projectDir = resolve(__dirname, '..')
const requireFromProject = createRequire(join(projectDir, 'package.json'))
@@ -223,7 +228,9 @@ function createPackagedRuntimeNodeModuleResources(electronPlatformName = process
return collectPackagedRuntimePackages(electronPlatformName).map(([packageName, packageDir]) => ({
from: packageDir,
to: join('node_modules', ...packageName.split('/')),
- ...(packageName === '@orca/process-host' ? { filter: ['package.json', 'dist/**/*'] } : {})
+ ...(packageName === '@orca/process-host' ? { filter: ['package.json', 'dist/**/*'] } : {}),
+ // Why: another worktree can rename these away mid-copy, failing the walk with ENOENT.
+ ...(NATIVE_BUILD_PACKAGES[packageName] ? { filter: TRANSIENT_NATIVE_BUILD_ENTRY_FILTER } : {})
}))
}
@@ -658,6 +665,8 @@ function assertPackagedNativeVariantsInstalled(electronPlatformName, electronArc
function prunePackagedRuntimeNodeModules(resourcesDir, electronPlatformName, electronArch) {
const architecture = normalizeElectronArchitecture(electronArch)
+ // Before the node-pty prune, which judges build/Release.
+ adoptPackagedNativeBuildEntries(resourcesDir, electronPlatformName, architecture)
prunePackagedNodePty(resourcesDir, electronPlatformName, architecture)
prunePackagedParcelWatcher(resourcesDir, electronPlatformName, architecture)
// Why before the filename walk: zod/src is deleted wholesale, so walking it first is wasted work.
diff --git a/config/patches/@vscode__windows-process-tree@0.8.0.patch b/config/patches/@vscode__windows-process-tree@0.8.0.patch
index bc50a8cfa631..ac0defcee9fa 100644
--- a/config/patches/@vscode__windows-process-tree@0.8.0.patch
+++ b/config/patches/@vscode__windows-process-tree@0.8.0.patch
@@ -32,13 +32,17 @@ index 11a5e71..1b2e74b 100644
"/guard:cf",
"/sdl",
diff --git a/lib/index.js b/lib/index.js
-index e586cd6..0ee62c3 100644
+index 9747a7402600cd252859144d32580ed45c8c93f7..69cf7fe6d14e52b199ac9ab869567be5d9bb708c 100644
--- a/lib/index.js
+++ b/lib/index.js
-@@ -7,11 +7,14 @@
+@@ -6,12 +6,17 @@
+ Object.defineProperty(exports, "__esModule", { value: true });
exports.getAllProcesses = exports.getProcessTree = exports.getProcessCpuUsage = exports.getProcessList = exports.filterProcessList = exports.buildProcessTree = exports.ProcessDataFlag = void 0;
const util_1 = require("util");
- const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined;
+-const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined;
++// Orca's per-ABI build, published into package dirs shared across worktrees, wins over build/Release.
++const orcaNativeBuild = `${__dirname}/../build/orca-napi-win32-${process.arch}/windows_process_tree.node`;
++const native = process.platform === 'win32' ? require(require('fs').existsSync(orcaNativeBuild) ? orcaNativeBuild : '../build/Release/windows_process_tree.node') : undefined;
+exports.getProcessCreationTime = native === undefined ? undefined : native.getProcessCreationTime;
+exports.supportedProcessDataFlags = native === undefined ? undefined : native.supportedProcessDataFlags;
var ProcessDataFlag;
@@ -50,7 +54,7 @@ index e586cd6..0ee62c3 100644
})(ProcessDataFlag = exports.ProcessDataFlag || (exports.ProcessDataFlag = {}));
// requestInProgress is used for any function that uses CreateToolhelp32Snapshot, as multiple calls
// to this cannot be done at the same time.
-@@ -66,11 +69,12 @@
+@@ -66,11 +71,12 @@ function buildProcessTree(rootPid, processList, maxDepth = MAX_FILTER_DEPTH) {
// • the properties are inlined/splatted
// • the 'ppid' field is omitted
// • the depth of the tree is limited by `maxDepth`
@@ -65,13 +69,17 @@ index e586cd6..0ee62c3 100644
});
return buildNode(root, maxDepth);
diff --git a/lib/index.ts b/lib/index.ts
-index 7b53aad..284dae1 100644
+index 7b53aad05c3682a5c7d814d81a39c3f391ae97e3..364b384e7d55f3682515431839352b5b4a9ffa93 100644
--- a/lib/index.ts
+++ b/lib/index.ts
-@@ -6,12 +6,16 @@
+@@ -5,13 +5,19 @@
+
import { promisify } from 'util';
- const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined;
+-const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined;
++// Orca's per-ABI build, published into package dirs shared across worktrees, wins over build/Release.
++const orcaNativeBuild = `${__dirname}/../build/orca-napi-win32-${process.arch}/windows_process_tree.node`;
++const native = process.platform === 'win32' ? require(require('fs').existsSync(orcaNativeBuild) ? orcaNativeBuild : '../build/Release/windows_process_tree.node') : undefined;
+export const getProcessCreationTime: ((pid: number) => number | undefined) | undefined = native?.getProcessCreationTime;
+/** The flag bits this compiled addon reports; undefined off win32. */
+export const supportedProcessDataFlags: number | undefined = native?.supportedProcessDataFlags;
@@ -86,7 +94,7 @@ index 7b53aad..284dae1 100644
}
type RequestCallback = (processList: IProcessInfo[]) => void;
-@@ -81,11 +85,12 @@
+@@ -81,11 +87,12 @@ export function buildProcessTree(rootPid: number, processList: Iterable src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts"
+ "node config/scripts/run-ssh-staged-upload-reliability.mjs --powershell src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/connection/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts"
],
"testFiles": [
"src/main/ssh/sftp-upload.test.ts",
"src/main/ssh/ssh-file-transfer-abort.test.ts",
"src/main/ssh/sftp-namespace-resolution.test.ts",
- "src/main/ssh/ssh-connection-sftp-wire.test.ts",
+ "src/main/ssh/connection/ssh-connection-sftp-wire.test.ts",
"src/main/ssh/ssh-remote-commands.test.ts"
],
"assertionRefs": [
@@ -3817,7 +3896,7 @@
"date": "2026-07-31",
"runner": "local",
"platform": "macos",
- "command": "node config/scripts/run-ssh-staged-upload-reliability.mjs --powershell src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts",
+ "command": "node config/scripts/run-ssh-staged-upload-reliability.mjs --powershell src/main/ssh/sftp-upload.test.ts src/main/ssh/ssh-file-transfer-abort.test.ts src/main/ssh/sftp-namespace-resolution.test.ts src/main/ssh/connection/ssh-connection-sftp-wire.test.ts src/main/ssh/ssh-remote-commands.test.ts",
"result": "passed",
"durationSeconds": 57.9,
"summary": "Eleven focused files passed 152 tests with 3 platform skips using exact POSIX sh, native ARM macOS PowerShell 7.6.4, real ssh2 split-SFTP wire sessions, fixed-slot identity races, bounded recovery, joined cancellation teardown, and cross-version isolation."
@@ -4465,7 +4544,7 @@
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/worktree-base-directory-event-filter.test.ts src/main/ipc/worktree-base-directory-watcher.test.ts src/main/ipc/worktree-base-directory-poller.test.ts src/main/ipc/worktree-head-identity-reader.test.ts src/renderer/src/hooks/worktree-head-identity-apply.test.ts src/renderer/src/components/right-sidebar/git-status-push-signal-refresh.test.ts src/renderer/src/hooks/useIpcEvents-browser-tab-create.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/watched-worktree-catalog-notification.test.ts src/main/runtime/external-worktree-paired-client-discovery.integration.test.ts src/main/window/attach-main-window-services.test.ts",
"ORCA_E2E_FORCE_HEADFUL=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-external-worktree-discovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
- "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-connection-generation.test.ts src/main/ssh/ssh-provider-authority.test.ts --reporter=dot",
+ "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/connection/ssh-connection-generation.test.ts src/main/ssh/ssh-provider-authority.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/hooks/useIpcEvents-agent-status-ssh-authority.test.ts --reporter=dot"
],
"testFiles": [
@@ -4481,7 +4560,7 @@
"src/main/window/attach-main-window-services.test.ts",
"tests/e2e/paired-external-worktree-discovery.spec.ts",
"src/renderer/src/hooks/useIpcEvents-agent-status-ssh-authority.test.ts",
- "src/main/ssh/ssh-connection-generation.test.ts",
+ "src/main/ssh/connection/ssh-connection-generation.test.ts",
"src/main/ssh/ssh-provider-authority.test.ts"
],
"assertionRefs": [
@@ -4581,7 +4660,7 @@
]
},
{
- "file": "src/main/ssh/ssh-connection-generation.test.ts",
+ "file": "src/main/ssh/connection/ssh-connection-generation.test.ts",
"assertions": [
"one exhausted target rolls the process generation scope and invalidates every sibling target token",
"old-scope mutation expectations fail while the new-scope authority continues rotating"
@@ -4609,7 +4688,7 @@
"date": "2026-07-27",
"runner": "local",
"platform": "macos",
- "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/ssh-connection-generation.test.ts src/main/ssh/ssh-provider-authority.test.ts --reporter=dot",
+ "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ssh/connection/ssh-connection-generation.test.ts src/main/ssh/ssh-provider-authority.test.ts --reporter=dot",
"result": "passed",
"durationSeconds": 1.61,
"summary": "Three main-process catalog and authority files passed with 118 tests, including contradictory catalog provenance rejection and all-target generation-scope revocation. The third file, src/main/ipc/repos-remote.test.ts, was later deleted with the SSH repo IPC handlers."
@@ -9914,7 +9993,8 @@
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/resume-sleeping-agent-session.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts tests/e2e/completed-worker-retirement-resume.unit.test.ts --reporter=verbose",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/resume-sleeping-agent-session.test.ts src/main/providers/local-pty-provider-spawn-session.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/ipc/pty-pane-materialization-race.test.ts src/main/ipc/pty-persisted-incarnation-repair.test.ts src/main/ipc/pty-pane-reservation-settlement.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/lib/pane-manager/pane-fit.test.ts src/renderer/src/components/terminal-pane/pty-connection-hidden-replay.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-host-session-launch.test.ts",
- "pnpm exec electron-vite build --mode e2e && SKIP_BUILD=1 pnpm exec playwright test tests/e2e/live-background-terminal-mount-authority.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
+ "pnpm exec electron-vite build --mode e2e && SKIP_BUILD=1 pnpm exec playwright test tests/e2e/live-background-terminal-mount-authority.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
+ "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/structured-agent-cli-writers.test.ts src/main/runtime/rpc/methods/structured-agent-session-adoption-replay.test.ts src/main/providers/agent-foreground-process-git-bash.win32.test.ts"
],
"testFiles": [
"src/renderer/src/lib/resume-sleeping-agent-session.test.ts",
@@ -9929,9 +10009,27 @@
"src/renderer/src/lib/pane-manager/pane-fit.test.ts",
"src/renderer/src/components/terminal-pane/pty-connection-hidden-replay.test.ts",
"src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-host-session-launch.test.ts",
- "tests/e2e/live-background-terminal-mount-authority.spec.ts"
+ "tests/e2e/live-background-terminal-mount-authority.spec.ts",
+ "src/main/runtime/structured-agent-cli-writers.test.ts",
+ "src/main/runtime/rpc/methods/structured-agent-session-adoption-replay.test.ts",
+ "src/main/providers/agent-foreground-process-git-bash.win32.test.ts"
],
"assertionRefs": [
+ {
+ "file": "src/main/runtime/structured-agent-cli-writers.test.ts",
+ "assertions": [
+ "accepts exact exited or unchanged shell-only proof, never loss of evidence",
+ "inspects the terminal a conversation leaves, whatever its rows now say"
+ ]
+ },
+ {
+ "file": "src/main/runtime/rpc/methods/structured-agent-session-adoption-replay.test.ts",
+ "assertions": ["captures source before routing (unary=%s)"]
+ },
+ {
+ "file": "src/main/providers/agent-foreground-process-git-bash.win32.test.ts",
+ "assertions": ["sets aside a detached ssh-agent for adoption, but not a background job"]
+ },
{
"file": "tests/e2e/completed-worker-retirement-resume.unit.test.ts",
"assertions": [
@@ -13059,7 +13157,7 @@
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/terminal-error-accumulation.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-attach-subscription.test.ts --reporter=dot",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-attach-subscription.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-host-surface-replacement.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-stream-reconnect.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-expired-pane-recovery.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-outcome-recovery.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-create-handoff.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/hooks/ipc-events/direct-ssh-connect-reply-routing.test.ts src/renderer/src/store/slices/terminals-hydration.test.ts src/renderer/src/store/slices/repos-ssh-host-reconciliation.test.ts src/renderer/src/hooks/direct-ssh-state-routing.test.ts src/renderer/src/components/terminal-pane/pty-connection-spawn-and-startup.test.ts src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.test.ts --reporter=dot",
- "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh/ssh.test.ts src/main/ipc/worktrees-ssh-repo-owner-resolution.test.ts src/main/ipc/worktrees-lineage-hydration.test.ts src/main/runtime/public-ssh-state.test.ts src/main/ssh/ssh-connection-manager.test.ts src/main/ssh/ssh-connection.test.ts src/main/ssh/ssh-provider-authority.test.ts src/preload/ssh-authority-forwarding.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/runtime/runtime-environment-ssh-state.test.ts src/shared/ssh-retained-payload-admission.test.ts --reporter=dot",
+ "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/ssh/ssh.test.ts src/main/ipc/worktrees-ssh-repo-owner-resolution.test.ts src/main/ipc/worktrees-lineage-hydration.test.ts src/main/runtime/public-ssh-state.test.ts src/main/ssh/connection/ssh-connection-manager.test.ts src/main/ssh/connection/ssh-connection.test.ts src/main/ssh/ssh-provider-authority.test.ts src/preload/ssh-authority-forwarding.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/runtime/runtime-environment-ssh-state.test.ts src/shared/ssh-retained-payload-admission.test.ts --reporter=dot",
"pnpm exec electron-vite build --mode e2e",
"pnpm run build:web-from-renderer",
"SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
@@ -13106,8 +13204,8 @@
"src/main/ipc/worktrees-ssh-repo-owner-resolution.test.ts",
"src/main/ipc/worktrees-lineage-hydration.test.ts",
"src/main/runtime/public-ssh-state.test.ts",
- "src/main/ssh/ssh-connection-manager.test.ts",
- "src/main/ssh/ssh-connection.test.ts",
+ "src/main/ssh/connection/ssh-connection-manager.test.ts",
+ "src/main/ssh/connection/ssh-connection.test.ts",
"src/main/ssh/ssh-provider-authority.test.ts",
"src/preload/ssh-authority-forwarding.test.ts",
"src/renderer/src/runtime/runtime-client-events.test.ts",
@@ -13269,13 +13367,13 @@
]
},
{
- "file": "src/main/ssh/ssh-connection-manager.test.ts",
+ "file": "src/main/ssh/connection/ssh-connection-manager.test.ts",
"assertions": [
"disconnect invalidates a pending transport attempt immediately so late rejection or resolution cannot remove the replacement"
]
},
{
- "file": "src/main/ssh/ssh-connection.test.ts",
+ "file": "src/main/ssh/connection/ssh-connection.test.ts",
"assertions": [
"late ssh2 ready and startup error events after disconnect cannot resurrect or overwrite disconnected state"
]
diff --git a/config/renderer-layout-writer-baseline.txt b/config/renderer-layout-writer-baseline.txt
index 1a780e4baa63..8576722aabb8 100644
--- a/config/renderer-layout-writer-baseline.txt
+++ b/config/renderer-layout-writer-baseline.txt
@@ -88,7 +88,7 @@
23 src/renderer/src/store/slices/tabs/tabs-drag-preview-actions.ts
11 src/renderer/src/store/slices/tabs/tabs-drop-actions.ts
4 src/renderer/src/store/slices/tabs/tabs-focus-actions.ts
-13 src/renderer/src/store/slices/tabs/tabs-group-actions.ts
+12 src/renderer/src/store/slices/tabs/tabs-group-actions.ts
1 src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts
8 src/renderer/src/store/slices/tabs/tabs-label-actions.ts
10 src/renderer/src/store/slices/tabs/tabs-move-actions.ts
diff --git a/config/scripts/adhoc-windows-parallelism.test.mjs b/config/scripts/adhoc-windows-parallelism.test.mjs
new file mode 100644
index 000000000000..9f3dd54feeeb
--- /dev/null
+++ b/config/scripts/adhoc-windows-parallelism.test.mjs
@@ -0,0 +1,79 @@
+import { readFileSync } from 'node:fs'
+import { describe, expect, it } from 'vitest'
+import { parse } from 'yaml'
+import { compileAggregateExpression } from './pr-aggregate-gate-context.mjs'
+
+const { jobs } = parse(readFileSync('.github/workflows/adhoc-mac-build.yml', 'utf8'))
+const mac = jobs['build-adhoc-mac']
+const windows = jobs['build-adhoc-win']
+const publish = jobs['publish-adhoc-win']
+
+function qualification(template = 'success', ships = 'true') {
+ return {
+ github: { repository: 'stablyai/orca' },
+ needs: {
+ 'resolve-ref': { result: 'success', outputs: { sha: 'a'.repeat(40) } },
+ 'orcad-template-support': { result: 'success', outputs: { ships } },
+ 'orcad-template': { result: template }
+ }
+ }
+}
+
+describe('adhoc Windows compilation overlaps Mac without publishing early', () => {
+ it('starts both platforms from the same qualified inputs without a Mac dependency', () => {
+ expect(windows.needs).toEqual(mac.needs)
+ expect(windows.needs).not.toContain('build-adhoc-mac')
+ expect(windows.with.stage_only).toBe(true)
+ expect(windows.with.ref).toBe('${{ needs.resolve-ref.outputs.sha }}')
+ expect(windows.with.tag).toBe('v${{ needs.orcad-template-support.outputs.version }}')
+ expect(windows.with.version).toBe(mac.outputs.version)
+ expect(windows.with.orcad_template).toBe(
+ "${{ needs.orcad-template-support.outputs.ships == 'true' }}"
+ )
+ const identity = jobs['orcad-template-support']
+ expect(identity.outputs.version).toBe('${{ steps.adhoc.outputs.version }}')
+ expect(identity.steps.filter((step) => step.id === 'adhoc')).toHaveLength(1)
+ expect(mac.steps.some((step) => step.id === 'adhoc')).toBe(false)
+ expect(mac.steps.find((step) => step.name === 'Create adhoc release').env.TAG).toBe(
+ windows.with.tag
+ )
+ })
+
+ it.each([
+ ['success', 'true', true],
+ ['failure', 'true', false],
+ ['cancelled', 'true', false],
+ ['skipped', 'true', false],
+ ['skipped', 'false', true]
+ ])('preserves qualification for %s / template=%s', (result, ships, allowed) => {
+ const run = compileAggregateExpression(windows.if, ['needs', 'github'])
+ const context = qualification(result, ships)
+ expect(run(context).coerceString()).toBe(String(allowed))
+ expect(run(context, true).coerceString()).toBe('false')
+ context.needs['orcad-template-support'].result = 'failure'
+ expect(run(context).coerceString()).toBe('false')
+ })
+
+ it.each(['success', 'failure', 'cancelled', 'skipped', ''])(
+ 'requires successful Mac publication and Windows compilation: %s',
+ (result) => {
+ const run = compileAggregateExpression(publish.if)
+ const needs = {
+ 'build-adhoc-mac': { result: 'success', outputs: { published: 'true' } },
+ 'build-adhoc-win': { result: 'success' }
+ }
+ expect(publish.needs).toEqual(['build-adhoc-mac', 'build-adhoc-win'])
+ expect(run({ needs }).coerceString()).toBe('true')
+ expect(run({ needs }, true).coerceString()).toBe('false')
+ for (const job of publish.needs) {
+ expect(run({ needs: { ...needs, [job]: { ...needs[job], result } } }).coerceString()).toBe(
+ String(result === 'success')
+ )
+ }
+ for (const published of ['false', '', undefined]) {
+ needs['build-adhoc-mac'].outputs.published = published
+ expect(run({ needs }).coerceString()).toBe('false')
+ }
+ }
+ )
+})
diff --git a/config/scripts/audit-localization-coverage.mjs b/config/scripts/audit-localization-coverage.mjs
index 4e7e83a85ba6..a7813745d599 100644
--- a/config/scripts/audit-localization-coverage.mjs
+++ b/config/scripts/audit-localization-coverage.mjs
@@ -11,6 +11,8 @@ const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mts', '.cts']
const SKIP_PATH_PARTS = new Set(['.git', 'dist', 'node_modules', 'out', '__snapshots__', 'assets'])
const LOCALIZATION_CALL_NAMES = new Set(['t', 'translate'])
const CLASS_PROPERTY_NAMES = new Set(['className', 'classNames'])
+// Why: the children of these elements are CSS or code, never copy a translator should see.
+const NON_COPY_JSX_ELEMENTS = new Set(['style', 'script'])
const USER_VISIBLE_JSX_ATTRIBUTES = new Set([
'ariaLabel',
'aria-label',
@@ -314,8 +316,16 @@ function isUserVisibleCallArgument(node) {
)
}
+function isInsideNonCopyJsxElement(node) {
+ const element = findAncestor(node, (ancestor) => ts.isJsxElement(ancestor))
+ return Boolean(element && NON_COPY_JSX_ELEMENTS.has(element.openingElement.tagName.getText()))
+}
+
function classifyStringNode(node) {
- if (hasAncestorObjectPropertyName(node, CLASS_PROPERTY_NAMES)) {
+ if (
+ hasAncestorObjectPropertyName(node, CLASS_PROPERTY_NAMES) ||
+ isInsideNonCopyJsxElement(node)
+ ) {
return undefined
}
diff --git a/config/scripts/audit-localization-coverage.test.mjs b/config/scripts/audit-localization-coverage.test.mjs
index 1615bc10556c..36498c9f228e 100644
--- a/config/scripts/audit-localization-coverage.test.mjs
+++ b/config/scripts/audit-localization-coverage.test.mjs
@@ -31,6 +31,23 @@ describe('localization coverage candidates', () => {
expect(reports.map((report) => report.text)).toEqual(['Retry the sync'])
})
+ it('ignores CSS and script bodies but still sees copy beside them', () => {
+ const reports = candidates(
+ 'Sample.tsx',
+ `export function Sample() {
+ return (
+
+
+
+
Choose a workspace
+
+ )
+ }`
+ )
+
+ expect(reports.map((report) => report.text)).toEqual(['Choose a workspace'])
+ })
+
it('ignores literals used as comparison operands', () => {
const reports = candidates(
'Sample.tsx',
diff --git a/config/scripts/benchmark-windows-native-preparation.mjs b/config/scripts/benchmark-windows-native-preparation.mjs
index 9d0124a0f832..e9ddc5332904 100644
--- a/config/scripts/benchmark-windows-native-preparation.mjs
+++ b/config/scripts/benchmark-windows-native-preparation.mjs
@@ -274,7 +274,7 @@ export async function benchmarkWindowsNativePreparation(
await command('source-clean', 'git', ['diff', '--exit-code', 'HEAD', '--'])
const tree = (await command('source-tree', 'git', ['rev-parse', 'HEAD^{tree}'])).stdout.trim()
const pnpmVersion = (await pnpmCommand('pnpm-version', ['--version'], root)).stdout.trim()
- assert.equal(pnpmVersion, '12.8.1', 'Comparison requires pinned pnpm')
+ assert.equal(pnpmVersion, '12.14.0', 'Comparison requires pinned pnpm')
const action = readFileSync(
join(root, '.github/actions/prepare-native-runtime/action.yml'),
'utf8'
diff --git a/config/scripts/build-cli.mjs b/config/scripts/build-cli.mjs
new file mode 100644
index 000000000000..0495c0b92627
--- /dev/null
+++ b/config/scripts/build-cli.mjs
@@ -0,0 +1,65 @@
+import { availableParallelism } from 'node:os'
+import { fileURLToPath } from 'node:url'
+import { runProcess, runProcessSync } from '@orca/process-host'
+import { resolvePinnedBunExecutable } from './pinned-bun-executable.mjs'
+import { describeProcessFailure } from './process-failure-message.mjs'
+import { isDirectInvocation } from './script-entry-detection.mjs'
+import {
+ resolveProjectCompiler,
+ runTypecheckProjects,
+ selectTypecheckCompiler
+} from './run-typecheck-projects-in-parallel.mjs'
+
+const ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const CONFIG = 'config/tsconfig.build.cli.json'
+
+export async function buildCli({
+ root = ROOT,
+ outDir = 'out',
+ check = runTypecheckProjects,
+ execute = runProcess,
+ resolveBunExecutable = (directory) => resolvePinnedBunExecutable(directory, runProcessSync)
+} = {}) {
+ const selection = selectTypecheckCompiler('bun', { root, resolveBunExecutable })
+ const failures = await check({
+ ...selection,
+ root,
+ projects: [{ config: CONFIG, heapGib: 2 }],
+ bunThreads: availableParallelism()
+ })
+ if (failures.length > 0) {
+ throw failures[0]
+ }
+
+ // Bun has checked this exact project; retain TypeScript's emitter without checking it twice.
+ const result = await execute({
+ program: process.execPath,
+ args: [
+ resolveProjectCompiler(CONFIG, root),
+ '-p',
+ CONFIG,
+ '--outDir',
+ outDir,
+ '--composite',
+ 'false',
+ '--incremental',
+ 'false',
+ '--noCheck'
+ ],
+ cwd: root,
+ stdio: 'inherit',
+ timeoutMs: null
+ })
+ if (result.code !== 0) {
+ throw new Error(`CLI emission failed: ${describeProcessFailure(result)}`)
+ }
+}
+
+if (isDirectInvocation(import.meta.url, process.argv[1])) {
+ try {
+ await buildCli()
+ } catch (error) {
+ console.error(error.message)
+ process.exitCode = 1
+ }
+}
diff --git a/config/scripts/build-cli.test.mjs b/config/scripts/build-cli.test.mjs
new file mode 100644
index 000000000000..159dce8198b7
--- /dev/null
+++ b/config/scripts/build-cli.test.mjs
@@ -0,0 +1,183 @@
+import {
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync
+} from 'node:fs'
+import { availableParallelism, tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { runProcess, runProcessSync } from '@orca/process-host'
+import { afterEach, expect, it, vi } from 'vitest'
+import { buildCli } from './build-cli.mjs'
+import { resolvePinnedBunExecutable } from './pinned-bun-executable.mjs'
+import { resolveProjectCompiler } from './run-typecheck-projects-in-parallel.mjs'
+
+const ROOT = process.cwd()
+const resolveBunExecutable = () => resolvePinnedBunExecutable(ROOT, runProcessSync)
+const directories = []
+afterEach(() => {
+ for (const directory of directories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
+function fixture() {
+ const root = mkdtempSync(join(tmpdir(), 'orca-cli-build-'))
+ directories.push(root)
+ mkdirSync(join(root, 'config'))
+ mkdirSync(join(root, 'src'))
+ symlinkSync(resolve('node_modules'), join(root, 'node_modules'), 'junction')
+ writeFileSync(join(root, 'package.json'), '{"type":"commonjs"}')
+ writeFileSync(join(root, 'config', '.bun-version'), readFileSync('config/.bun-version'))
+ writeFileSync(
+ join(root, 'config', 'tsconfig.build.cli.json'),
+ JSON.stringify({
+ compilerOptions: {
+ module: 'Node20',
+ moduleResolution: 'Node16',
+ target: 'ES2022',
+ strict: true,
+ types: [],
+ rootDir: '../src',
+ declaration: true,
+ declarationMap: true,
+ sourceMap: true
+ },
+ include: ['../src/**/*.ts']
+ })
+ )
+ writeFileSync(join(root, 'src', 'constants.ts'), 'export const enum Mode { Ready = 7 }\n')
+ writeFileSync(
+ join(root, 'src', 'entry.ts'),
+ `import { Mode } from './constants'
+export interface Item { value: number }
+export const item: Item = { value: Mode.Ready }
+`
+ )
+ return root
+}
+
+it('checks the emit project first, retains all CPU cores and preserves the TypeScript emitter', async () => {
+ const order = []
+ const check = vi.fn(async () => {
+ order.push('check')
+ return []
+ })
+ const execute = vi.fn(async () => {
+ order.push('emit')
+ return { code: 0 }
+ })
+ await buildCli({ check, execute, resolveBunExecutable: () => 'pinned-bun' })
+ expect(order).toEqual(['check', 'emit'])
+ expect(check).toHaveBeenCalledWith(
+ expect.objectContaining({
+ compiler: 'bun',
+ bunExecutable: 'pinned-bun',
+ projects: [{ config: 'config/tsconfig.build.cli.json', heapGib: 2 }],
+ bunThreads: availableParallelism()
+ })
+ )
+ expect(execute).toHaveBeenCalledWith(
+ expect.objectContaining({
+ program: process.execPath,
+ args: [
+ resolveProjectCompiler('config/tsconfig.build.cli.json'),
+ '-p',
+ 'config/tsconfig.build.cli.json',
+ '--outDir',
+ 'out',
+ '--composite',
+ 'false',
+ '--incremental',
+ 'false',
+ '--noCheck'
+ ]
+ })
+ )
+})
+
+it('does not emit when checking fails or the pinned Bun is missing', async () => {
+ const execute = vi.fn()
+ await expect(
+ buildCli({
+ check: async () => [new Error('type error')],
+ execute,
+ resolveBunExecutable: () => 'pinned-bun'
+ })
+ ).rejects.toThrow('type error')
+ await expect(buildCli({ execute, resolveBunExecutable: () => null })).rejects.toThrow(
+ 'run pnpm install'
+ )
+ expect(execute).not.toHaveBeenCalled()
+})
+
+it('propagates emitter failures', async () => {
+ await expect(
+ buildCli({
+ check: async () => [],
+ resolveBunExecutable: () => 'pinned-bun',
+ execute: async () => ({ code: null, signal: 'SIGKILL' })
+ })
+ ).rejects.toThrow('signal=SIGKILL')
+})
+
+it('produces byte-identical JavaScript, declarations and maps with the real compilers', async () => {
+ const root = fixture()
+ const baseline = await runProcess({
+ program: process.execPath,
+ args: [
+ resolveProjectCompiler('config/tsconfig.build.cli.json', root),
+ '-p',
+ 'config/tsconfig.build.cli.json',
+ '--outDir',
+ 'baseline',
+ '--composite',
+ 'false',
+ '--incremental',
+ 'false'
+ ],
+ cwd: root
+ })
+ expect(baseline.code, baseline.stdout + baseline.stderr).toBe(0)
+ await buildCli({ root, outDir: 'optimized', resolveBunExecutable })
+ const files = readdirSync(join(root, 'baseline')).sort()
+ expect(readdirSync(join(root, 'optimized')).sort()).toEqual(files)
+ expect(files).toHaveLength(8)
+ for (const file of files) {
+ expect(
+ readFileSync(join(root, 'optimized', file)).equals(
+ readFileSync(join(root, 'baseline', file))
+ ),
+ file
+ ).toBe(true)
+ }
+})
+
+it('blocks output for real type errors and rejects compiler version drift', async () => {
+ const root = fixture()
+ writeFileSync(join(root, 'src', 'entry.ts'), 'export const value: number = "wrong"\n')
+ const execute = vi.fn()
+ await expect(buildCli({ root, execute, resolveBunExecutable })).rejects.toThrow(
+ 'exited with code 1'
+ )
+ expect(execute).not.toHaveBeenCalled()
+
+ const drift = fixture()
+ rmSync(join(drift, 'node_modules'))
+ mkdirSync(join(drift, 'node_modules', 'typescript'), { recursive: true })
+ writeFileSync(
+ join(drift, 'node_modules', 'typescript', 'package.json'),
+ JSON.stringify({
+ name: 'typescript',
+ version: '99.0.0',
+ bin: { tsc: 'compiler.cjs' }
+ })
+ )
+ await expect(buildCli({ root: drift, execute, resolveBunExecutable })).rejects.toThrow(
+ 'resolves 99.0.0'
+ )
+ expect(execute).not.toHaveBeenCalled()
+})
diff --git a/config/scripts/build-mac-notarization-overlap.mjs b/config/scripts/build-mac-notarization-overlap.mjs
new file mode 100644
index 000000000000..9822fcb16b1d
--- /dev/null
+++ b/config/scripts/build-mac-notarization-overlap.mjs
@@ -0,0 +1,161 @@
+import { execFileSync } from 'node:child_process'
+import { createRequire } from 'node:module'
+import { join, resolve } from 'node:path'
+
+const ARCHITECTURES = [1, 3] // electron-builder: x64, arm64.
+
+export async function buildWithOverlappingNotarization({
+ config,
+ build,
+ notarize,
+ verify,
+ targets,
+ publish,
+ credentials,
+ log = console.log
+}) {
+ if (!config.forceCodeSigning || !config.mac?.hardenedRuntime || config.mac.notarize !== true) {
+ throw new Error('Overlapping notarization requires the complete macOS release signing policy')
+ }
+ for (const hook of ['afterSign', 'artifactBuildStarted']) {
+ if (config[hook] !== undefined && typeof config[hook] !== 'function') {
+ throw new Error(`Unsupported ${hook} hook: expected a function`)
+ }
+ }
+ const tasks = new Map()
+ const paths = new Set()
+ let aborted = false
+ let resolvePrepared
+ let rejectPrepared
+ const prepared = new Promise((resolve, reject) => {
+ resolvePrepared = resolve
+ rejectPrepared = reject
+ })
+ // The first target can be waiting while the second architecture is still signing.
+ prepared.catch(() => {})
+ const waitForNotarization = async () => {
+ await prepared
+ const results = await Promise.all(tasks.values())
+ const failure = results.find((result) => 'error' in result)
+ if (failure) {
+ throw failure.error
+ }
+ if (aborted) {
+ throw new Error('macOS packaging was aborted')
+ }
+ }
+ try {
+ const artifacts = await build({
+ targets,
+ publish,
+ config: {
+ ...config,
+ // Only this invocation replaces builder's blocking notary wait with the guarded hooks.
+ mac: { ...config.mac, notarize: false },
+ concurrency: { ...config.concurrency, jobs: 1 },
+ afterSign: async (context) => {
+ await config.afterSign?.(context)
+ if (context.electronPlatformName !== 'darwin' || !ARCHITECTURES.includes(context.arch)) {
+ throw new Error('Unexpected platform or architecture in macOS signing hook')
+ }
+ const appPath = join(context.appOutDir, `${context.packager.appInfo.productFilename}.app`)
+ if (tasks.has(context.arch) || paths.has(appPath)) {
+ throw new Error('macOS architectures must have separate, freshly signed app bundles')
+ }
+ paths.add(appPath)
+ const task = Promise.resolve()
+ .then(async () => {
+ log(`[mac-notary] submitting ${appPath}`)
+ await notarize({ appPath, ...credentials })
+ await verify(appPath)
+ log(`[mac-notary] accepted, stapled and verified ${appPath}`)
+ return {}
+ })
+ .catch((error) => {
+ rejectPrepared(error)
+ return { error }
+ })
+ tasks.set(context.arch, task)
+ if (tasks.size === ARCHITECTURES.length) {
+ resolvePrepared()
+ }
+ },
+ artifactBuildStarted: async (event) => {
+ if (!ARCHITECTURES.includes(event.arch)) {
+ throw new Error('Unexpected architecture in macOS artifact hook')
+ }
+ await waitForNotarization()
+ await config.artifactBuildStarted?.(event)
+ }
+ }
+ })
+ if (tasks.size !== ARCHITECTURES.length) {
+ throw new Error('Both macOS architectures must be signed before artifacts can succeed')
+ }
+ await waitForNotarization()
+ return artifacts
+ } catch (error) {
+ aborted = true
+ rejectPrepared(error)
+ throw error
+ } finally {
+ // Drain outstanding notary work before a failed invocation returns to retry.
+ await Promise.all(tasks.values())
+ }
+}
+
+export async function main(args = process.argv.slice(2)) {
+ if (process.platform !== 'darwin') {
+ throw new Error('macOS release packaging requires macOS')
+ }
+ const publish = args[0] === '--publish' && ['always', 'never'].includes(args[1]) ? args[1] : null
+ if (!publish || args.slice(2).some((arg) => arg !== '--draft') || args.length > 3) {
+ throw new Error('Usage: build-mac-notarization-overlap.mjs --publish always|never [--draft]')
+ }
+ const require = createRequire(import.meta.url)
+ const { build, Platform, Arch } = require('electron-builder')
+ // Use the same pinned notarization implementation as electron-builder, including its retries.
+ const builderRequire = createRequire(require.resolve('electron-builder'))
+ const { notarize } = createRequire(builderRequire.resolve('app-builder-lib'))(
+ '@electron/notarize'
+ )
+ const config = require('../electron-builder.config.cjs')
+ if (args.includes('--draft')) {
+ config.publish = { ...config.publish, releaseType: 'draft' }
+ }
+ const credentials = {
+ appleId: process.env.APPLE_ID,
+ appleIdPassword: process.env.APPLE_APP_SPECIFIC_PASSWORD,
+ teamId: process.env.APPLE_TEAM_ID
+ }
+ if (Object.values(credentials).some((value) => !value?.trim())) {
+ throw new Error('macOS notarization credentials are incomplete')
+ }
+ return buildWithOverlappingNotarization({
+ config,
+ build,
+ notarize,
+ verify: (appPath) => {
+ execFileSync('codesign', ['--verify', '--deep', '--strict', appPath], { stdio: 'inherit' })
+ execFileSync('xcrun', ['stapler', 'validate', appPath], { stdio: 'inherit' })
+ },
+ targets: new Map([
+ [
+ Platform.MAC,
+ new Map([
+ [Arch.x64, ['dmg', 'zip']],
+ [Arch.arm64, ['dmg', 'zip']]
+ ])
+ ]
+ ]),
+ publish,
+ credentials
+ })
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) {
+ main().catch((error) => {
+ console.error(error)
+ process.exitCode = 1
+ })
+}
diff --git a/config/scripts/build-mac-notarization-overlap.test.mjs b/config/scripts/build-mac-notarization-overlap.test.mjs
new file mode 100644
index 000000000000..dc90c0f9ba53
--- /dev/null
+++ b/config/scripts/build-mac-notarization-overlap.test.mjs
@@ -0,0 +1,229 @@
+import { describe, expect, it, vi } from 'vitest'
+import { buildWithOverlappingNotarization } from './build-mac-notarization-overlap.mjs'
+
+const deferred = () => {
+ let resolve, reject
+ const promise = new Promise((yes, no) => {
+ resolve = yes
+ reject = no
+ })
+ return { promise, resolve, reject }
+}
+const context = (arch) => ({
+ arch,
+ electronPlatformName: 'darwin',
+ appOutDir: `dist/mac-${arch}`,
+ packager: { appInfo: { productFilename: 'Orca' } }
+})
+const signedConfig = () => ({
+ forceCodeSigning: true,
+ mac: { notarize: true, hardenedRuntime: true },
+ concurrency: { jobs: 8 },
+ publish: { releaseType: 'draft' }
+})
+const options = (build, notarize = vi.fn(async () => {}), verify = vi.fn()) => ({
+ config: signedConfig(),
+ build,
+ notarize,
+ verify,
+ targets: 'both-mac-targets',
+ publish: 'never',
+ credentials: { appleId: 'test' },
+ log: vi.fn()
+})
+
+// The real builder queues each artifact task without awaiting it in its serial pack loop.
+function queuedBuilder(events, afterSign) {
+ return async ({ config, ...buildOptions }) => {
+ expect(buildOptions).toMatchObject({ targets: 'both-mac-targets', publish: 'never' })
+ expect(config.mac).toMatchObject({ notarize: false, hardenedRuntime: true })
+ expect(config.forceCodeSigning).toBe(true)
+ expect(config.concurrency.jobs).toBe(1)
+ const artifacts = []
+ for (const arch of [1, 3]) {
+ events.push(`pack:${arch}`)
+ await config.afterSign(context(arch))
+ artifacts.push(
+ (async () => {
+ await config.artifactBuildStarted({ arch })
+ events.push(`archive:${arch}`)
+ })()
+ )
+ await afterSign?.(arch)
+ }
+ await Promise.all(artifacts)
+ return ['x64.dmg', 'arm64.dmg']
+ }
+}
+
+describe('overlapping macOS notarization', () => {
+ it('reduces a modeled pair of 4-second Apple waits from 10 to 6 seconds', async () => {
+ vi.useFakeTimers()
+ try {
+ const started = Date.now()
+ const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+ const build = async ({ config }) => {
+ const artifacts = []
+ for (const arch of [1, 3]) {
+ await delay(1000)
+ await config.afterSign(context(arch))
+ artifacts.push(config.artifactBuildStarted({ arch }))
+ }
+ await Promise.all(artifacts)
+ return []
+ }
+ const result = buildWithOverlappingNotarization(options(build, () => delay(4000)))
+ await vi.runAllTimersAsync()
+ await result
+ expect(Date.now() - started).toBe(6000)
+ expect(2 * (1000 + 4000)).toBe(10000)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('packs serially, overlaps Apple waits, and gates every artifact on both verified tickets', async () => {
+ const events = []
+ const x64 = deferred(),
+ arm64 = deferred(),
+ bothSubmitted = deferred()
+ const notarize = vi.fn(({ appPath }) => {
+ events.push(`submit:${appPath}`)
+ if (appPath.includes('mac-3')) {
+ bothSubmitted.resolve()
+ }
+ return appPath.includes('mac-1') ? x64.promise : arm64.promise
+ })
+ const verify = vi.fn((appPath) => {
+ events.push(`verify:${appPath}`)
+ })
+ const result = buildWithOverlappingNotarization(
+ options(queuedBuilder(events), notarize, verify)
+ )
+ await bothSubmitted.promise
+ expect(events.filter((event) => event.startsWith('pack:'))).toEqual(['pack:1', 'pack:3'])
+ x64.resolve()
+ await vi.waitFor(() => expect(verify).toHaveBeenCalledTimes(1))
+ expect(events.some((event) => event.startsWith('archive:'))).toBe(false)
+ arm64.resolve()
+ await expect(result).resolves.toEqual(['x64.dmg', 'arm64.dmg'])
+ expect(events.findIndex((event) => event.startsWith('archive:'))).toBeGreaterThan(
+ events.findLastIndex((event) => event.startsWith('verify:'))
+ )
+ })
+
+ it.each([1, 3])('never archives when architecture %i notarization fails', async (failedArch) => {
+ const events = []
+ const failure = new Error('Apple rejected the app')
+ const notarize = vi.fn(async ({ appPath }) => {
+ if (appPath.includes(`mac-${failedArch}`)) {
+ throw failure
+ }
+ })
+ await expect(
+ buildWithOverlappingNotarization(options(queuedBuilder(events), notarize))
+ ).rejects.toBe(failure)
+ expect(events.some((event) => event.startsWith('archive:'))).toBe(false)
+ })
+
+ it('never archives when signature or stapler validation fails', async () => {
+ const events = []
+ await expect(
+ buildWithOverlappingNotarization(
+ options(queuedBuilder(events), undefined, () => {
+ throw new Error('invalid staple')
+ })
+ )
+ ).rejects.toThrow('invalid staple')
+ expect(events.some((event) => event.startsWith('archive:'))).toBe(false)
+ })
+
+ it('rejects waiting artifacts and drains the first notary if the second pack fails', async () => {
+ const waiting = deferred(),
+ started = deferred()
+ let artifactResult
+ const failure = new Error('second native rebuild failed')
+ const build = async ({ config }) => {
+ await config.afterSign(context(1))
+ artifactResult = config.artifactBuildStarted({ arch: 1 }).catch((error) => error)
+ throw failure
+ }
+ const result = buildWithOverlappingNotarization(
+ options(build, async () => {
+ started.resolve()
+ await waiting.promise
+ })
+ )
+ const settled = vi.fn()
+ result.catch(settled)
+ await started.promise
+ expect(await artifactResult).toBe(failure)
+ expect(settled).not.toHaveBeenCalled()
+ waiting.resolve()
+ await expect(result).rejects.toBe(failure)
+ })
+
+ it('does not mask a build failure with a later notary failure', async () => {
+ const failure = new Error('pack failed')
+ const build = async ({ config }) => {
+ await config.afterSign(context(1))
+ throw failure
+ }
+ await expect(
+ buildWithOverlappingNotarization(
+ options(build, async () => {
+ throw new Error('notary failed too')
+ })
+ )
+ ).rejects.toBe(failure)
+ })
+
+ it('preserves hooks and runs artifact hooks after verification', async () => {
+ const events = []
+ const config = signedConfig()
+ config.afterSign = vi.fn()
+ config.artifactBuildStarted = vi.fn()
+ await buildWithOverlappingNotarization({ ...options(queuedBuilder(events)), config })
+ expect(config.afterSign).toHaveBeenCalledTimes(2)
+ expect(config.artifactBuildStarted).toHaveBeenCalledTimes(2)
+ expect(config.mac.notarize).toBe(true)
+ })
+
+ it.each([
+ { forceCodeSigning: false },
+ { mac: { notarize: false, hardenedRuntime: true } },
+ { mac: { notarize: true, hardenedRuntime: false } },
+ { afterSign: 'hook.cjs' }
+ ])('fails closed on an unsupported signing policy: %j', async (override) => {
+ const build = vi.fn()
+ await expect(
+ buildWithOverlappingNotarization({
+ ...options(build),
+ config: { ...signedConfig(), ...override }
+ })
+ ).rejects.toThrow()
+ expect(build).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ [context(1), context(1)],
+ [context(1), { ...context(3), appOutDir: 'dist/mac-1' }],
+ [{ ...context(1), electronPlatformName: 'linux' }],
+ [context(4)]
+ ])('rejects duplicate, shared, or unexpected app bundles: %j', async (...contexts) => {
+ const build = async ({ config }) => {
+ for (const value of contexts) {
+ await config.afterSign(value)
+ }
+ }
+ await expect(buildWithOverlappingNotarization(options(build))).rejects.toThrow()
+ })
+
+ it('rejects a build that silently omits an architecture', async () => {
+ const build = async ({ config }) => {
+ await config.afterSign(context(1))
+ return []
+ }
+ await expect(buildWithOverlappingNotarization(options(build))).rejects.toThrow('Both macOS')
+ })
+})
diff --git a/config/scripts/build-mobile-web-app-bundle.mjs b/config/scripts/build-mobile-web-app-bundle.mjs
index 1f6ce2e19df5..ce882911c779 100644
--- a/config/scripts/build-mobile-web-app-bundle.mjs
+++ b/config/scripts/build-mobile-web-app-bundle.mjs
@@ -34,6 +34,8 @@ const entryPoint = join(mobileDir, 'web-entry', 'index.tsx')
const { MOBILE_WEB_BUNDLE_DIR: defaultOutDir } = createRequire(import.meta.url)(
'./verify-packaged-mobile-web-bundle.cjs'
)
+// The desktop serves this page, so a dark switch the desktop was built with reaches it too.
+const { turnedOnDarkBuildDefines } = createRequire(import.meta.url)('../dark-build-switches.cjs')
/**
* Every shim the app bundle needs, each one a documented Metro/RN-Web gap. `appliesTo` reads the
@@ -347,7 +349,9 @@ export function mobileWebAppBuildOptions(routes) {
__DEV__: 'false',
'process.env.NODE_ENV': '"production"',
'process.env.EXPO_OS': '"web"',
- 'process.env.EXPO_ROUTER_IMPORT_MODE': '"sync"'
+ 'process.env.EXPO_ROUTER_IMPORT_MODE': '"sync"',
+ // Only a switch asked for: an off build stays byte-identical, and its switch reads off.
+ ...turnedOnDarkBuildDefines()
}
}
}
diff --git a/config/scripts/build-orcad-template.mjs b/config/scripts/build-orcad-template.mjs
index f5b2a9552755..f477c2b012f6 100644
--- a/config/scripts/build-orcad-template.mjs
+++ b/config/scripts/build-orcad-template.mjs
@@ -38,7 +38,7 @@ import {
import { runProcessSync } from '@orca/process-host'
import { verifyPackagedOrcadTemplate } from './verify-packaged-orcad-template.cjs'
import { readOrcadMobileWebAssetFilenames } from './orcad-mobile-web-bundle.mjs'
-import { assertNotDarkBuild } from '../dark-build-switches.cjs'
+import { assertNotDarkBuild, darkBuildMarkers } from '../dark-build-switches.cjs'
const root = resolve(import.meta.dirname, '../..')
const outputDir = join(root, 'out', 'orcad-template')
@@ -175,13 +175,23 @@ function stageCompatTarget(compat, basePackageDir) {
return { files }
}
+/**
+ * `--dark-test-template` lets a dark switch through for test servers only; the template then
+ * carries the switch's marker, which packaging refuses.
+ */
+export function assertTemplateBuildAllowed(argv = process.argv, env = process.env) {
+ if (!argv.includes('--dark-test-template')) {
+ // The template is what releases ship as the server; a dark build must never become one.
+ assertNotDarkBuild([], env)
+ }
+}
+
function sameBytes(left, right) {
return sha256(left) === sha256(right)
}
async function main() {
- // The template is what releases ship as the server; a dark build must never become one.
- assertNotDarkBuild([])
+ assertTemplateBuildAllowed()
const templateTargets = requestedTemplateTargets()
rmSync(buildDir, { recursive: true, force: true })
const packages = Object.fromEntries(
@@ -224,6 +234,10 @@ async function main() {
`${JSON.stringify({ schemaVersion: 3, commonSha256, targets }, null, 2)}\n`
)
verifyPackagedOrcadTemplate(join(root, 'out'), templateTargets)
+ // Outside the manifest, so deploys never copy it; it marks the whole template as test-only.
+ for (const marker of darkBuildMarkers()) {
+ writeFileSync(join(outputDir, marker.fileName), marker.text)
+ }
rmSync(buildDir, { recursive: true, force: true })
process.stdout.write(`[build-orcad-template] ok — ${Object.keys(targets).length} targets\n`)
}
diff --git a/config/scripts/build-orcad-template.test.mjs b/config/scripts/build-orcad-template.test.mjs
index c501b25a6a9d..f39cb62879f2 100644
--- a/config/scripts/build-orcad-template.test.mjs
+++ b/config/scripts/build-orcad-template.test.mjs
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest'
import { ORCAD_TEMPLATE_TARGETS } from '../../src/shared/node-runtime-pin.ts'
-import { requestedTemplateTargets } from './build-orcad-template.mjs'
+import { assertTemplateBuildAllowed, requestedTemplateTargets } from './build-orcad-template.mjs'
describe('requestedTemplateTargets', () => {
it('builds every target unless a subset is named', () => {
@@ -29,3 +29,21 @@ describe('requestedTemplateTargets', () => {
}
)
})
+
+describe('assertTemplateBuildAllowed', () => {
+ const argv = ['node', 'build-orcad-template.mjs']
+ const dark = { ORCA_LAYOUT_OWNER: 'main' }
+
+ it('refuses a dark switch for a release template', () => {
+ expect(() => assertTemplateBuildAllowed(argv, dark)).toThrow('ORCA_LAYOUT_OWNER=main')
+ })
+
+ it('lets a dark switch through only for an explicit test template', () => {
+ expect(() => assertTemplateBuildAllowed([...argv, '--dark-test-template'], dark)).not.toThrow()
+ })
+
+ it('allows a plain template either way', () => {
+ expect(() => assertTemplateBuildAllowed(argv, {})).not.toThrow()
+ expect(() => assertTemplateBuildAllowed([...argv, '--dark-test-template'], {})).not.toThrow()
+ })
+})
diff --git a/config/scripts/build-orcad.mjs b/config/scripts/build-orcad.mjs
index 17d25f1c1007..057636d8a116 100644
--- a/config/scripts/build-orcad.mjs
+++ b/config/scripts/build-orcad.mjs
@@ -60,6 +60,8 @@ import { runProcessSync } from '@orca/process-host'
import { describeProcessFailure } from './process-failure-message.mjs'
import { resolveBuildSourceCommit } from './build-source-commit.mjs'
+const { darkBuildMarkers } = createRequire(import.meta.url)('../dark-build-switches.cjs')
+
const ROOT = join(import.meta.dirname, '..', '..')
const OUT_DIR = process.env.ORCAD_OUT_DIR
? resolve(process.env.ORCAD_OUT_DIR)
@@ -159,6 +161,10 @@ function stageNodePty(target) {
rmSync(OUT_DIR, { recursive: true, force: true })
mkdirSync(OUT_DIR, { recursive: true })
writeFileSync(join(OUT_DIR, ORCAD_SERVER_TARGET_FILENAME), `${BUILD_TARGET}\n`)
+// Outside the artifact list, so the version hash and every deploy ignore it.
+for (const marker of darkBuildMarkers()) {
+ writeFileSync(join(OUT_DIR, marker.fileName), marker.text)
+}
writeFileSync(
join(OUT_DIR, ORCAD_NODE_RUNTIME_MARKER_FILENAME),
`${runtimeAsset.executableSha256}\n`
diff --git a/config/scripts/build-wayland-clipboard.test.mjs b/config/scripts/build-wayland-clipboard.test.mjs
index 64ebce4770f7..32cd61fe8778 100644
--- a/config/scripts/build-wayland-clipboard.test.mjs
+++ b/config/scripts/build-wayland-clipboard.test.mjs
@@ -173,6 +173,7 @@ describe('Wayland clipboard CI cache recovery', () => {
fi
return "$REBUILD_STATUS"
}
+ export -f node
${build.run}
`
const result = runProcessSync({
@@ -181,6 +182,7 @@ describe('Wayland clipboard CI cache recovery', () => {
cwd: root,
env: {
...process.env,
+ GITHUB_ACTION_PATH: join(process.cwd(), '.github/actions/prepare-wayland-clipboard'),
CACHE_HIT: String(hit),
INITIAL_STATUS: String(initial),
REBUILD_STATUS: String(rebuild)
diff --git a/config/scripts/build-windows-process-tree-addon.mjs b/config/scripts/build-windows-process-tree-addon.mjs
index 71a745f440b9..b7486a58089e 100644
--- a/config/scripts/build-windows-process-tree-addon.mjs
+++ b/config/scripts/build-windows-process-tree-addon.mjs
@@ -19,9 +19,21 @@
* node config/scripts/build-windows-process-tree-addon.mjs --arch=arm64
*/
import { execFileSync } from 'node:child_process'
-import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import {
+ copyFileSync,
+ cpSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ writeFileSync
+} from 'node:fs'
import { createRequire } from 'node:module'
-import { join, resolve } from 'node:path'
+import { tmpdir } from 'node:os'
+import { join, relative, resolve } from 'node:path'
+import { isSharedPackageDir } from './native-build-entry.cjs'
import { canReusePreparedAddon } from './windows-process-tree-prepared-addon.mjs'
import { ORCAD_WINDOWS_PROCESS_TREE_FILENAME } from '../../src/shared/orcad-artifacts.ts'
import {
@@ -30,7 +42,7 @@ import {
nodeGypRebuildInvocation,
stageWindowsProcessTreeNodeAddonApiHeaders,
windowsProcessTreeAddonHasBreakawayLauncher,
- WINDOWS_PROCESS_TREE_PACKAGE_DIR as PACKAGE_DIR
+ WINDOWS_PROCESS_TREE_PACKAGE_DIR as INSTALLED_PACKAGE_DIR
} from './windows-process-tree-gyp-rebuild.mjs'
const { PE_MACHINE, describePeMachine, readPeMachine } = createRequire(import.meta.url)(
@@ -39,6 +51,21 @@ const { PE_MACHINE, describePeMachine, readPeMachine } = createRequire(import.me
const ROOT = resolve(import.meta.dirname, '..', '..')
const SUPPORTED_ARCHES = ['x64', 'arm64']
+// Why: under pnpm's global virtual store the installed dir is every worktree's, so the source
+// repairs and the build below run in a private copy instead.
+const SCRATCH_DIR =
+ existsSync(INSTALLED_PACKAGE_DIR) && isSharedPackageDir(INSTALLED_PACKAGE_DIR, ROOT)
+ ? mkdtempSync(join(tmpdir(), 'orca-windows-process-tree-'))
+ : null
+const PACKAGE_DIR = SCRATCH_DIR === null ? INSTALLED_PACKAGE_DIR : join(SCRATCH_DIR, 'package')
+if (SCRATCH_DIR !== null) {
+ const physicalDir = realpathSync(INSTALLED_PACKAGE_DIR)
+ cpSync(physicalDir, PACKAGE_DIR, {
+ recursive: true,
+ dereference: true,
+ filter: (path) => relative(physicalDir, path) !== 'build'
+ })
+}
function parseArgs(argv) {
const arch = argv.find((a) => a.startsWith('--arch='))?.slice('--arch='.length) ?? process.arch
@@ -409,7 +436,7 @@ function applyWindowsProcessTreeBuildFixes() {
writeFileSync(processPath, processCc)
}
const repairedCreationTime = repairCreationTimeSources()
- stageWindowsProcessTreeNodeAddonApiHeaders(PACKAGE_DIR)
+ stageWindowsProcessTreeNodeAddonApiHeaders(PACKAGE_DIR, INSTALLED_PACKAGE_DIR)
const repairedCommandLine = ensureWindowsProcessTreeCommandLinePatch(PACKAGE_DIR)
const repaired =
bindingGyp !== originalBinding ||
@@ -447,7 +474,7 @@ function main() {
) {
console.log(`[windows-process-tree] reusing the prepared ${arch} addon`)
} else {
- const gyp = nodeGypRebuildInvocation(arch)
+ const gyp = nodeGypRebuildInvocation(arch, PACKAGE_DIR)
console.log(`[windows-process-tree] building ${arch} from ${gyp.cwd}`)
execFileSync(process.execPath, gyp.args, { cwd: gyp.cwd, stdio: 'inherit' })
}
@@ -489,5 +516,9 @@ try {
main()
} catch (error) {
console.error(`[windows-process-tree] ${error instanceof Error ? error.message : String(error)}`)
- process.exit(1)
+ process.exitCode = 1
+} finally {
+ if (SCRATCH_DIR !== null) {
+ rmSync(SCRATCH_DIR, { recursive: true, force: true })
+ }
}
diff --git a/config/scripts/check-deprecated-runtime-owner-ratchet.mjs b/config/scripts/check-deprecated-runtime-owner-ratchet.mjs
deleted file mode 100644
index a28544037379..000000000000
--- a/config/scripts/check-deprecated-runtime-owner-ratchet.mjs
+++ /dev/null
@@ -1,70 +0,0 @@
-import process from 'node:process'
-import { pathToFileURL } from 'node:url'
-import { readTrackedSources, runEntryRatchet } from './ratchet-baseline.mjs'
-
-// Ratchet on renderer files that import the deprecated runtime-owner resolvers. Both read a
-// worktree that two hosts publish as "not a runtime", which callers then treat as local, so a stop,
-// sleep or kill can land on the wrong machine. New code resolves the owner with its host and
-// ambiguity kept; the importer list may only shrink.
-
-const BASELINE_PATH = 'config/deprecated-runtime-owner-importers-baseline.txt'
-const SCAN_ROOT = 'src/renderer/src'
-const DEPRECATED_NAMES = ['getRuntimeEnvironmentIdForWorktree', 'resolveRuntimeOwnerForWorktree']
-// The defining module and its own tests are not importers.
-const DEFINING_FILE = 'src/renderer/src/lib/worktree-runtime-owner.ts'
-
-// Any reference to a deprecated name in a file that loads the defining module, whatever the import
-// form (named, aliased, namespace, dynamic, `.then` destructuring, re-export). Simpler than tracking
-// bindings, and in production code a mention of these names is a use.
-const DEPRECATED_NAME = new RegExp(`\\b(?:${DEPRECATED_NAMES.join('|')})\\b`)
-const DEFINING_MODULE_SPECIFIER = /['"`][^'"`]*\/worktree-runtime-owner['"`]/
-// Block and line comments; `://` inside a string (a URL) is not a line comment.
-const COMMENT = /\/\*[\s\S]*?\*\/|(? importsDeprecatedRuntimeOwner(source))
- .map(([rel]) => rel)
- .sort()
-}
-
-const RATCHET = {
- name: 'deprecated-runtime-owner',
- baselinePath: BASELINE_PATH,
- header: [
- '# Renderer files that import getRuntimeEnvironmentIdForWorktree / resolveRuntimeOwnerForWorktree.',
- '# This is a RATCHET: the list may only SHRINK. Both resolvers read an ambiguous (two-host)',
- '# worktree as "not a runtime", which callers treat as local.',
- '# Prune after migrating a file: pnpm check:deprecated-runtime-owner-ratchet --prune'
- ],
- collect: collectDeprecatedRuntimeOwnerImporters,
- fix: [
- 'getRuntimeEnvironmentIdForWorktree / resolveRuntimeOwnerForWorktree collapse an owner that two',
- 'hosts publish into "local". Resolve the owner with its host kept and refuse on ambiguity',
- 'instead (see src/renderer/src/lib/resolve-owner.ts).'
- ]
-}
-
-export function main(root = process.cwd(), arg = undefined) {
- return runEntryRatchet(RATCHET, root, arg)
-}
-
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
- process.exit(main(process.cwd(), process.argv[2]))
-}
diff --git a/config/scripts/check-deprecated-runtime-owner-ratchet.test.mjs b/config/scripts/check-deprecated-runtime-owner-ratchet.test.mjs
deleted file mode 100644
index 8de50f668909..000000000000
--- a/config/scripts/check-deprecated-runtime-owner-ratchet.test.mjs
+++ /dev/null
@@ -1,101 +0,0 @@
-import { describe, expect, it } from 'vitest'
-
-import {
- importsDeprecatedRuntimeOwner,
- isScannedPath
-} from './check-deprecated-runtime-owner-ratchet.mjs'
-
-describe('importsDeprecatedRuntimeOwner', () => {
- it('finds a named import of either deprecated resolver', () => {
- expect(
- importsDeprecatedRuntimeOwner(
- "import { getRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner'\n"
- )
- ).toBe(true)
- expect(
- importsDeprecatedRuntimeOwner(
- "import {\n foo,\n resolveRuntimeOwnerForWorktree\n} from './worktree-runtime-owner'\n"
- )
- ).toBe(true)
- })
-
- it('finds an aliased import, a re-export and a dynamic import', () => {
- expect(
- importsDeprecatedRuntimeOwner(
- "import { getRuntimeEnvironmentIdForWorktree as envFor } from '@/lib/worktree-runtime-owner'"
- )
- ).toBe(true)
- expect(
- importsDeprecatedRuntimeOwner(
- "export { resolveRuntimeOwnerForWorktree } from './worktree-runtime-owner'"
- )
- ).toBe(true)
- expect(
- importsDeprecatedRuntimeOwner(
- "const { getRuntimeEnvironmentIdForWorktree } = await import('@/lib/worktree-runtime-owner')"
- )
- ).toBe(true)
- })
-
- it('finds a use through a namespace import or a module-valued dynamic import', () => {
- expect(
- importsDeprecatedRuntimeOwner(
- "import * as owner from '@/lib/worktree-runtime-owner'\nowner.getRuntimeEnvironmentIdForWorktree(s, id)"
- )
- ).toBe(true)
- expect(
- importsDeprecatedRuntimeOwner(
- "const mod = await import('@/lib/worktree-runtime-owner')\nmod.resolveRuntimeOwnerForWorktree(s, id)"
- )
- ).toBe(true)
- })
-
- it('finds destructuring in a promise-based dynamic import', () => {
- expect(
- importsDeprecatedRuntimeOwner(
- "import('@/lib/worktree-runtime-owner').then(({ resolveRuntimeOwnerForWorktree }) => resolveRuntimeOwnerForWorktree(s, id))"
- )
- ).toBe(true)
- })
-
- it('reads through comments inside an import list, and ignores a commented-out import', () => {
- expect(
- importsDeprecatedRuntimeOwner(
- "import { /* legacy */ getRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner'"
- )
- ).toBe(true)
- expect(
- importsDeprecatedRuntimeOwner(
- "import {\n a, // keep\n resolveRuntimeOwnerForWorktree\n} from './worktree-runtime-owner'"
- )
- ).toBe(true)
- expect(
- importsDeprecatedRuntimeOwner(
- "// import { getRuntimeEnvironmentIdForWorktree } from '@/lib/worktree-runtime-owner'\n/* owner.resolveRuntimeOwnerForWorktree() */"
- )
- ).toBe(false)
- })
-
- it('ignores other names from the module, and same-named functions from other modules', () => {
- expect(
- importsDeprecatedRuntimeOwner(
- "import { getExecutionHostIdForWorktree } from '@/lib/worktree-runtime-owner'"
- )
- ).toBe(false)
- expect(
- importsDeprecatedRuntimeOwner(
- "import { getRuntimeEnvironmentIdForWorktree } from '@/lib/other-owner'"
- )
- ).toBe(false)
- })
-})
-
-describe('isScannedPath', () => {
- it('scans renderer sources only, not tests or the defining module', () => {
- expect(isScannedPath('src/renderer/src/store/terminals/terminal-shutdown.ts')).toBe(true)
- expect(isScannedPath('src/renderer/src/store/terminals/terminal-shutdown.test.ts')).toBe(false)
- expect(isScannedPath('src/renderer/src/store/host-twin.test-fixture.ts')).toBe(false)
- expect(isScannedPath('src/renderer/src/lib/worktree-runtime-owner.ts')).toBe(false)
- expect(isScannedPath('src/main/runtime/foo.ts')).toBe(false)
- })
-})
diff --git a/config/scripts/check-e2e-typecheck-ratchet.mjs b/config/scripts/check-e2e-typecheck-ratchet.mjs
index 941b6d99d039..6dfad74d6606 100644
--- a/config/scripts/check-e2e-typecheck-ratchet.mjs
+++ b/config/scripts/check-e2e-typecheck-ratchet.mjs
@@ -1,8 +1,8 @@
-import { spawnSync } from 'node:child_process'
import fs from 'node:fs'
import path from 'node:path'
import process from 'node:process'
import { pathToFileURL } from 'node:url'
+import { runE2eTypecheck } from './run-e2e-typecheck.mjs'
import {
diffCounts,
parseCountBaseline,
@@ -15,7 +15,6 @@ import {
// lane runs it. Counts per file may only go down.
const BASELINE_PATH = 'config/e2e-typecheck-baseline.txt'
-const TSCONFIG = 'config/tsconfig.e2e.json'
const HEADER = [
'# Type errors per file under tests/ (tsc -p config/tsconfig.e2e.json).',
'# This is a RATCHET: counts may only go DOWN. Fix the errors in a file you touch.',
@@ -37,21 +36,11 @@ export function countTestTypeErrors(tscOutput) {
return counts
}
-function runTsc(root) {
- const result = spawnSync(
- process.execPath,
- ['node_modules/typescript/bin/tsc', '--noEmit', '--pretty', 'false', '-p', TSCONFIG],
- { cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }
- )
- const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
- // tsc exits 2 when it reports errors; anything else without diagnostics means it never ran.
- if (result.error || (result.status !== 0 && !/error TS\d+:/.test(output))) {
- throw new Error(`tsc -p ${TSCONFIG} failed to run:\n${result.error ?? output}`)
- }
- return output
-}
-
-export function main(root = process.cwd(), arg = undefined, tscOutput = runTsc(root)) {
+export function main(
+ root = process.cwd(),
+ arg = undefined,
+ tscOutput = runE2eTypecheck({ root }).output
+) {
const current = countTestTypeErrors(tscOutput)
if (arg === '--prune') {
const total = pruneCountBaseline(root, BASELINE_PATH, HEADER, current)
diff --git a/config/scripts/check-feature-wall-assets.mjs b/config/scripts/check-feature-wall-assets.mjs
index c9e3fa028052..d65a228d5d39 100644
--- a/config/scripts/check-feature-wall-assets.mjs
+++ b/config/scripts/check-feature-wall-assets.mjs
@@ -1,29 +1,16 @@
#!/usr/bin/env node
import { readdir, stat } from 'node:fs/promises'
import path from 'node:path'
+import { FEATURE_WALL_TILES, isFeatureWallMediaTile } from '../../src/shared/feature-wall-tiles.ts'
const __dirname = import.meta.dirname
const ROOT = path.join(__dirname, '..', '..')
const FEATURE_WALL_ASSET_DIR = path.join(ROOT, 'resources', 'onboarding', 'feature-wall')
const MAX_BYTES = 11 * 1024 * 1024
-const MEDIA_TILE_IDS = [
- 'tile-01',
- 'tile-02',
- 'tile-03',
- 'tile-04',
- 'tile-05',
- 'tile-06',
- 'tile-07',
- 'tile-08',
- 'tile-09',
- 'tile-10',
- 'tile-11',
- 'tile-12'
-]
-const EXPECTED_FILES = MEDIA_TILE_IDS.flatMap((id) => [
- `${id}.gif`,
- `${id}.poster.jpg`,
- `${id}.recorded-at.json`
+const EXPECTED_FILES = FEATURE_WALL_TILES.filter(isFeatureWallMediaTile).flatMap((tile) => [
+ tile.gifPath,
+ tile.posterPath,
+ tile.recordedAtPath
])
async function collectFiles(dir) {
diff --git a/config/scripts/check-focus-read-ratchet.mjs b/config/scripts/check-focus-read-ratchet.mjs
index 6ba68dc41279..83179dc4ccc5 100644
--- a/config/scripts/check-focus-read-ratchet.mjs
+++ b/config/scripts/check-focus-read-ratchet.mjs
@@ -36,7 +36,9 @@ const SEED_FOCUS_READERS = [
'defaultCreationHost',
'defaultScopeHost',
'defaultScopeSource',
- 'rowLessSourceTarget'
+ 'rowLessSourceTarget',
+ // Its legacy branch returns the focused runtime inside an object literal, which discovery skips.
+ 'resolveOwnerWithLegacyFallback'
]
const FOCUS_SCAN_ROOTS = [SCAN_ROOT, 'src/shared']
@@ -110,7 +112,7 @@ function statementsFrom(body, start) {
break
}
depth -= 1
- } else if (char === '\n' && depth === 0) {
+ } else if (depth === 0 && (char === '\n' || (char === ',' && match[0].startsWith('=>')))) {
break
}
}
diff --git a/config/scripts/check-focus-read-ratchet.test.mjs b/config/scripts/check-focus-read-ratchet.test.mjs
index 3ddbaca03f7b..13d0943b72fc 100644
--- a/config/scripts/check-focus-read-ratchet.test.mjs
+++ b/config/scripts/check-focus-read-ratchet.test.mjs
@@ -193,10 +193,57 @@ describe('countFocusSettingReads', () => {
expect(readers.namesFor('src/renderer/src/hook.ts').has('sideEffectOnly')).toBe(false)
})
+ it('stops an arrow result before the next argument without losing nested reader results', () => {
+ const readers = discoverFocusReaders(
+ new Map([
+ [
+ 'src/renderer/src/focus.ts',
+ 'export function focusedId(s) {\n return s.activeRuntimeEnvironmentId\n}\n'
+ ],
+ [
+ 'src/renderer/src/hook.ts',
+ [
+ "import { focusedId } from './focus'",
+ 'export function useCloseActions(s) {',
+ ' const closeTabs = () => { focusedId(s) }',
+ ' const closeOthers = () => {',
+ ' confirmClose(() => selectIds(s), closeTabs)',
+ ' }',
+ ' return { closeOthers }',
+ '}',
+ 'export const wrappedReader = (s) => forward(null, focusedId(s))',
+ 'export const commaReader = (s) => (null, focusedId(s))'
+ ].join('\n')
+ ]
+ ])
+ )
+ const names = readers.namesFor('src/renderer/src/hook.ts')
+ expect(names.has('useCloseActions')).toBe(false)
+ expect(names.has('wrappedReader')).toBe(true)
+ expect(names.has('commaReader')).toBe(true)
+ })
+
it('counts the default scope source as a read', () => {
expect(countFocusSettingReads('linearConnect(defaultScopeSource(get().settings), key)')).toBe(1)
})
+ it('counts the legacy-fallback owner adapter and a wrapper that returns its result', () => {
+ const readers = discoverFocusReaders(
+ new Map([
+ [
+ 'src/renderer/src/owner.ts',
+ [
+ 'export function envUnlessTwin(state, id) {',
+ ' const match = resolveOwnerWithLegacyFallback(state, { workspaceId: id })',
+ " return match.kind === 'resolved' ? match.owner : null",
+ '}'
+ ].join('\n')
+ ]
+ ])
+ )
+ expect(readers.has('envUnlessTwin')).toBe(true)
+ })
+
it('counts destructuring reads', () => {
expect(countFocusSettingReads('const { activeRuntimeEnvironmentId } = s')).toBe(1)
expect(
diff --git a/config/scripts/check-id-only-store-write-ratchet.mjs b/config/scripts/check-id-only-store-write-ratchet.mjs
new file mode 100644
index 000000000000..b0000aa25c7d
--- /dev/null
+++ b/config/scripts/check-id-only-store-write-ratchet.mjs
@@ -0,0 +1,219 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import process from 'node:process'
+import { pathToFileURL } from 'node:url'
+import { readTrackedSources, runEntryRatchet } from './ratchet-baseline.mjs'
+
+// Ratchet on Store writes addressed by a bare repo or worktree id (R4-2). A local row and an SSH
+// row can share an id, so an id-only write lands on whichever row matches first. Writes take the
+// row's identity instead: `RepoHostTarget` (src/shared/repo-host-identity.ts) or
+// `HostWorktreeTarget` (src/shared/worktree/host-qualified-identity.ts). Entries may only go away.
+// Matched by the first parameter's NAME, whatever its type (`string`, `Repo['id']`, a brand), so a
+// retyped id still counts. Known limit: an id inside an object parameter (`{ projectId }`) is not seen.
+
+const BASELINE_PATH = 'config/id-only-store-write-baseline.txt'
+const COMPOSITION = 'src/main/persistence/loading-store/store-domain-composition.ts'
+// The only type names that make an id-named parameter host-carrying.
+const HOST_TARGET_TYPE = /^(?:RepoHostTarget|HostWorktreeTarget)\b/
+const ID_PARAM = /^(?:id|ids|\w*Ids?|(?:\w*(?:Worktree|Workspace)|worktree|workspace)Keys?)$/
+const READ_METHOD = /^(?:get|list|has|peek|read|is|select|find|automation)[A-Z]/
+// A public method or arrow property with its first parameter's name and type. `private`,
+// `protected` and `#` members don't match: they aren't on the Store's surface.
+const MEMBER =
+ /^ {2}(?:(?:public|override|static|async|readonly)\s+)*(\w+)\s*(?:<[^>\n]*>)?(?:\s*=\s*(?:async\s*)?(?:<[^>\n]*>)?)?\(\s*(\w+)\??\s*:\s*([^,)=]+)/gm
+
+/** `Class.param` first parameters whose id has no host twin, with the reason. */
+export const NOT_HOST_TWINNED_PARAMS = new Map([
+ ['AutomationPersistence.automationId', 'automation ids are minted UUIDs'],
+ ['OrcadSourceExportPersistence.targetId', 'the id names the host itself'],
+ ['RemoteLayoutSnapshotPersistence.environmentId', 'the id names the host itself'],
+ ['SessionHostPartitionOperations.hostId', 'the id names the host itself'],
+ ['SshLeaseRecoveryOperations.targetId', 'the id names the host itself'],
+ ['SshProfileOperations.oldTargetId', 'the id names the host itself']
+])
+
+// The store keeps one row per id: minted UUIDs, de-duplicated on load, and a conflicting
+// migration import is refused (orcad-catalog-row-identity.ts `selectNewRows`).
+const UNIQUE_GROUP_ID = 'group ids are minted UUIDs, one row per id in the store'
+const UNIQUE_FOLDER_ID = 'folder workspace ids are minted UUIDs, one row per id in the store'
+
+/** Matches exempt by method, with the reason. Keep each reason true. */
+export const NOT_HOST_TWINNED = new Map([
+ ['AutomationPersistence.advanceAutomationNextRun', 'automation ids are minted UUIDs'],
+ ['AutomationPersistence.deleteAutomation', 'automation ids are minted UUIDs'],
+ ['AutomationPersistence.updateAutomation', 'automation ids are minted UUIDs'],
+ [
+ 'MetadataLineageOperations.removeUnhostedWorktreeMeta',
+ 'acts only on a row no host stamped; it refuses a stamped one'
+ ],
+ [
+ 'OrcadMigrationBlobInstallPersistence.installDeferredOrcadMigrationShellHistory',
+ 'a retry trigger: the id only gates a store-wide reinstall; nothing is written under it'
+ ],
+ ['ProfilePreferences.recordFeatureInteraction', 'a feature id, not a row'],
+ ['ProjectCollectionOperations.deleteProjectGroup', UNIQUE_GROUP_ID],
+ ['ProjectCollectionOperations.removeFolderWorkspace', UNIQUE_FOLDER_ID],
+ ['ProjectCollectionOperations.updateFolderWorkspace', UNIQUE_FOLDER_ID],
+ ['ProjectCollectionOperations.updateProject', 'one Project row spans every host'],
+ ['ProjectCollectionOperations.updateProjectGroup', UNIQUE_GROUP_ID],
+ [
+ 'RepoLifecycleOperations.reorderRepos',
+ 'a whole-catalog permutation: a duplicated id fails its check, so it refuses instead of guessing'
+ ],
+ ['RepoLifecycleOperations.reorderReposForHost', 'its second parameter scopes it to one host'],
+ [
+ 'RetiredWorktreeNamePersistence.addRetiredWorktreeName',
+ 'twins share one id-keyed bucket by design; re-keying it is a non-goal (D7)'
+ ],
+ [
+ 'RetiredWorktreeNamePersistence.mergeRetiredWorktreeNames',
+ 'twins share one id-keyed bucket by design; re-keying it is a non-goal (D7)'
+ ],
+ [
+ 'SparsePresetPersistence.removeSparsePreset',
+ 'twins share one id-keyed bucket by design; re-keying it is a non-goal (D7)'
+ ],
+ ['SshProfileOperations.removeSshTarget', 'the id names the host itself'],
+ ['SshProfileOperations.updateSshTarget', 'the id names the host itself']
+])
+
+/** `[className, file]` for every operations class the Store mixes in, in composition order. */
+export function listStoreOperationClasses(root) {
+ const text = fs.readFileSync(path.join(root, COMPOSITION), 'utf8')
+ const imports = new Map(
+ [...text.matchAll(/^import \{ (\w+) \} from '([^']+)'/gm)].map((m) => [m[1], m[2]])
+ )
+ const list = /STORE_DOMAIN_OPERATION_CLASSES = \[([\s\S]*?)\]/.exec(text)
+ if (!list) {
+ throw new Error(`${COMPOSITION}: STORE_DOMAIN_OPERATION_CLASSES not found`)
+ }
+ return list[1]
+ .split(',')
+ .map((name) => name.trim())
+ .filter(Boolean)
+ .map((name) => {
+ const specifier = imports.get(name)
+ if (!specifier) {
+ throw new Error(`${COMPOSITION}: no import for ${name}`)
+ }
+ return [name, path.posix.join(path.posix.dirname(COMPOSITION), `${specifier}.ts`)]
+ })
+}
+
+/** Public mutating methods of `className` in `source` whose first parameter is a bare id string. */
+export function findIdOnlyWrites(className, source) {
+ const start = source.indexOf(`export class ${className}`)
+ if (start === -1) {
+ return []
+ }
+ const rest = source.slice(start)
+ const end = rest.search(/^\}/m)
+ const body = end < 0 ? rest : rest.slice(0, end)
+ const found = []
+ for (const [, method, param, type] of body.matchAll(MEMBER)) {
+ if (
+ method !== 'constructor' &&
+ !READ_METHOD.test(method) &&
+ ID_PARAM.test(param) &&
+ !NOT_HOST_TWINNED_PARAMS.has(`${className}.${param}`) &&
+ !HOST_TARGET_TYPE.test(type.trim())
+ ) {
+ found.push(`${className}.${method}`)
+ }
+ }
+ return found
+}
+
+// Lookups that pick a repo row by id alone. A write fed by one edits whichever twin comes first, even
+// when the method itself takes the row (review-5 #10: an SSH setup edit renamed its local sibling).
+const LOOKUP_ROOT = 'src/main/persistence/'
+const REPO_ROW_LOOKUP = /\brepos\s*\??\.\s*(?:find|findIndex|findLast|findLastIndex)\s*\(/g
+// A predicate that names a host (or compares whole rows' host ids) is host-scoped.
+const HOST_SCOPED_PREDICATE = /host|Host|connectionId/
+const ID_COMPARISON = /\bid\s*===|===\s*(?:[\w.]*\.)?id\b/
+const DECLARATION =
+ /^[ \t]*(?:export\s+)?(?:(?:async|public|private|protected|static|override|readonly)\s+)*(?:function\s+(\w+)|(?:const|let)\s+(\w+)\s*=\s*(?:async\s*)?\([^)]*\)\s*(?::[^=\n]*)?=>|(\w+)\s*(?:<[^>\n]*>)?\s*\([^)]*\)\s*(?::[^{;\n]*)?\{)/gm
+const NOT_A_DECLARATION = new Set(['if', 'for', 'while', 'switch', 'return', 'catch', 'function'])
+
+/** The text from an opening paren to its match, so a multi-line predicate is read whole. */
+function balancedArgs(source, openIndex) {
+ let depth = 0
+ for (let i = openIndex; i < source.length; i++) {
+ if (source[i] === '(') {
+ depth++
+ } else if (source[i] === ')' && --depth === 0) {
+ return source.slice(openIndex, i + 1)
+ }
+ }
+ return source.slice(openIndex)
+}
+
+function enclosingDeclaration(source, index) {
+ let name = ''
+ for (const match of source.slice(0, index).matchAll(DECLARATION)) {
+ const candidate = match[1] ?? match[2] ?? match[3]
+ if (candidate && !NOT_A_DECLARATION.has(candidate)) {
+ name = candidate
+ }
+ }
+ return name
+}
+
+/** `file#function` for every repo-row lookup in `source` whose predicate matches by id alone. */
+export function findIdOnlyLookups(rel, source) {
+ const found = []
+ for (const match of source.matchAll(REPO_ROW_LOOKUP)) {
+ const predicate = balancedArgs(source, match.index + match[0].length - 1)
+ if (ID_COMPARISON.test(predicate) && !HOST_SCOPED_PREDICATE.test(predicate)) {
+ found.push(`lookup:${rel}#${enclosingDeclaration(source, match.index)}`)
+ }
+ }
+ return found
+}
+
+function collectIdOnlyLookups(root) {
+ const sources = readTrackedSources(
+ root,
+ [LOOKUP_ROOT],
+ (rel) => rel.endsWith('.ts') && !/\.(?:test|spec)\.ts$/.test(rel)
+ )
+ return [...sources].flatMap(([rel, source]) => findIdOnlyLookups(rel, source))
+}
+
+export function collectIdOnlyStoreWrites(root = process.cwd()) {
+ const entries = []
+ for (const [className, rel] of listStoreOperationClasses(root)) {
+ const source = fs.readFileSync(path.join(root, rel), 'utf8')
+ entries.push(...findIdOnlyWrites(className, source).filter((e) => !NOT_HOST_TWINNED.has(e)))
+ }
+ entries.push(...collectIdOnlyLookups(root))
+ return [...new Set(entries)].sort()
+}
+
+export function main(root = process.cwd(), arg = undefined) {
+ return runEntryRatchet(
+ {
+ name: 'id-only-store-write',
+ baselinePath: BASELINE_PATH,
+ header: [
+ '# Store writes whose first parameter is a bare repo or worktree id (R4-2), as Class.method,',
+ '# and persistence repo-row lookups by id alone (review-5 #10), as lookup:file#function.',
+ '# This is a RATCHET: entries may only be REMOVED. Take RepoHostTarget or HostWorktreeTarget,',
+ '# or match the row by id AND host (getRepoExecutionHostId).',
+ '# Prune after a fix: pnpm check:id-only-store-write-ratchet --prune'
+ ],
+ collect: collectIdOnlyStoreWrites,
+ fix: [
+ 'Take the row, not its id: RepoHostTarget (src/shared/repo-host-identity.ts) for repo rows,',
+ 'HostWorktreeTarget (src/shared/worktree/host-qualified-identity.ts) for worktree rows. A caller',
+ 'with only an id resolves the row where its host is known (RPC selector, IPC hostId).'
+ ]
+ },
+ root,
+ arg
+ )
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ process.exit(main(process.cwd(), process.argv[2]))
+}
diff --git a/config/scripts/check-id-only-store-write-ratchet.test.mjs b/config/scripts/check-id-only-store-write-ratchet.test.mjs
new file mode 100644
index 000000000000..738269d2a988
--- /dev/null
+++ b/config/scripts/check-id-only-store-write-ratchet.test.mjs
@@ -0,0 +1,121 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+ collectIdOnlyStoreWrites,
+ findIdOnlyLookups,
+ findIdOnlyWrites,
+ listStoreOperationClasses,
+ NOT_HOST_TWINNED
+} from './check-id-only-store-write-ratchet.mjs'
+
+const SOURCE = `export class RowOps {
+ constructor(runtime: string) {}
+
+ moveProjectToGroup(repoId: string, groupId: string | null): void {}
+
+ updateFolderWorkspace(
+ id: string,
+ updates: object
+ ): void {}
+
+ removeWorktreeMeta(worktreeId?: string): void {}
+
+ getWorktreeMeta(worktreeId: string): void {}
+
+ updateRepo(target: RepoHostTarget): void {}
+
+ reorderRepos(orderedIds: string[]): void {}
+
+ setRemoteLayoutSnapshot(environmentId: string): void {}
+
+ private writeRow(repoId: string): void {}
+
+ public renameRepo(repoId: Repo['id']): void {}
+
+ override setLineage(worktreeKey: WorktreeKey, value: T): void {}
+
+ removeWorkspaceLineage(childWorkspaceKey: WorkspaceKey): void {}
+
+ dropRow = (id: BrandedRepoId): void => {}
+
+ moveRow(target: RepoHostTarget): void {}
+
+ dropWorktree(worktreeId: HostWorktreeTarget): void {}
+
+ removeSessionHost(hostId: ExecutionHostId): void {}
+}
+
+export class Other {
+ setWorktreeLineage(worktreeId: string): void {}
+}
+`
+
+describe('findIdOnlyWrites', () => {
+ it('lists public writes whose first parameter is an id by name, whatever its type', () => {
+ expect(findIdOnlyWrites('RowOps', SOURCE)).toEqual([
+ 'RowOps.moveProjectToGroup',
+ 'RowOps.updateFolderWorkspace',
+ 'RowOps.removeWorktreeMeta',
+ 'RowOps.reorderRepos',
+ 'RowOps.setRemoteLayoutSnapshot',
+ 'RowOps.renameRepo',
+ 'RowOps.setLineage',
+ 'RowOps.removeWorkspaceLineage',
+ 'RowOps.dropRow',
+ // Param exemptions are per class, so these host-named params count on RowOps.
+ 'RowOps.removeSessionHost'
+ ])
+ })
+
+ it('stays inside the named class', () => {
+ expect(findIdOnlyWrites('Other', SOURCE)).toEqual(['Other.setWorktreeLineage'])
+ expect(findIdOnlyWrites('Missing', SOURCE)).toEqual([])
+ })
+})
+
+describe('collectIdOnlyStoreWrites', () => {
+ it('scans every Store operations class and skips the documented exemptions', () => {
+ expect(listStoreOperationClasses(process.cwd()).length).toBeGreaterThan(10)
+ const entries = collectIdOnlyStoreWrites(process.cwd())
+ expect(entries).not.toContain('RepoLifecycleOperations.updateRepo')
+ for (const exempt of NOT_HOST_TWINNED.keys()) {
+ expect(entries).not.toContain(exempt)
+ }
+ })
+})
+
+const LOOKUP_SOURCE = `export class SetupOps {
+ updateProjectHostSetup(args: Args): Result | null {
+ const repo = setup.repoId
+ ? this.state.repos.find((entry) => entry.id === setup.repoId)
+ : undefined
+ return repo
+ }
+
+ private findSetupRepo(setup: ProjectHostSetup): Repo | undefined {
+ return this.state.repos.find(
+ (entry) => entry.id === setup.repoId && getRepoExecutionHostId(entry) === setup.hostId
+ )
+ }
+}
+
+export const pickRepo = (state: State, repoId: string): number =>
+ state.repos.findIndex(
+ (repo) =>
+ repo.id === repoId
+ )
+
+function pickOptional(state: State, repoId: string): Repo | undefined {
+ return state.repos?.find(({ id }) => id === repoId)
+}
+`
+
+describe('findIdOnlyLookups', () => {
+ it('flags a repo-row lookup by id alone (review-5 #10), by its enclosing function', () => {
+ expect(findIdOnlyLookups('ops.ts', LOOKUP_SOURCE)).toEqual([
+ 'lookup:ops.ts#updateProjectHostSetup',
+ 'lookup:ops.ts#pickRepo',
+ 'lookup:ops.ts#pickOptional'
+ ])
+ })
+})
diff --git a/config/scripts/check-node-runtime-pin.mjs b/config/scripts/check-node-runtime-pin.mjs
index 6d5205aa525c..d606fb3ff59c 100644
--- a/config/scripts/check-node-runtime-pin.mjs
+++ b/config/scripts/check-node-runtime-pin.mjs
@@ -109,6 +109,22 @@ export function findNodeRuntimePinProblems({
`NODE_RUNTIME_PIN.version ${pin.version} is not package.json engines.node major ${engineMajor}`
)
}
+ const runtimeMajor = majorOf(pin.version)
+ const declaredNodeTypes =
+ packageJson.devDependencies?.['@types/node'] ?? packageJson.dependencies?.['@types/node']
+ if (majorOf(declaredNodeTypes) !== runtimeMajor) {
+ problems.push(
+ `package.json @types/node is ${declaredNodeTypes}, but NODE_RUNTIME_PIN.version major is ${runtimeMajor}`
+ )
+ }
+ const lockedNodeTypes = lockedVersion(
+ rootImporter.devDependencies?.['@types/node'] ?? rootImporter.dependencies?.['@types/node']
+ )
+ if (majorOf(lockedNodeTypes) !== runtimeMajor) {
+ problems.push(
+ `pnpm-lock.yaml resolves @types/node ${lockedNodeTypes}, but NODE_RUNTIME_PIN.version major is ${runtimeMajor}`
+ )
+ }
if (!Number.isInteger(pin.napi) || pin.napi < 1) {
problems.push(`NODE_RUNTIME_PIN.napi must be a positive integer, got ${pin.napi}`)
}
diff --git a/config/scripts/check-unproven-pty-retirement-ratchet.mjs b/config/scripts/check-unproven-pty-retirement-ratchet.mjs
new file mode 100644
index 000000000000..c4115a47de47
--- /dev/null
+++ b/config/scripts/check-unproven-pty-retirement-ratchet.mjs
@@ -0,0 +1,66 @@
+import process from 'node:process'
+import { pathToFileURL } from 'node:url'
+import { readTrackedSources, runCountRatchet } from './ratchet-baseline.mjs'
+
+// Ratchet on PTY retirements that rest on no exit proof (src/main/pty-exit-proof.ts): unproven
+// ones, and ownership drops outside `finishPtyShutdown`, which name no evidence at all. Each drops
+// the handle a later stop could use while the process may still run. Counts may only go down.
+
+const BASELINE_PATH = 'config/unproven-pty-retirement-baseline.txt'
+const SCAN_ROOTS = ['src']
+const UNPROVEN_RETIREMENT = /\bPtyRetirementWithoutProof\.because\(|\bptyOwnership\.delete\(/g
+const COMMENT = /\/\*[\s\S]*?\*\/|(? 0) {
+ counts.set(rel, count)
+ }
+ }
+ return counts
+}
+
+export function main(root = process.cwd(), arg = undefined) {
+ return runCountRatchet(
+ {
+ name: 'unproven-pty-retirement',
+ baselinePath: BASELINE_PATH,
+ header: [
+ '# PTY retirements on no exit proof per file: PtyRetirementWithoutProof.because, and raw',
+ '# ptyOwnership.delete outside finishPtyShutdown. This is a RATCHET: counts may only go DOWN.',
+ '# Retire through finishPtyShutdown on a PtyExitProof, or record the stop durably first.',
+ '# Prune after a fix: pnpm check:unproven-pty-retirement-ratchet --prune'
+ ],
+ collect: collectUnprovenPtyRetirementCounts,
+ unit: 'unproven PTY retirement(s)',
+ fix: [
+ 'Retire a PTY through finishPtyShutdown on a PtyExitProof (src/main/pty-exit-proof.ts), or',
+ 'keep its ownership until one arrives. A close the host cannot answer must read unverifiable.'
+ ]
+ },
+ root,
+ arg
+ )
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ process.exit(main(process.cwd(), process.argv[2]))
+}
diff --git a/config/scripts/check-unproven-pty-retirement-ratchet.test.mjs b/config/scripts/check-unproven-pty-retirement-ratchet.test.mjs
new file mode 100644
index 000000000000..4ee420f134bf
--- /dev/null
+++ b/config/scripts/check-unproven-pty-retirement-ratchet.test.mjs
@@ -0,0 +1,36 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+ countUnprovenPtyRetirements,
+ isScannedPath
+} from './check-unproven-pty-retirement-ratchet.mjs'
+
+describe('countUnprovenPtyRetirements', () => {
+ it('counts each unproven mint and each raw ownership drop', () => {
+ expect(
+ countUnprovenPtyRetirements(
+ 'const a = PtyRetirementWithoutProof.because(R)\nf(PtyRetirementWithoutProof.because("x"))'
+ )
+ ).toBe(2)
+ expect(countUnprovenPtyRetirements('ptyOwnership.delete(id)\nptyOwnership.get(id)')).toBe(1)
+ })
+
+ it('ignores comments, proofs and the type name alone', () => {
+ expect(
+ countUnprovenPtyRetirements(
+ '// PtyRetirementWithoutProof.because(r)\n/* PtyRetirementWithoutProof.because(r) */\n' +
+ "PtyExitProof.from('absent')\nx instanceof PtyRetirementWithoutProof"
+ )
+ ).toBe(0)
+ })
+})
+
+describe('isScannedPath', () => {
+ it('scans production TypeScript only', () => {
+ expect(isScannedPath('src/main/ipc/pty/runtime/kill.ts')).toBe(true)
+ expect(isScannedPath('src/main/ipc/pty/runtime/kill.test.ts')).toBe(false)
+ expect(isScannedPath('src/main/runtime/a.spec.ts')).toBe(false)
+ expect(isScannedPath('src/main/ipc/pty/daemon-late-exit-test-fixture.ts')).toBe(false)
+ expect(isScannedPath('src/main/ipc/pty/provider/liveness.ts')).toBe(false)
+ })
+})
diff --git a/config/scripts/check-worktree-id-composer-ratchet.mjs b/config/scripts/check-worktree-id-composer-ratchet.mjs
index 14af4a11a37d..e9097abd4a80 100644
--- a/config/scripts/check-worktree-id-composer-ratchet.mjs
+++ b/config/scripts/check-worktree-id-composer-ratchet.mjs
@@ -1,14 +1,6 @@
-import fs from 'node:fs'
-import path from 'node:path'
import process from 'node:process'
import { pathToFileURL } from 'node:url'
-import {
- diffCounts,
- parseCountBaseline,
- pruneCountBaseline,
- readTrackedSources,
- totalCount
-} from './ratchet-baseline.mjs'
+import { readTrackedSources, runCountRatchet } from './ratchet-baseline.mjs'
// Ratchet on worktree ids built by hand (`${repo.id}::${path}`) instead of through the composers in
// src/shared/worktree/id.ts. About 22 hand-built ids each pick whichever path spelling is at hand,
@@ -57,38 +49,20 @@ export function collectHandBuiltWorktreeIdCounts(root = process.cwd()) {
}
export function main(root = process.cwd(), arg = undefined) {
- const current = collectHandBuiltWorktreeIdCounts(root)
- if (arg === '--prune') {
- const total = pruneCountBaseline(root, BASELINE_PATH, HEADER, current)
- console.log(`worktree-id-composer baseline pruned to ${total}.`)
- return 0
- }
- const baselineFile = path.join(root, BASELINE_PATH)
- const baseline = parseCountBaseline(
- fs.existsSync(baselineFile) ? fs.readFileSync(baselineFile, 'utf8') : ''
+ return runCountRatchet(
+ {
+ name: 'worktree-id-composer',
+ baselinePath: BASELINE_PATH,
+ header: HEADER,
+ collect: collectHandBuiltWorktreeIdCounts,
+ unit: 'hand-built worktree id(s)',
+ fix: [
+ 'Compose worktree ids with composeWorktreeId / getRepoMainWorktreeId (src/shared/worktree/id.ts).'
+ ]
+ },
+ root,
+ arg
)
- const { grown, shrunk } = diffCounts(current, baseline)
- for (const { file, now, allowed } of grown) {
- console.error(`${file}: ${now} hand-built worktree id(s), baseline allows ${allowed}.`)
- }
- if (grown.length > 0) {
- console.error(
- 'Compose worktree ids with composeWorktreeId / getRepoMainWorktreeId (src/shared/worktree/id.ts).'
- )
- }
- for (const { file, now, allowed } of shrunk) {
- console.error(`${file}: now ${now}, baseline ${allowed}.`)
- }
- if (shrunk.length > 0) {
- console.error(
- 'The baseline may only shrink. Fix it: pnpm check:worktree-id-composer-ratchet --prune'
- )
- }
- if (grown.length + shrunk.length > 0) {
- return 1
- }
- console.log(`worktree-id-composer ratchet OK — ${totalCount(current)} grandfathered id(s).`)
- return 0
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
diff --git a/config/scripts/ci-background-steps.test.mjs b/config/scripts/ci-background-steps.test.mjs
new file mode 100644
index 000000000000..b68be808d0d5
--- /dev/null
+++ b/config/scripts/ci-background-steps.test.mjs
@@ -0,0 +1,95 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+import { parse } from 'yaml'
+import { expect, it } from 'vitest'
+
+// Why: the runner rewrites $GITHUB_EVENT_PATH in place (truncate, then write) at every step start,
+// and background steps start on their own task. A JavaScript action (@actions/github parses the
+// payload at import) or composite (its inner steps start later) in the background therefore races
+// the foreground's step starts and reads an empty file: "Unexpected end of JSON input".
+const workflowFiles = readdirSync('.github/workflows')
+ .filter((name) => /\.ya?ml$/.test(name))
+ .map((name) => join('.github/workflows', name))
+const actionFiles = readdirSync('.github/actions')
+ .map((name) => join('.github/actions', name, 'action.yml'))
+ .filter((file) => existsSync(file))
+
+// Single-step JavaScript actions whose bundle never reads the payload (checked by searching dist/index.js).
+const payloadFreeActions = ['ruby/setup-ruby@v1']
+// The runner runs 10 at once; a queued step starts (and rewrites the payload) mid-foreground-step.
+const backgroundSlots = 10
+
+function stepLists(file) {
+ const document = parse(readFileSync(file, 'utf8'))
+ return document.jobs
+ ? Object.entries(document.jobs).map(([job, value]) => [job, value.steps ?? []])
+ : [['composite', document.runs?.steps ?? []]]
+}
+
+function backgroundSteps() {
+ const found = []
+ for (const file of [...workflowFiles, ...actionFiles]) {
+ for (const [job, steps] of stepLists(file)) {
+ for (const step of steps) {
+ if (step.background) {
+ found.push({ where: `${file} ${job} ${step.id ?? step.name}`, step })
+ }
+ }
+ }
+ }
+ return found
+}
+
+function readsEventPayload(run) {
+ const scripts = [...run.matchAll(/(?:config\/scripts|\.github\/actions)\/[\w./-]+/g)].map(
+ (match) => match[0]
+ )
+ return (
+ run.includes('GITHUB_EVENT_PATH') ||
+ scripts.some(
+ (file) =>
+ existsSync(file) &&
+ statSync(file).isFile() &&
+ readFileSync(file, 'utf8').includes('GITHUB_EVENT_PATH')
+ )
+ )
+}
+
+it('finds the background steps it guards', () => {
+ expect(backgroundSteps().length).toBeGreaterThan(10)
+})
+
+it('backgrounds only single run steps, never actions that start later steps or parse the payload', () => {
+ const actions = backgroundSteps()
+ .filter(({ step }) => typeof step.run !== 'string' && !payloadFreeActions.includes(step.uses))
+ .map(({ where }) => where)
+ expect(actions).toEqual([])
+})
+
+it('keeps the event payload out of background run steps', () => {
+ const readers = backgroundSteps()
+ .filter(({ step }) => typeof step.run === 'string' && readsEventPayload(step.run))
+ .map(({ where }) => where)
+ expect(readers).toEqual([])
+})
+
+it('never queues a background step behind a full slot pool', () => {
+ const crowded = []
+ for (const file of workflowFiles) {
+ for (const [job, steps] of stepLists(file)) {
+ const open = new Set()
+ for (const [index, step] of steps.entries()) {
+ if (step.background) {
+ open.add(step.id ?? index)
+ }
+ for (const id of [step.wait ?? []].flat()) {
+ open.delete(id)
+ }
+ if (open.size > backgroundSlots) {
+ crowded.push(`${file} ${job}`)
+ }
+ }
+ }
+ }
+ expect(crowded).toEqual([])
+})
diff --git a/config/scripts/ci-dependency-download-cache.test.mjs b/config/scripts/ci-dependency-download-cache.test.mjs
index e0c875093d25..fbed68525810 100644
--- a/config/scripts/ci-dependency-download-cache.test.mjs
+++ b/config/scripts/ci-dependency-download-cache.test.mjs
@@ -283,7 +283,9 @@ describe('CI dependency download caches', () => {
).jobs
const warmer = platformJobs['warm-windows'].steps.find((step) => step.uses === installer)
expect(action.inputs['cache-pnpm-store'].default).toBe('true')
- expect(persistence.with['cache-pnpm-store']).toBe("${{ runner.os != 'Windows' }}")
+ expect(persistence.with['cache-pnpm-store']).toBe(
+ "${{ !inputs.build_template && runner.os != 'Windows' }}"
+ )
expect(ssh.with['cache-pnpm-store']).toBe('false')
expect(warmer.with['cache-pnpm-store']).toBe('false')
expect(warmer.with['native-runtime']).toBe('node')
diff --git a/config/scripts/ci-e2e-job-selection.mjs b/config/scripts/ci-e2e-job-selection.mjs
index 00be43ec1ad5..a26aa934925f 100644
--- a/config/scripts/ci-e2e-job-selection.mjs
+++ b/config/scripts/ci-e2e-job-selection.mjs
@@ -11,7 +11,7 @@ export const WINDOWS_MISSING_APPDATA_E2E_SPEC = 'tests/e2e/windows-missing-appda
export const WINDOWS_COPY_FILE_E2E_SPEC = 'tests/e2e/file-explorer-copy-file-windows.spec.ts'
// Needs out/orcad, which only the mode-switch job builds; it runs there beside that spec.
export const LAYOUT_ORACLE_HEADLESS_E2E_SPEC = 'tests/e2e/workspace-layout-oracle-headless.spec.ts'
-// One Docker job runs every managed-SSH spec; it builds the orcad template they need.
+// Managed-SSH specs need the orcad template built by their Docker lane.
export const MANAGED_SSH_E2E_SPEC_PREFIX = 'tests/e2e/ssh-orcad-'
export const DEDICATED_E2E_SPECS = [
NODE_NETWORK_E2E_SPEC,
@@ -46,6 +46,38 @@ function parseSpecs(input) {
return specs
}
+export function selectManagedSshE2eSpecs(input) {
+ if (input === '') {
+ return [MANAGED_SSH_E2E_SPEC_PREFIX]
+ }
+ const specs = [...new Set(parseSpecs(input))].filter((spec) =>
+ spec.startsWith(MANAGED_SSH_E2E_SPEC_PREFIX)
+ )
+ if (
+ specs.length === 0 ||
+ specs.some(
+ (spec) =>
+ spec !== MANAGED_SSH_E2E_SPEC_PREFIX &&
+ !/^tests\/e2e\/ssh-orcad-[\w-]+\.spec\.ts$/.test(spec)
+ )
+ ) {
+ throw new Error('Expected at least one managed SSH spec')
+ }
+ return specs.includes(MANAGED_SSH_E2E_SPEC_PREFIX) ? [MANAGED_SSH_E2E_SPEC_PREFIX] : specs
+}
+
+export function managedSshShards(input) {
+ try {
+ const specs = selectManagedSshE2eSpecs(input)
+ if (!specs.includes(MANAGED_SSH_E2E_SPEC_PREFIX) && specs.length <= 2) {
+ return ['1/1']
+ }
+ } catch {
+ // Invalid evidence keeps the full allocation; the test consumer rejects it.
+ }
+ return ['1/2', '2/2']
+}
+
export function classifyE2eJobs(input) {
const conservative = { e2e_run_changed: true, e2e_needs_build: true }
let specs
@@ -74,6 +106,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
for (const [name, value] of Object.entries(classifyE2eJobs(input))) {
process.stdout.write(`${name}=${value}\n`)
}
+ process.stdout.write(`e2e_ssh_shards=${JSON.stringify(managedSshShards(input))}\n`)
} else {
for (const spec of selectGeneralE2eSpecs(parseSpecs(input))) {
if (/[\r\n]/.test(spec)) {
diff --git a/config/scripts/ci-managed-ssh-execution.test.mjs b/config/scripts/ci-managed-ssh-execution.test.mjs
new file mode 100644
index 000000000000..090facefb92b
--- /dev/null
+++ b/config/scripts/ci-managed-ssh-execution.test.mjs
@@ -0,0 +1,177 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { delimiter, join } from 'node:path'
+import { parse } from 'yaml'
+import { runProcess } from '@orca/process-host'
+import { managedSshShards, MANAGED_SSH_E2E_SPEC_PREFIX } from './ci-e2e-job-selection.mjs'
+import { compileAggregateExpression } from './pr-aggregate-gate-context.mjs'
+import { selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
+
+const workflow = parse(readFileSync('.github/workflows/e2e.yml', 'utf8'))
+const prWorkflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8'))
+const lane = workflow.jobs['orcad-ssh-docker']
+const command = lane.steps.find(
+ (step) => step.name === 'Run managed SSH specs on a Docker host'
+).run
+const selector = readFileSync('config/scripts/ci-e2e-job-selection.mjs', 'utf8')
+const small = ['tests/e2e/ssh-orcad-open-in-owner.spec.ts', 'tests/e2e/ssh-orcad-idle-exit.spec.ts']
+const directories = []
+afterEach(() => {
+ for (const directory of directories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
+it('uses one PR shard for at most two selected files and preserves the full default', () => {
+ // This fork's PR workflow has no e2e caller, so the lane takes the selector's output directly.
+ expect(prWorkflow.jobs.e2e).toBeUndefined()
+ const matrix = compileAggregateExpression(lane.strategy.matrix.shard, ['inputs'])
+ for (const [specs, expected] of [
+ [[small[0]], ['1/1']],
+ [small, ['1/1']],
+ [
+ [...small, 'tests/e2e/ssh-orcad-workspace-tools.spec.ts'],
+ ['1/2', '2/2']
+ ],
+ [[MANAGED_SSH_E2E_SPEC_PREFIX], ['1/2', '2/2']]
+ ]) {
+ const shards = JSON.stringify(managedSshShards(JSON.stringify(specs)))
+ expect(
+ matrix({ inputs: { managed_ssh_shards: shards } })
+ .values()
+ .map((item) => item.value)
+ ).toEqual(expected)
+ }
+ expect(
+ matrix({ inputs: {} })
+ .values()
+ .map((item) => item.value)
+ ).toEqual(['1/2', '2/2'])
+})
+
+it('keeps shared SSH fixtures on the full suite but excludes the experimental sidebar spec', () => {
+ for (const file of [
+ 'tests/e2e/helpers/orcad-managed-workspace.ts',
+ 'tests/e2e/helpers/paired-host-terminal.ts',
+ '.github/workflows/e2e.yml'
+ ]) {
+ expect(selectPrE2eSpecs([file])).toContain(MANAGED_SSH_E2E_SPEC_PREFIX)
+ }
+ for (const file of [
+ 'tests/e2e/ssh-orcad-phone-feed-list.spec.ts',
+ 'tests/e2e/helpers/phone-feed-list.ts',
+ 'tests/e2e/helpers/sidebar-inputs-build.ts',
+ 'src/shared/sidebar-list/sidebar-feed-list.ts'
+ ]) {
+ expect(
+ selectPrE2eSpecs([file]).some((spec) => spec.startsWith(MANAGED_SSH_E2E_SPEC_PREFIX)),
+ file
+ ).toBe(false)
+ }
+})
+
+async function execute(input, version = 'current') {
+ const directory = mkdtempSync(join(tmpdir(), 'orca-ssh-selection-'))
+ directories.push(directory)
+ const bin = join(directory, 'bin')
+ mkdirSync(bin)
+ const moduleDir = join(directory, 'config', 'scripts')
+ mkdirSync(moduleDir, { recursive: true })
+ if (version !== 'missing') {
+ writeFileSync(
+ join(moduleDir, 'ci-e2e-job-selection.mjs'),
+ version === 'current' ? selector : 'export const old = true\n'
+ )
+ }
+ writeFileSync(
+ join(bin, 'xvfb-run'),
+ `#!/usr/bin/env bash
+while [ "$#" -gt 0 ] && [ "$1" != pnpm ]; do shift; done
+exec "$@"
+`,
+ { mode: 0o755 }
+ )
+ const capture = join(directory, 'argv.json')
+ writeFileSync(
+ join(bin, 'pnpm'),
+ `#!/usr/bin/env node
+require('node:fs').writeFileSync(process.env.ARGV_CAPTURE, JSON.stringify(process.argv.slice(2)))
+`,
+ { mode: 0o755 }
+ )
+ const result = await runProcess({
+ program: 'bash',
+ args: ['-e', '-o', 'pipefail', '-c', command],
+ cwd: directory,
+ env: {
+ ...process.env,
+ PATH: bin + delimiter + process.env.PATH,
+ TEST_FILES_JSON: input,
+ RUNNER_TEMP: directory,
+ ARGV_CAPTURE: capture,
+ SHARD: '1/1',
+ LAYOUT_OWNER: '',
+ SHARD_INDEX: '0'
+ },
+ timeoutMs: 10000
+ })
+ return {
+ result,
+ args: existsSync(capture) ? JSON.parse(readFileSync(capture, 'utf8')) : undefined
+ }
+}
+
+// The production lane runs Bash on Linux; the stubs never start Docker, Electron, or Xvfb.
+describe.skipIf(process.platform === 'win32')('managed SSH workflow command', () => {
+ it('passes only the selected SSH files to Playwright, with literal file matching', async () => {
+ const { result, args } = await execute(
+ JSON.stringify([...small, 'tests/e2e/other.spec.ts', small[0]])
+ )
+ expect(result.code, result.stderr).toBe(0)
+ const filters = args.slice(
+ args.indexOf('tests/playwright.config.ts') + 1,
+ args.indexOf('--project=electron-headless')
+ )
+ expect(filters).toHaveLength(2)
+ expect(small.map((file) => filters.some((filter) => new RegExp(filter).test(file)))).toEqual([
+ true,
+ true
+ ])
+ expect(
+ filters.some((filter) => new RegExp(filter).test('tests/e2e/ssh-orcad-open-in-ownerXspecYts'))
+ ).toBe(false)
+ expect(
+ filters.some((filter) => new RegExp(filter).test('tests/e2e/ssh-orcad-unselected.spec.ts'))
+ ).toBe(false)
+ expect(args).toContain('--shard=1/1')
+ })
+
+ it.each(['', JSON.stringify([MANAGED_SSH_E2E_SPEC_PREFIX, small[0]])])(
+ 'keeps full daily and explicit suite execution for %s',
+ async (input) => {
+ const { result, args } = await execute(input)
+ expect(result.code, result.stderr).toBe(0)
+ expect(args).toContain(MANAGED_SSH_E2E_SPEC_PREFIX)
+ }
+ )
+
+ it.each(['legacy', 'missing'])('keeps compatibility with %s checkout refs', async (version) => {
+ const { result, args } = await execute(JSON.stringify(small), version)
+ expect(result.code, result.stderr).toBe(0)
+ expect(args).toContain(MANAGED_SSH_E2E_SPEC_PREFIX)
+ })
+
+ it.each([
+ 'malformed',
+ '[]',
+ '[null]',
+ JSON.stringify(['tests/e2e/other.spec.ts']),
+ JSON.stringify(['tests/e2e/ssh-orcad-../../other.spec.ts'])
+ ])('refuses invalid selection before starting Playwright: %s', async (input) => {
+ const { result, args } = await execute(input)
+ expect(result.code).not.toBe(0)
+ expect(args).toBeUndefined()
+ expect(managedSshShards(input)).toEqual(['1/2', '2/2'])
+ })
+})
diff --git a/config/scripts/ci-native-cache.test.mjs b/config/scripts/ci-native-cache.test.mjs
index 816f2cd830bf..4cb1110a7bee 100644
--- a/config/scripts/ci-native-cache.test.mjs
+++ b/config/scripts/ci-native-cache.test.mjs
@@ -174,7 +174,6 @@ describe('CI native cache ownership', () => {
const cliWait = steps.find((step) => step.wait === 'e2e-cli')
const electron = steps.find((step) => step.id === 'electron-native-runtime')
const upload = steps.find((step) => step.name === 'Upload E2E build output')
- const uploadWait = steps.find((step) => step.wait === upload.id)
expect(install.with).toMatchObject({
'native-runtime': 'node',
'persist-native-cache': 'false',
@@ -186,11 +185,11 @@ describe('CI native cache ownership', () => {
key: '${{ steps.deps.outputs.native-cache-key }}',
path: '${{ steps.deps.outputs.native-cache-path }}'
})
- const ordered = [install, save, build, cliWait, upload, electron, uploadWait]
+ const ordered = [install, save, build, cliWait, upload, electron]
expect(ordered.map((step) => steps.indexOf(step))).toEqual(
ordered.map((step) => steps.indexOf(step)).sort((left, right) => left - right)
)
- expect(upload.background).toBe(true)
+ expect(upload.background).toBeUndefined()
expect(upload.if).toBeUndefined()
expect(electron.background).toBeUndefined()
expect(electron.uses).toBe('./.github/actions/prepare-native-runtime')
@@ -198,7 +197,6 @@ describe('CI native cache ownership', () => {
'native-runtime': 'electron',
'node-version': '${{ steps.deps.outputs.node-version }}'
})
- expect(uploadWait).toEqual({ wait: upload.id })
for (const name of ['e2e', 'changed-e2e', 'orcad-serve-mode-switch', 'orcad-ssh-docker']) {
expect(e2e.jobs[name].needs, name).toEqual(['build'])
expect(
diff --git a/config/scripts/ci-pnpm-store-benchmark.mjs b/config/scripts/ci-pnpm-store-benchmark.mjs
index 0e613965a754..dbb728972a9d 100644
--- a/config/scripts/ci-pnpm-store-benchmark.mjs
+++ b/config/scripts/ci-pnpm-store-benchmark.mjs
@@ -32,6 +32,8 @@ const manifests = [
'package.json',
'pnpm-lock.yaml',
'pnpm-workspace.yaml',
+ // The lockfile pins its checksum, so a frozen install fails without it.
+ 'config/pnpmfile.cjs',
...workspacePackageManifests(repository)
]
const pnpm = resolvePnpmCliInvocation()
diff --git a/config/scripts/ci-pnpm-store-mode.test.mjs b/config/scripts/ci-pnpm-store-mode.test.mjs
index a98dfaa35254..7cebed7b9896 100644
--- a/config/scripts/ci-pnpm-store-mode.test.mjs
+++ b/config/scripts/ci-pnpm-store-mode.test.mjs
@@ -52,8 +52,8 @@ function resolveMode(request, node, manager) {
describe('automatic pnpm store mode', () => {
it.each([
- ['auto', '24', 'pnpm@12.8.1', '', true],
- ['auto', '25', 'pnpm@12.8.1', 'pnpm', false],
+ ['auto', '24', 'pnpm@12.14.0', '', true],
+ ['auto', '25', 'pnpm@12.14.0', 'pnpm', false],
['auto', '24', 'pnpm@13.0.0', 'pnpm', false],
['true', '25', 'pnpm@13.0.0', '', true]
])(
@@ -129,18 +129,18 @@ describe('automatic pnpm store mode', () => {
})
it.each([
- ['24', 'pnpm@12.8.1+sha512.fixture', 'true'],
- ['24.x', 'pnpm@12.8.1', 'false'],
+ ['24', 'pnpm@12.14.0+sha512.fixture', 'true'],
+ ['24.x', 'pnpm@12.14.0', 'false'],
['24', 'pnpm@12.8.2', 'false'],
- ['24', 'pnpm@12.8.10', 'false'],
+ ['24', 'pnpm@12.14.00', 'false'],
['24', undefined, 'false'],
- [undefined, 'pnpm@12.8.1', 'false'],
+ [undefined, 'pnpm@12.14.0', 'false'],
['24', 12, 'false']
])('checks manifest Node %s and manager %s before choosing lookup', (node, manager, expected) => {
expect(resolveMode('auto', node, manager)).toBe(`lookup-only=${expected}\n`)
})
it('checks uppercase auto requests consistently with GitHub expression comparisons', () => {
- expect(resolveMode('AUTO', '25', 'pnpm@12.8.1')).toBe('lookup-only=false\n')
+ expect(resolveMode('AUTO', '25', 'pnpm@12.14.0')).toBe('lookup-only=false\n')
})
})
diff --git a/config/scripts/ci-pnpm-verification-cache.test.mjs b/config/scripts/ci-pnpm-verification-cache.test.mjs
index 95029d23eb54..875d6f6e9fea 100644
--- a/config/scripts/ci-pnpm-verification-cache.test.mjs
+++ b/config/scripts/ci-pnpm-verification-cache.test.mjs
@@ -22,7 +22,7 @@ const evaluate = (expression, context) => runInNewContext(expression, context)
function resolveRecord({
container = false,
os = 'Linux',
- packageManager = 'pnpm@12.8.1+sha512.test'
+ packageManager = 'pnpm@12.14.0+sha512.test'
} = {}) {
const root = mkdtempSync(join(tmpdir(), 'orca-verification-record-'))
try {
@@ -43,7 +43,7 @@ function resolveRecord({
printf '%s\\n' "$*" >> "$TEST_CALLS"
case "$*" in
'cache path') printf '%s\\n' "$TEST_CACHE_PATH" ;;
- '--version') printf '%s\\n' '12.8.1' ;;
+ '--version') printf '%s\\n' '12.14.0' ;;
*) return 2 ;;
esac
}
@@ -93,7 +93,7 @@ describe('pnpm-owned verification record', () => {
'path=/cache with spaces/pnpm/lockfile-verified.jsonl'
)
expect(container.output).toContain(
- 'key=pnpm-verification-v1-Linux-ARM64-12.8.1-policy-digest'
+ 'key=pnpm-verification-v1-Linux-ARM64-12.14.0-policy-digest'
)
expect(host.calls).toBe('cache path\n--version\n')
expect(container.calls).toBe('')
@@ -103,7 +103,7 @@ describe('pnpm-owned verification record', () => {
it.skipIf(process.platform === 'win32').each([
{ container: true, os: 'Windows' },
{ container: true, os: 'macOS' },
- { container: true, packageManager: 'npm@12.8.1' }
+ { container: true, packageManager: 'npm@12.14.0' }
])('rejects an unsupported container identity: %j', (options) => {
const result = resolveRecord(options)
expect(result.code).not.toBe(0)
diff --git a/config/scripts/ci-shard-timings.json b/config/scripts/ci-shard-timings.json
index 381b6d63a91a..dd0f11ba04b7 100644
--- a/config/scripts/ci-shard-timings.json
+++ b/config/scripts/ci-shard-timings.json
@@ -5692,7 +5692,7 @@
"src/main/managed-server/install/remote-node-runtime-store-lock-host-release.test.ts": 2252,
"src/main/managed-server/install/remote-node-runtime-store-windows.test.ts": 7184,
"src/main/managed-server/install/remote-versioned-install.test.ts": 283,
- "src/main/ssh/removed-ssh-target-tombstone-retention.test.ts": 72,
+ "src/main/ssh/targets/removed-ssh-target-tombstone-retention.test.ts": 72,
"src/main/ssh/runtime-ssh-access-verification.test.ts": 290,
"src/main/ssh/runtime-ssh-access.test.ts": 621,
"src/main/ssh/sftp-namespace-resolution.test.ts": 66,
@@ -5712,30 +5712,30 @@
"src/main/ssh/config/ssh-config-parser.test.ts": 331,
"src/main/ssh/config/ssh-config-resolver.test.ts": 40,
"src/main/ssh/ssh-connect-attempt-cancellation.test.ts": 85,
- "src/main/ssh/ssh-connection-attribution.test.ts": 34,
- "src/main/ssh/ssh-connection-auth-fallback.test.ts": 269,
- "src/main/ssh/ssh-connection-channel-error.test.ts": 142,
- "src/main/ssh/ssh-connection-channel-lifetime.test.ts": 58,
- "src/main/ssh/ssh-connection-channel-open.test.ts": 930,
- "src/main/ssh/ssh-connection-direct-startup.test.ts": 128,
- "src/main/ssh/ssh-connection-file-transfer.test.ts": 142,
- "src/main/ssh/ssh-connection-generation.test.ts": 41,
- "src/main/ssh/ssh-connection-github-probe.test.ts": 174,
- "src/main/ssh/ssh-connection-gssapi-fallback.test.ts": 237,
- "src/main/ssh/ssh-connection-host-key-store-wiring.test.ts": 524,
- "src/main/ssh/ssh-connection-host-key-verification.test.ts": 216,
- "src/main/ssh/ssh-connection-manager-registry.test.ts": 402,
- "src/main/ssh/ssh-connection-manager-sign-in-reuse.test.ts": 522,
- "src/main/ssh/ssh-connection-manager.test.ts": 274,
- "src/main/ssh/ssh-connection-reconnect-handshake-loss.test.ts": 254,
- "src/main/ssh/ssh-connection-reconnect-ladder.test.ts": 354,
- "src/main/ssh/ssh-connection-sftp-namespace.test.ts": 1227,
- "src/main/ssh/ssh-connection-sftp-wire.test.ts": 254,
- "src/main/ssh/ssh-connection-store-orcad-ownership.test.ts": 293,
- "src/main/ssh/ssh-connection-store.test.ts": 337,
- "src/main/ssh/ssh-connection-system-transport.test.ts": 206,
- "src/main/ssh/ssh-connection-utils.test.ts": 244,
- "src/main/ssh/ssh-connection.test.ts": 1226,
+ "src/main/ssh/connection/ssh-connection-attribution.test.ts": 34,
+ "src/main/ssh/connection/ssh-connection-auth-fallback.test.ts": 269,
+ "src/main/ssh/connection/ssh-connection-channel-error.test.ts": 142,
+ "src/main/ssh/connection/ssh-connection-channel-lifetime.test.ts": 58,
+ "src/main/ssh/connection/ssh-connection-channel-open.test.ts": 930,
+ "src/main/ssh/connection/ssh-connection-direct-startup.test.ts": 128,
+ "src/main/ssh/connection/ssh-connection-file-transfer.test.ts": 142,
+ "src/main/ssh/connection/ssh-connection-generation.test.ts": 41,
+ "src/main/ssh/connection/ssh-connection-github-probe.test.ts": 174,
+ "src/main/ssh/connection/ssh-connection-gssapi-fallback.test.ts": 237,
+ "src/main/ssh/connection/ssh-connection-host-key-store-wiring.test.ts": 524,
+ "src/main/ssh/connection/ssh-connection-host-key-verification.test.ts": 216,
+ "src/main/ssh/connection/ssh-connection-manager-registry.test.ts": 402,
+ "src/main/ssh/connection/ssh-connection-manager-sign-in-reuse.test.ts": 522,
+ "src/main/ssh/connection/ssh-connection-manager.test.ts": 274,
+ "src/main/ssh/connection/ssh-connection-reconnect-handshake-loss.test.ts": 254,
+ "src/main/ssh/connection/ssh-connection-reconnect-ladder.test.ts": 354,
+ "src/main/ssh/connection/ssh-connection-sftp-namespace.test.ts": 1227,
+ "src/main/ssh/connection/ssh-connection-sftp-wire.test.ts": 254,
+ "src/main/ssh/connection/ssh-connection-store-orcad-ownership.test.ts": 293,
+ "src/main/ssh/connection/ssh-connection-store.test.ts": 337,
+ "src/main/ssh/connection/ssh-connection-system-transport.test.ts": 206,
+ "src/main/ssh/connection/ssh-connect-config.test.ts": 244,
+ "src/main/ssh/connection/ssh-connection.test.ts": 1226,
"src/main/ssh/ssh-control-socket.test.ts": 50,
"src/main/ssh/ssh-credential-decline.test.ts": 252,
"src/main/ssh/ssh-exec-rc-stdout-noise.test.ts": 260,
@@ -5804,11 +5804,11 @@
"src/main/ssh/ssh-session-limit-error.test.ts": 27,
"src/main/ssh/system-transport/system-ssh-operations.test.ts": 275,
"src/main/ssh/ssh-system-transport.integration.test.ts": 277,
- "src/main/ssh/ssh-target-id-migration.test.ts": 46,
- "src/main/ssh/ssh-target-orcad-claims.test.ts": 306,
- "src/main/ssh/ssh-target-orcad-dependents.test.ts": 269,
- "src/main/ssh/ssh-target-orcad-preflight.test.ts": 1582,
- "src/main/ssh/ssh-target-readoption.test.ts": 52,
+ "src/main/ssh/targets/ssh-target-id-migration.test.ts": 46,
+ "src/main/ssh/targets/ssh-target-orcad-claims.test.ts": 306,
+ "src/main/ssh/targets/ssh-target-orcad-dependents.test.ts": 269,
+ "src/main/ssh/targets/ssh-target-orcad-preflight.test.ts": 1582,
+ "src/main/ssh/targets/ssh-target-readoption.test.ts": 52,
"src/main/ssh/ssh-tcp-forwarding-probe.test.ts": 810,
"src/main/ssh/ssh-windows-host-cells.test.ts": 39,
"src/main/ssh/port-forwarding/ssh2-port-forward-listen.test.ts": 30,
@@ -11691,12 +11691,12 @@
"src/shared/orca-yaml-bounds.test.ts": 169,
"src/shared/orcad-artifacts.test.ts": 84,
"src/shared/orcad-build-identity.test.ts": 46,
- "src/shared/orcad-migration-dormant-session-validation.test.ts": 314,
+ "src/shared/orcad-migration/orcad-migration-dormant-session-validation.test.ts": 314,
"src/shared/orcad-migration-manifest-contract.test.ts": 351,
- "src/shared/orcad-migration-manifest.test.ts": 393,
- "src/shared/orcad-migration-scrollback.test.ts": 240,
- "src/shared/orcad-migration-source-kind.test.ts": 362,
- "src/shared/orcad-migration-staged-catalog-validation.test.ts": 299,
+ "src/shared/orcad-migration/orcad-migration-manifest.test.ts": 393,
+ "src/shared/orcad-migration/orcad-migration-scrollback.test.ts": 240,
+ "src/shared/orcad-migration/orcad-migration-source-kind.test.ts": 362,
+ "src/shared/orcad-migration/orcad-migration-staged-catalog-validation.test.ts": 299,
"src/shared/orcad-profile-preflight.test.ts": 220,
"src/shared/orcad-stop-request.test.ts": 239,
"src/shared/orchestration-ask-timeout.test.ts": 30,
diff --git a/config/scripts/ci-shard-timings.md b/config/scripts/ci-shard-timings.md
index eee0dc34bf90..91312e4318da 100644
--- a/config/scripts/ci-shard-timings.md
+++ b/config/scripts/ci-shard-timings.md
@@ -57,13 +57,13 @@ errors. Artifact digests, all 20 report hashes, source/runtime provenance, full
discovery, and each saved assignment were checked before importing. The E2E
weights remain unchanged.
-| Shard | Job ID | Measured worker-seconds | Job seconds |
-| --- | --- | ---: | ---: |
-| 1 | 116137259336 | 1,966.470 | 568 |
-| 2 | 116137259460 | 2,080.624 | 604 |
-| 3 | 116137259318 | 2,121.495 | 611 |
-| 4 | 116137259783 | 2,094.487 | 614 |
-| 5 | 116137259540 | 2,190.636 | 641 |
+| Shard | Job ID | Measured worker-seconds | Job seconds |
+| ----- | ------------ | ----------------------: | ----------: |
+| 1 | 116137259336 | 1,966.470 | 568 |
+| 2 | 116137259460 | 2,080.624 | 604 |
+| 3 | 116137259318 | 2,121.495 | 611 |
+| 4 | 116137259783 | 2,094.487 | 614 |
+| 5 | 116137259540 | 2,190.636 | 641 |
Retrospectively assigning those measurements projects 2,090.726–2,090.747
worker-seconds per shard, instead of 1,966.470–2,190.636. That is a scheduling
diff --git a/config/scripts/ci-unit-files.mjs b/config/scripts/ci-unit-files.mjs
index a8cc794828d7..e1c265dbcd64 100644
--- a/config/scripts/ci-unit-files.mjs
+++ b/config/scripts/ci-unit-files.mjs
@@ -36,6 +36,8 @@ export const UNIT_EXCLUDE = [
'tests/e2e/relay-region-compatibility.unit.test.ts',
'tests/e2e/relay-region-correction.unit.test.ts',
// Electron probes with in-process deadlines; the package job runs them serially.
+ 'src/main/browser/popup-origin-chrome.electron.test.ts',
+ 'src/main/browser/browser-link-materialization.electron.test.ts',
'src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts',
'src/main/browser/browser-route-tcp-egress.electron.test.ts',
'src/main/browser/browser-route-webrtc-egress.electron.test.ts',
@@ -43,6 +45,7 @@ export const UNIT_EXCLUDE = [
'src/main/browser/browser-route-dns-prefetch.electron.test.ts',
'src/main/persistence/profile-state/profile-state-writer-stall.electron.test.ts',
'src/main/browser/browser-popup-compatibility.electron.test.ts',
+ 'src/main/browser/browser-popup-adapter-compatibility.electron.test.ts',
'src/main/browser/browser-popup-native-compatibility.electron.test.ts',
'src/main/browser/browser-popup-background-compatibility.electron.test.ts',
`${CROSS_VERSION_WIRE_DIR}**`
diff --git a/config/scripts/ci-unit-module-cache.mjs b/config/scripts/ci-unit-module-cache.mjs
index f7e76882dc94..30d448aa9f53 100644
--- a/config/scripts/ci-unit-module-cache.mjs
+++ b/config/scripts/ci-unit-module-cache.mjs
@@ -60,7 +60,7 @@ export async function moduleCachePolicyHash(root = ROOT) {
'pnpm-lock.yaml',
'pnpm-workspace.yaml',
'.npmrc',
- '.pnpmfile.cjs',
+ 'config/pnpmfile.cjs',
'config/.bun-version',
'config/scripts/ci-unit-module-cache.mjs',
'config/scripts/ci-unit-module-cache-contents.mjs',
diff --git a/config/scripts/ci-unit-plan.mjs b/config/scripts/ci-unit-plan.mjs
index 32c1ac1dbb3f..fce7526f9c8a 100644
--- a/config/scripts/ci-unit-plan.mjs
+++ b/config/scripts/ci-unit-plan.mjs
@@ -1,4 +1,4 @@
-import { appendFileSync, readFileSync } from 'node:fs'
+import { appendFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { runProcessSync } from '@orca/process-host'
import { collectUnitDependencyGraph } from './ci-unit-dependency-graph.mjs'
@@ -15,7 +15,8 @@ export function prepareUnitPlan(env = process.env) {
const files = discoverUnitFiles()
let plan
try {
- const event = JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, 'utf8'))
+ // From the workflow, not the payload file: background steps can read it mid-rewrite.
+ const event = { pull_request: { draft: env.ORCA_PULL_REQUEST_DRAFT === 'true' } }
if (env.GITHUB_EVENT_NAME !== 'pull_request') {
throw new Error('Full reference run')
}
diff --git a/config/scripts/ci-unit-plan.test.mjs b/config/scripts/ci-unit-plan.test.mjs
index ffbe1ce3139f..f2d14d23e301 100644
--- a/config/scripts/ci-unit-plan.test.mjs
+++ b/config/scripts/ci-unit-plan.test.mjs
@@ -48,8 +48,6 @@ it.each([
git(commit)
}
const sourceSha = git(['rev-parse', 'HEAD'])
- const eventPath = join(root, 'event.json')
- writeFileSync(eventPath, JSON.stringify({ pull_request: { draft: true } }))
const result = runProcessSync({
program: process.execPath,
args: [fileURLToPath(new URL('./ci-unit-plan.mjs', import.meta.url))],
@@ -59,7 +57,7 @@ it.each([
ORCA_BACKGROUND_LAUNCH: '1',
ORCA_UNIT_SELECTION_MODE: 'selected',
GITHUB_EVENT_NAME: 'pull_request',
- GITHUB_EVENT_PATH: eventPath,
+ ORCA_PULL_REQUEST_DRAFT: 'true',
GITHUB_SHA: sourceSha,
ORCA_SHARD_SOURCE_SHA: sourceSha,
GITHUB_OUTPUT: join(root, 'outputs'),
@@ -95,8 +93,6 @@ it.each(['5', '10', 'invalid'])(
try {
mkdirSync(join(root, 'src'))
writeFileSync(join(root, 'src/retained.test.ts'), 'export const retained = true')
- const eventPath = join(root, 'event.json')
- writeFileSync(eventPath, '{}')
const result = runProcessSync({
program: process.execPath,
args: [fileURLToPath(new URL('./ci-unit-plan.mjs', import.meta.url))],
@@ -106,7 +102,6 @@ it.each(['5', '10', 'invalid'])(
ORCA_BACKGROUND_LAUNCH: '1',
ORCA_UNIT_FULL_SHARD_COUNT: requestedCount,
GITHUB_EVENT_NAME: 'workflow_dispatch',
- GITHUB_EVENT_PATH: eventPath,
ORCA_SHARD_SOURCE_SHA: 'full-count-fixture'
}
})
diff --git a/config/scripts/compact-packaged-orcad-ripgrep.cjs b/config/scripts/compact-packaged-orcad-ripgrep.cjs
new file mode 100644
index 000000000000..31b71f997c69
--- /dev/null
+++ b/config/scripts/compact-packaged-orcad-ripgrep.cjs
@@ -0,0 +1,73 @@
+const { createHash } = require('node:crypto')
+const { existsSync, lstatSync, readFileSync, unlinkSync, writeFileSync } = require('node:fs')
+const { join } = require('node:path')
+const {
+ ORCAD_TEMPLATE_MANIFEST_FILENAME,
+ ORCAD_TEMPLATE_TARGETS_DIR,
+ orcadRipgrepArtifact,
+ orcadTemplatePackagedRipgrep
+} = require('../../src/shared/orcad-artifacts.ts')
+const { verifyPackagedOrcadTemplate } = require('./verify-packaged-orcad-template.cjs')
+
+function sha256(path) {
+ return createHash('sha256').update(readFileSync(path)).digest('hex')
+}
+
+/** Only compact the desktop's copy; the qualified template artifact remains self-contained. */
+function compactPackagedOrcadRipgrep(resourcesDir) {
+ const templateDir = join(resourcesDir, 'orcad-template')
+ const manifestPath = join(templateDir, ORCAD_TEMPLATE_MANIFEST_FILENAME)
+ verifyPackagedOrcadTemplate(resourcesDir)
+ const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'))
+ let removedBytes = 0
+ for (const [target, entry] of Object.entries(manifest.targets)) {
+ // Darwin signing rewrites each copy independently; keep their existing signing path.
+ if (target.startsWith('darwin-') || entry.packagedRipgrep) {
+ continue
+ }
+ const filename = orcadRipgrepArtifact(target)
+ const shared = join(resourcesDir, ...filename.split('/'))
+ if (!existsSync(shared)) {
+ continue
+ }
+ const metadata = lstatSync(shared)
+ if (!metadata.isFile() || metadata.isSymbolicLink()) {
+ throw new Error(`Packaged ripgrep must be a regular file: ${shared}`)
+ }
+ // A different desktop binary remains separate; only byte-identical payloads may share storage.
+ if (sha256(shared) !== entry.files[filename]) {
+ continue
+ }
+ const duplicate = join(templateDir, ORCAD_TEMPLATE_TARGETS_DIR, target, ...filename.split('/'))
+ removedBytes += lstatSync(duplicate).size
+ unlinkSync(duplicate)
+ entry.packagedRipgrep = true
+ }
+ if (removedBytes > 0) {
+ manifest.schemaVersion = 4
+ writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`)
+ }
+ verifyPackagedOrcadTemplate(resourcesDir)
+ return removedBytes
+}
+
+/** Reseal only shared binaries explicitly returned by the Windows signer. */
+function resealSignedPackagedRipgrep(resourcesDir, signedResourcePaths) {
+ const templateDir = join(resourcesDir, 'orcad-template')
+ const manifestPath = join(templateDir, ORCAD_TEMPLATE_MANIFEST_FILENAME)
+ const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'))
+ const signed = new Set(signedResourcePaths)
+ for (const [target, entry] of Object.entries(manifest.targets)) {
+ const filename = orcadTemplatePackagedRipgrep(
+ manifest.schemaVersion,
+ target,
+ entry.packagedRipgrep
+ )
+ if (filename?.startsWith('ripgrep/win32-') && signed.has(filename)) {
+ entry.files[filename] = sha256(join(resourcesDir, ...filename.split('/')))
+ }
+ }
+ writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`)
+}
+
+module.exports = { compactPackagedOrcadRipgrep, resealSignedPackagedRipgrep }
diff --git a/config/scripts/compact-packaged-orcad-ripgrep.test.mjs b/config/scripts/compact-packaged-orcad-ripgrep.test.mjs
new file mode 100644
index 000000000000..d79d34239f09
--- /dev/null
+++ b/config/scripts/compact-packaged-orcad-ripgrep.test.mjs
@@ -0,0 +1,219 @@
+import { createHash } from 'node:crypto'
+import { existsSync } from 'node:fs'
+import { copyFile, mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'
+import { createRequire } from 'node:module'
+import { tmpdir } from 'node:os'
+import { basename, dirname, join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { assembleOrcadArtifact } from '../../src/main/orcad/orcad-artifact-materializer.ts'
+import { readOrcadArtifactIdentity } from '../../src/main/orcad/orcad-artifact-identity.ts'
+import {
+ ORCAD_TEMPLATE_MANIFEST_FILENAME,
+ orcadRipgrepArtifact
+} from '../../src/shared/orcad-artifacts.ts'
+import { writeOrcadTemplateTestFixture } from './orcad-template-test-fixture.mjs'
+
+const require = createRequire(import.meta.url)
+const { compactPackagedOrcadRipgrep } = require('./compact-packaged-orcad-ripgrep.cjs')
+const { verifyPackagedOrcadTemplate } = require('./verify-packaged-orcad-template.cjs')
+const {
+ finalizePackagedOrcadTemplate,
+ resealSignedWindowsApp
+} = require('./packaged-orcad-template.cjs')
+const roots = []
+const hash = (bytes) => createHash('sha256').update(bytes).digest('hex')
+
+afterEach(async () => {
+ vi.restoreAllMocks()
+ await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
+})
+
+async function fixture() {
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ const appDir = await mkdtemp(join(tmpdir(), 'orca-compact-ripgrep-'))
+ roots.push(appDir)
+ const resourcesDir = join(appDir, 'resources')
+ const templateDir = await writeOrcadTemplateTestFixture(resourcesDir, { compat: true })
+ const manifestPath = join(templateDir, ORCAD_TEMPLATE_MANIFEST_FILENAME)
+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
+ for (const [target, entry] of Object.entries(manifest.targets)) {
+ const filename = orcadRipgrepArtifact(target)
+ const desktop = join(resourcesDir, filename)
+ await mkdir(dirname(desktop), { recursive: true })
+ const bytes = Buffer.from(`${filename}:static binary`)
+ await writeFile(desktop, bytes)
+ await writeFile(join(templateDir, 'targets', target, filename), bytes)
+ entry.files[filename] = hash(bytes)
+ }
+ await writeFile(manifestPath, JSON.stringify(manifest))
+ return { appDir, resourcesDir, templateDir, manifest, manifestPath }
+}
+
+describe('packaged ripgrep storage', () => {
+ it('preserves every deployed slot byte and version, including old schema 3 templates', async () => {
+ const f = await fixture()
+ const original = new Map()
+ const cacheRoot = join(f.appDir, 'cache')
+ for (const target of Object.keys(f.manifest.targets)) {
+ const slot = await assembleOrcadArtifact({ templateDir: f.templateDir, cacheRoot, target })
+ original.set(target, basename(slot))
+ }
+ expect(compactPackagedOrcadRipgrep(f.resourcesDir)).toBeGreaterThan(0)
+ expect(compactPackagedOrcadRipgrep(f.resourcesDir)).toBe(0)
+ for (const target of Object.keys(f.manifest.targets)) {
+ const slot = await assembleOrcadArtifact({
+ templateDir: f.templateDir,
+ cacheRoot: join(f.appDir, 'fresh-cache'),
+ target
+ })
+ expect(basename(slot)).toBe(original.get(target))
+ expect(await readOrcadArtifactIdentity(slot)).toBe(original.get(target))
+ const filename = orcadRipgrepArtifact(target)
+ expect(await readFile(join(slot, filename))).toEqual(
+ await readFile(join(f.resourcesDir, filename))
+ )
+ if (process.platform !== 'win32' && !target.startsWith('win32-')) {
+ expect((await stat(join(slot, filename))).mode & 0o777).toBe(0o755)
+ }
+ expect(existsSync(join(f.templateDir, 'targets', target, filename))).toBe(
+ target.startsWith('darwin-')
+ )
+ }
+ // No template storage schema reaches the deployment slot.
+ expect(
+ existsSync(
+ join(
+ cacheRoot,
+ 'linux-x64-glibc',
+ original.get('linux-x64-glibc'),
+ ORCAD_TEMPLATE_MANIFEST_FILENAME
+ )
+ )
+ ).toBe(false)
+ })
+
+ it('compacts through the real desktop finalization hook', async () => {
+ const f = await fixture()
+ await finalizePackagedOrcadTemplate(f.resourcesDir, {
+ platform: 'linux',
+ signMacBinary: vi.fn()
+ })
+ const manifest = JSON.parse(await readFile(f.manifestPath, 'utf8'))
+ expect(manifest.schemaVersion).toBe(4)
+ expect(Object.values(manifest.targets).filter((entry) => entry.packagedRipgrep)).toHaveLength(7)
+ })
+
+ it('keeps binaries that differ and never removes Darwin signing inputs', async () => {
+ const f = await fixture()
+ const desktop = join(f.resourcesDir, 'ripgrep/linux-arm64/rg')
+ await writeFile(desktop, 'different desktop bytes')
+ compactPackagedOrcadRipgrep(f.resourcesDir)
+ const manifest = JSON.parse(await readFile(f.manifestPath, 'utf8'))
+ expect(manifest.schemaVersion).toBe(4)
+ for (const [target, entry] of Object.entries(manifest.targets)) {
+ expect(entry.packagedRipgrep).toBe(
+ target.startsWith('darwin-') || target.startsWith('linux-arm64') ? undefined : true
+ )
+ }
+ expect(await readFile(desktop, 'utf8')).toBe('different desktop bytes')
+ })
+
+ it('does not launder corrupted target bytes while removing duplicates', async () => {
+ const f = await fixture()
+ await writeFile(join(f.templateDir, 'targets/linux-x64-glibc/ripgrep/linux-x64/rg'), 'tampered')
+ expect(() => compactPackagedOrcadRipgrep(f.resourcesDir)).toThrow('checksum mismatch')
+ })
+
+ it.each(['tampered', 'missing'])('rejects %s shared bytes before deploying', async (damage) => {
+ const f = await fixture()
+ compactPackagedOrcadRipgrep(f.resourcesDir)
+ const shared = join(f.resourcesDir, 'ripgrep/linux-x64/rg')
+ await (damage === 'missing' ? rm(shared) : writeFile(shared, 'tampered'))
+ expect(() => verifyPackagedOrcadTemplate(f.resourcesDir)).toThrow()
+ await expect(
+ assembleOrcadArtifact({
+ templateDir: f.templateDir,
+ cacheRoot: join(f.appDir, 'cache'),
+ target: 'linux-x64-glibc'
+ })
+ ).rejects.toThrow()
+ })
+
+ it.each([3, 4])('rejects arbitrary reference paths in schema %s', async (schemaVersion) => {
+ const f = await fixture()
+ f.manifest.schemaVersion = schemaVersion
+ f.manifest.targets['linux-x64-glibc'].packagedRipgrep = '../../outside/rg'
+ await writeFile(f.manifestPath, JSON.stringify(f.manifest))
+ expect(() => verifyPackagedOrcadTemplate(f.resourcesDir)).toThrow(
+ 'Invalid packaged ripgrep reference'
+ )
+ await expect(
+ assembleOrcadArtifact({
+ templateDir: f.templateDir,
+ cacheRoot: join(f.appDir, 'cache'),
+ target: 'linux-x64-glibc'
+ })
+ ).rejects.toThrow()
+ })
+
+ it('rejects references under schema 3 and Darwin references under schema 4', async () => {
+ const f = await fixture()
+ for (const [schemaVersion, target] of [
+ [3, 'linux-x64-glibc'],
+ [4, 'darwin-arm64']
+ ]) {
+ const entry = f.manifest.targets[target]
+ f.manifest.schemaVersion = schemaVersion
+ entry.packagedRipgrep = true
+ await writeFile(f.manifestPath, JSON.stringify(f.manifest))
+ expect(() => verifyPackagedOrcadTemplate(f.resourcesDir)).toThrow(
+ 'Invalid packaged ripgrep reference'
+ )
+ await expect(
+ assembleOrcadArtifact({
+ templateDir: f.templateDir,
+ cacheRoot: join(f.appDir, 'cache'),
+ target
+ })
+ ).rejects.toThrow('Invalid packaged ripgrep reference')
+ delete entry.packagedRipgrep
+ }
+ })
+
+ it('rejects an extra target copy left beside a reference', async () => {
+ const f = await fixture()
+ compactPackagedOrcadRipgrep(f.resourcesDir)
+ await copyFile(
+ join(f.resourcesDir, 'ripgrep/linux-x64/rg'),
+ join(f.templateDir, 'targets/linux-x64-glibc/ripgrep/linux-x64/rg')
+ )
+ expect(() => verifyPackagedOrcadTemplate(f.resourcesDir)).toThrow('file inventory mismatch')
+ })
+
+ it.skipIf(process.platform === 'win32')('rejects a shared symlink', async () => {
+ const f = await fixture()
+ const shared = join(f.resourcesDir, 'ripgrep/linux-x64/rg')
+ const other = `${shared}.other`
+ await copyFile(shared, other)
+ await rm(shared)
+ await symlink(other, shared)
+ expect(() => compactPackagedOrcadRipgrep(f.resourcesDir)).toThrow('must be a regular file')
+ })
+
+ it('reseals only shared Windows binaries explicitly returned by SignPath', async () => {
+ const f = await fixture()
+ compactPackagedOrcadRipgrep(f.resourcesDir)
+ const filename = 'ripgrep/win32-x64/rg.exe'
+ const shared = join(f.resourcesDir, filename)
+ await writeFile(shared, 'authenticode-signed binary')
+ const list = join(f.appDir, 'inner-signing-list.txt')
+ await writeFile(list, 'Orca.exe\r\n')
+ expect(() => resealSignedWindowsApp(f.appDir, list)).toThrow('checksum mismatch')
+ await writeFile(list, 'Orca.exe\r\nresources\\ripgrep\\win32-x64\\rg.exe\r\n')
+ resealSignedWindowsApp(f.appDir, list)
+ const manifest = JSON.parse(await readFile(f.manifestPath, 'utf8'))
+ expect(manifest.targets['win32-x64'].files[filename]).toBe(hash(await readFile(shared)))
+ await writeFile(join(f.resourcesDir, 'ripgrep/linux-x64/rg'), 'unrelated change')
+ expect(() => resealSignedWindowsApp(f.appDir, list)).toThrow('checksum mismatch')
+ })
+})
diff --git a/config/scripts/dark-build-switches.test.mjs b/config/scripts/dark-build-switches.test.mjs
index 3b435aa4a6af..f9ff85af1979 100644
--- a/config/scripts/dark-build-switches.test.mjs
+++ b/config/scripts/dark-build-switches.test.mjs
@@ -4,8 +4,13 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
-const { assertNotDarkBuild, darkBuildDefines, darkBuildMarkers, turnedOnDarkBuildDefines } =
- createRequire(import.meta.url)('../dark-build-switches.cjs')
+const {
+ assertNoDarkOutputOfOffSwitches,
+ assertNotDarkBuild,
+ darkBuildDefines,
+ darkBuildMarkers,
+ turnedOnDarkBuildDefines
+} = createRequire(import.meta.url)('../dark-build-switches.cjs')
const scratch = []
afterEach(async () => {
@@ -48,4 +53,18 @@ describe('dark build switches', () => {
expect(() => assertNotDarkBuild([clean, dark], {})).toThrow(dark)
}
)
+
+ it.each([
+ ['ORCA_LAYOUT_OWNER', 'ORCA_SIDEBAR_INPUTS'],
+ ['ORCA_SIDEBAR_INPUTS', 'ORCA_LAYOUT_OWNER']
+ ])('lets a dark build carry its own %s output, never one of %s', async (own, other) => {
+ const dark = await mkdtemp(join(tmpdir(), 'orca-dark-build-own-'))
+ scratch.push(dark)
+ const [marker] = darkBuildMarkers({ [own]: 'main' })
+ await writeFile(join(dark, marker.fileName), marker.text)
+
+ expect(() => assertNoDarkOutputOfOffSwitches([dark], { [own]: 'main' })).not.toThrow()
+ expect(() => assertNoDarkOutputOfOffSwitches([dark], { [other]: 'main' })).toThrow(dark)
+ expect(() => assertNoDarkOutputOfOffSwitches([dark], {})).toThrow(dark)
+ })
})
diff --git a/config/scripts/dev-channel-windows-staging.test.mjs b/config/scripts/dev-channel-windows-staging.test.mjs
new file mode 100644
index 000000000000..b0d81cc42345
--- /dev/null
+++ b/config/scripts/dev-channel-windows-staging.test.mjs
@@ -0,0 +1,137 @@
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, describe, expect, it } from 'vitest'
+import { parse } from 'yaml'
+import { runProcess } from '@orca/process-host'
+
+function readWorkflow(name) {
+ return parse(readFileSync(new URL(`../../.github/workflows/${name}`, import.meta.url), 'utf8'))
+}
+
+const build = readWorkflow('dev-channel-win-build.yml')
+const publish = readWorkflow('dev-channel-win-publish.yml')
+const buildSteps = build.jobs['build-win'].steps
+const publishSteps = publish.jobs['publish-win'].steps
+const directory = mkdtempSync(join(tmpdir(), 'dev-channel-windows-staging-'))
+afterAll(() => rmSync(directory, { recursive: true, force: true }))
+
+async function runScript(script, { cwd = directory, env = {} } = {}) {
+ return runProcess({ program: 'bash', args: ['-c', script], cwd, env: { ...process.env, ...env } })
+}
+
+describe('Windows dev-channel artifact staging', () => {
+ it('keeps existing callers publishing by default', () => {
+ expect(build.on.workflow_call.inputs.stage_only).toMatchObject({
+ type: 'boolean',
+ default: false,
+ required: false
+ })
+ expect(build.on.workflow_dispatch.inputs.stage_only).toBeUndefined()
+ const normalPackage = buildSteps.find((step) => step.name === 'Publish Windows artifacts')
+ expect(normalPackage.with.command).toContain('--publish always')
+ expect(normalPackage.if).toBe('${{ !inputs.stage_only }}')
+ })
+
+ it('never mints or consumes release-write credentials while staging', () => {
+ for (const step of buildSteps) {
+ const credentialed = JSON.stringify(step).includes('secrets.') || step.env?.GH_TOKEN
+ if (credentialed) {
+ expect(step.if, step.name).toBe('${{ !inputs.stage_only }}')
+ }
+ }
+ const stagePackage = buildSteps.find(
+ (step) => step.name === 'Package Windows artifacts without publishing'
+ )
+ expect(stagePackage.if).toBe('inputs.stage_only')
+ expect(stagePackage.with.command).toContain('--publish never')
+ expect(stagePackage.env.GH_TOKEN).toBeUndefined()
+ for (const workflow of [build, publish]) {
+ const contents = JSON.stringify(workflow.jobs)
+ expect(contents).not.toMatch(/SignPath|SIGNPATH|signpath/)
+ }
+ })
+
+ it('passes exactly the installer, blockmap, and update manifest within the same run', () => {
+ const upload = buildSteps.find((step) => step.uses?.startsWith('actions/upload-artifact@'))
+ const download = publishSteps.find((step) =>
+ step.uses?.startsWith('actions/download-artifact@')
+ )
+ expect(upload.if).toBe('inputs.stage_only')
+ expect(upload.with.path.trim().split('\n')).toEqual([
+ 'dist/orca-windows-setup.exe',
+ 'dist/orca-windows-setup.exe.blockmap',
+ 'dist/latest.yml'
+ ])
+ expect(upload.with['if-no-files-found']).toBe('error')
+ expect(download.with).toEqual({
+ name: upload.with.name,
+ path: 'staged-windows'
+ })
+ expect(build.concurrency.group).not.toBe(publish.concurrency.group)
+ })
+
+ it.each([
+ ['build', buildSteps, 'dist'],
+ ['publish', publishSteps, 'staged-windows']
+ ])('fails %s on any missing or empty staged file', async (_name, steps, folder) => {
+ const step = steps.find((entry) => entry.name === 'Validate staged Windows artifacts')
+ const attempt = mkdtempSync(join(directory, 'files-'))
+ mkdirSync(join(attempt, folder))
+ expect((await runScript(step.run, { cwd: attempt })).code).toBe(1)
+ for (const file of [
+ 'orca-windows-setup.exe',
+ 'orca-windows-setup.exe.blockmap',
+ 'latest.yml'
+ ]) {
+ writeFileSync(join(attempt, folder, file), '')
+ expect((await runScript(step.run, { cwd: attempt })).code).toBe(1)
+ writeFileSync(join(attempt, folder, file), 'artifact')
+ }
+ expect((await runScript(step.run, { cwd: attempt })).code).toBe(0)
+ })
+
+ it('validates the channel/tag before downloading artifacts or minting a token', async () => {
+ const step = publishSteps[0]
+ for (const [channel, tag, code] of [
+ ['adhoc', 'v1.4.232-adhoc.20261020052302', 0],
+ ['daily', 'v1.4.232-daily.20261020052302', 0],
+ ['hourly', 'v1.4.232-hourly.20261020052302', 0],
+ ['stable', 'v1.4.232-stable.20261020052302', 1],
+ ['adhoc', 'v1.4.232-daily.20261020052302', 1],
+ ['adhoc', '--help', 1],
+ ['adhoc', '$(exit 99)', 1]
+ ]) {
+ expect((await runScript(step.run, { env: { CHANNEL: channel, TAG: tag } })).code).toBe(code)
+ }
+ })
+
+ it('requires a live existing release and never creates one', async () => {
+ const guard = publishSteps.find((step) => step.name === 'Confirm the target release is live')
+ for (const [response, code] of [
+ ['false', 0],
+ ['true', 1],
+ ['', 1]
+ ]) {
+ const stub = 'gh() { printf "%s" "$STUB_RESPONSE"; }; export -f gh;\n'
+ expect(
+ (
+ await runScript(`${stub}${guard.run}`, {
+ env: { STUB_RESPONSE: response, CHANNEL: 'adhoc', TAG: 'v1.4.232-adhoc.1' }
+ })
+ ).code
+ ).toBe(code)
+ }
+ expect(
+ (
+ await runScript(`gh() { return 1; };\n${guard.run}`, {
+ env: { CHANNEL: 'adhoc', TAG: 'v1.4.232-adhoc.1' }
+ })
+ ).code
+ ).toBe(1)
+ const upload = publishSteps.find((step) => step.name === 'Publish Windows artifacts')
+ expect(upload.with.command).toContain('gh release upload')
+ expect(JSON.stringify(publish)).not.toContain('gh release create')
+ expect(publishSteps.indexOf(guard)).toBeLessThan(publishSteps.indexOf(upload))
+ })
+})
diff --git a/config/scripts/electron-builder-config.test.mjs b/config/scripts/electron-builder-config.test.mjs
index 37ae87b21edb..4d43b9523bb2 100644
--- a/config/scripts/electron-builder-config.test.mjs
+++ b/config/scripts/electron-builder-config.test.mjs
@@ -26,6 +26,7 @@ describe('electron-builder config', () => {
expect.arrayContaining([
'!src{,/**/*}',
'!config{,/**/*}',
+ '!cloud{,/**/*}',
'!docs{,/**/*}',
'!mobile{,/**/*}',
'!native{,/**/*}',
@@ -82,6 +83,10 @@ describe('electron-builder config', () => {
expect(packs(toolingPath)).toBe(false)
}
expect(packs('out/main/index.js')).toBe(true)
+ expect(packs('out/web/web-index.html')).toBe(false)
+ expect(packs('out/web/assets/shared.js')).toBe(false)
+ expect(packs('out/renderer/web-index.html')).toBe(true)
+ expect(packs('out/renderer/.orca-web-client-files.json')).toBe(true)
})
it.each(['file', 'directory'])('keeps a root notes %s out of app.asar', async (kind) => {
@@ -424,8 +429,9 @@ describe('electron-builder config', () => {
)
const isUnpacked = matcher.createFilter()
for (const webPath of [
- 'out/web/web-index.html',
- 'out/web/assets/index-abc123.js',
+ 'out/renderer/web-index.html',
+ 'out/renderer/assets/index-abc123.js',
+ 'out/renderer/.orca-web-client-files.json',
'out/mobile-web/manifest.json',
'out/mobile-web/assets/index-abc123.js'
]) {
@@ -610,6 +616,24 @@ describe('arch-aware packaging guard', () => {
}
})
+ it('refuses to package an orcad template marked by a dark test build', async () => {
+ const templateDir = join(scratch, 'orcad-template')
+ await mkdir(templateDir, { recursive: true })
+ await writeFile(join(templateDir, 'ORCA_LAYOUT_OWNER_MAIN'), 'dark\n')
+ try {
+ expect(() =>
+ electronBuilderConfig.beforePack(
+ { electronPlatformName: process.platform, arch: HOST_ARCH },
+ bundleDir,
+ () => undefined,
+ templateDir
+ )
+ ).toThrow(`${templateDir} holds an ORCA_LAYOUT_OWNER=main build`)
+ } finally {
+ await rm(templateDir, { recursive: true, force: true })
+ }
+ })
+
it('requires the other architecture natives to be installed', () => {
const otherSherpaInstalled = existsSync(
join(REPO_ROOT, 'node_modules', otherSherpa, 'package.json')
diff --git a/config/scripts/electron-builder-runtime-resources.test.mjs b/config/scripts/electron-builder-runtime-resources.test.mjs
index 021711ab574a..69f5acf2b4cc 100644
--- a/config/scripts/electron-builder-runtime-resources.test.mjs
+++ b/config/scripts/electron-builder-runtime-resources.test.mjs
@@ -561,11 +561,15 @@ async function seedPackagedServePayload(resourcesDir) {
const entry = join(resourcesDir, 'app.asar.unpacked', 'out', ...SERVE_SELECTION_ENTRY.split('/'))
await mkdir(dirname(entry), { recursive: true })
await writeFile(entry, '')
- const webDir = join(resourcesDir, 'app.asar.unpacked', 'out', 'web')
+ const webDir = join(resourcesDir, 'app.asar.unpacked', 'out', 'renderer')
await mkdir(join(webDir, 'assets'), { recursive: true })
await mkdir(join(resourcesDir, 'app.asar.unpacked', 'out', 'cli'), { recursive: true })
await writeFile(join(webDir, 'web-index.html'), '')
await writeFile(join(webDir, 'assets', 'web.js'), '')
+ await writeFile(
+ join(webDir, '.orca-web-client-files.json'),
+ JSON.stringify(['web-index.html', 'assets/web.js'])
+ )
}
async function seedBundledRipgrep(resourcesDir) {
diff --git a/config/scripts/ensure-native-runtime.mjs b/config/scripts/ensure-native-runtime.mjs
index 78251f8d6a45..f81c2a0f6460 100644
--- a/config/scripts/ensure-native-runtime.mjs
+++ b/config/scripts/ensure-native-runtime.mjs
@@ -3,7 +3,6 @@
import { spawnSync } from 'node:child_process'
import { createRequire } from 'node:module'
import { existsSync, readFileSync, realpathSync } from 'node:fs'
-import { release } from 'node:os'
import { basename, dirname, resolve } from 'node:path'
import {
ensureWindowsProcessTreeCommandLinePatch,
@@ -15,7 +14,18 @@ import {
} from './windows-process-tree-gyp-rebuild.mjs'
import { runProcessSync } from '@orca/process-host'
import { describeProcessFailure } from './process-failure-message.mjs'
-import { disableMsbuildFileTrackingOnWindows } from './msbuild-file-tracking.mjs'
+import {
+ isSharedNativePackage,
+ isSourceBuildDir,
+ nodePtyBuildArtifactPaths,
+ nodePtyNativeModuleName,
+ resolveNativePackageDir
+} from './native-build-entry.cjs'
+import {
+ checkNativeBuildDirLoads,
+ ensureSharedNativeBuild,
+ nativeBuildEnvironment
+} from './shared-native-build.mjs'
const require = createRequire(import.meta.url)
const { assertNodePtyJobOwnership, nodePtyAddonPath } = require('./node-pty-job-ownership.cjs')
@@ -23,6 +33,8 @@ const { assertWindowsProcessTreeCreationTime } = require('./windows-process-tree
const scriptPath = import.meta.filename
const projectDir = resolve(import.meta.dirname, '../..')
const runtime = readRuntimeArg()
+// Why: `pnpm rebuild:node` must rebuild even a loadable node-pty; a shared slot still reuses its usable entry.
+const forceRebuild = process.argv.includes('--force')
const NATIVE_MODULES = [
'node-pty',
@@ -70,12 +82,14 @@ function readRuntimeArg() {
function ensureNodeRuntime() {
const initial = runNodeCheck()
const patchedNodePtyRebuildReason = getPatchedNodePtyRebuildReason()
- if (initial.ok && !patchedNodePtyRebuildReason) {
+ if (initial.ok && !patchedNodePtyRebuildReason && !forceRebuild) {
return
}
- if (patchedNodePtyRebuildReason) {
- console.warn(`[native-runtime] ${patchedNodePtyRebuildReason}`)
+ if (patchedNodePtyRebuildReason || forceRebuild) {
+ console.warn(
+ `[native-runtime] ${patchedNodePtyRebuildReason ?? 'Forced rebuild of node-pty for Node.'}`
+ )
if (!initial.ok) {
printCheckError(initial)
}
@@ -303,28 +317,25 @@ function loadNodePtyNativeModule() {
require('node-pty')
const { loadNativeModule } = require('node-pty/lib/utils')
- const nativeName = getNodePtyNativeModuleName()
+ const nativeName = nodePtyNativeModuleName()
// Why: node-pty's Windows JS wrapper defers conpty.node/pty.node until a
// terminal is created, so require('node-pty') alone can miss ABI mismatches.
const native = loadNativeModule(nativeName)
- assertNodePtyWindowsConptyRuntime(native?.dir)
- assertNodePtyJobOwnership({
- nativeName,
- native,
- addonPath: nodePtyAddonPath(require.resolve('node-pty/lib/utils'), native, nativeName)
- })
- if (requiresPatchedNodePtySourceBuild() && !isNodePtyReleaseBuildDir(native?.dir)) {
+ const addonPath = nodePtyAddonPath(require.resolve('node-pty/lib/utils'), native, nativeName)
+ assertNodePtyWindowsConptyRuntime(native?.dir, dirname(addonPath))
+ assertNodePtyJobOwnership({ nativeName, native, addonPath })
+ if (requiresPatchedNodePtySourceBuild() && !isSourceBuildDir(native?.dir)) {
throw new Error(
`node-pty resolved to ${native.dir}; expected build/Release so Orca's node-pty patch is active`
)
}
}
-function assertNodePtyWindowsConptyRuntime(nativeDir) {
- if (process.platform !== 'win32' || !isNodePtyReleaseBuildDir(nativeDir)) {
+function assertNodePtyWindowsConptyRuntime(nativeDir, loadedDir) {
+ if (process.platform !== 'win32' || !isSourceBuildDir(nativeDir)) {
return
}
- const runtimeDir = resolve(projectDir, 'node_modules', 'node-pty', 'build', 'Release', 'conpty')
+ const runtimeDir = resolve(loadedDir, 'conpty')
const missingFile = NODE_PTY_CONPTY_RUNTIME_FILES.find(
(filename) => !existsSync(resolve(runtimeDir, filename))
)
@@ -333,14 +344,6 @@ function assertNodePtyWindowsConptyRuntime(nativeDir) {
}
}
-function getNodePtyNativeModuleName() {
- if (process.platform !== 'win32') {
- return 'pty'
- }
-
- return getWindowsBuildNumber() >= 18309 ? 'conpty' : 'pty'
-}
-
function getPatchedNodePtyRebuildReason() {
if (!requiresPatchedNodePtySourceBuild()) {
return null
@@ -349,7 +352,7 @@ function getPatchedNodePtyRebuildReason() {
// Why: a loadable upstream node-pty prebuild is not enough; Orca's Unix and
// Windows patches only land in the source-built build/Release artifacts.
const nodePtyDir = resolve(projectDir, 'node_modules', 'node-pty')
- const artifactPaths = patchedNodePtyArtifactPaths(nodePtyDir)
+ const artifactPaths = nodePtyBuildArtifactPaths(nodePtyDir)
const missingArtifact = artifactPaths.find((artifactPath) => !existsSync(artifactPath))
if (!missingArtifact) {
@@ -359,23 +362,6 @@ function getPatchedNodePtyRebuildReason() {
return 'Patched node-pty build artifacts are missing; rebuilding native deps.'
}
-function patchedNodePtyArtifactPaths(nodePtyDir) {
- if (process.platform === 'win32') {
- const releaseDir = resolve(nodePtyDir, 'build', 'Release')
- return [
- resolve(releaseDir, 'conpty.node'),
- ...NODE_PTY_CONPTY_RUNTIME_FILES.map((filename) => resolve(releaseDir, 'conpty', filename))
- ]
- }
-
- const artifactPaths = [resolve(nodePtyDir, 'build', 'Release', 'pty.node')]
- // Why: node-pty only builds spawn-helper on macOS; Linux builds only pty.node.
- if (process.platform === 'darwin') {
- artifactPaths.push(resolve(nodePtyDir, 'build', 'Release', 'spawn-helper'))
- }
- return artifactPaths
-}
-
function requiresPatchedNodePtySourceBuild() {
const nodePtyPatchPath = resolve(projectDir, 'config', 'patches', 'node-pty@1.1.0.patch')
if (!existsSync(nodePtyPatchPath)) {
@@ -385,18 +371,14 @@ function requiresPatchedNodePtySourceBuild() {
return existsSync(resolve(projectDir, 'node_modules', 'node-pty'))
}
-function isNodePtyReleaseBuildDir(nativeDir) {
- return typeof nativeDir === 'string' && nativeDir.replace(/\\/g, '/').includes('build/Release/')
-}
-
-function getWindowsBuildNumber() {
- const match = /(\d+)\.(\d+)\.(\d+)/g.exec(release())
- return match && match.length === 4 ? Number.parseInt(match[3], 10) : 0
-}
-
function rebuildNodeRuntimeModules(moduleNames) {
for (const moduleName of moduleNames) {
let moduleDir = dirname(require.resolve(`${moduleName}/package.json`))
+ if (isSharedNativePackage(moduleName, projectDir)) {
+ // Why: other worktrees load this very dir, so build beside it instead of over it.
+ buildSharedNodeRuntimeModule(moduleName, resolveNativePackageDir(moduleName, projectDir))
+ continue
+ }
if (moduleName === '@vscode/windows-process-tree') {
// Why before node-gyp: this module is rebuilt precisely because the
// binary was the unpatched one, and pnpm materializes it unpatched often
@@ -425,11 +407,23 @@ function rebuildNodeRuntimeModules(moduleNames) {
}
}
+function buildSharedNodeRuntimeModule(moduleName, packageDir) {
+ try {
+ ensureSharedNativeBuild({
+ moduleName,
+ packageDir,
+ checkBuildDir: (dir) => checkNativeBuildDirLoads(moduleName, dir),
+ nodeGypEntry: process.env.npm_config_node_gyp || undefined,
+ log: (message) => console.warn(`[native-runtime] ${message}`)
+ })
+ } catch (error) {
+ console.error(`[native-runtime] ${moduleName}: ${formatError(error)}`)
+ process.exit(1)
+ }
+}
+
function runNodeGyp(moduleName, { args, cwd }) {
- const env =
- process.platform === 'linux'
- ? { ...process.env, CXXFLAGS: `${process.env.CXXFLAGS ?? ''} -std=gnu++2a`.trim() }
- : disableMsbuildFileTrackingOnWindows({ ...process.env })
+ const env = nativeBuildEnvironment()
const result = runProcessSync({
program: process.execPath,
args,
diff --git a/config/scripts/install-electron-package-binary.mjs b/config/scripts/install-electron-package-binary.mjs
index 2ac195b00264..adb977c94486 100644
--- a/config/scripts/install-electron-package-binary.mjs
+++ b/config/scripts/install-electron-package-binary.mjs
@@ -17,6 +17,7 @@ import { platform as osPlatform, tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { getElectronPlatformPath } from './electron-platform-path.mjs'
import { getZipExtractorCommand } from './zip-extractor-command.mjs'
+import { isSharedPackageDir } from './native-build-entry.cjs'
import {
shareElectronDistFromCache,
hasAdoptedSharedElectronDist,
@@ -80,6 +81,8 @@ async function main() {
return
}
+ assertForeignTargetLeavesSharedElectronAlone()
+
// Why: PR tests run under system Node after native modules are rebuilt for
// Node. Install only Electron's npm package binary here; do not run the full
// Electron native-module rebuild path, which would undo the Node ABI rebuild.
@@ -95,6 +98,25 @@ async function main() {
}
}
+/**
+ * node_modules/electron/dist exists to run Electron on this machine; packaging downloads its own
+ * Electron per target. Under pnpm's global virtual store that dir is shared by every worktree, so a
+ * foreign-target install would hand them all an Electron they cannot run.
+ */
+function assertForeignTargetLeavesSharedElectronAlone() {
+ if (
+ (targetPlatform === osPlatform() && targetArch === process.arch) ||
+ !isSharedPackageDir(electronPackageDir, projectDir)
+ ) {
+ return
+ }
+ throw new Error(
+ `Refusing to install a ${targetPlatform}-${targetArch} Electron into ${electronPackageDir}, ` +
+ 'which every worktree shares through the global virtual store. Unset ELECTRON_INSTALL_ARCH ' +
+ '/ npm_config_arch, or install with ORCA_PNPM_GLOBAL_VIRTUAL_STORE=0.'
+ )
+}
+
function electronPackageIsUsable() {
try {
const installedPlatformPath = readFileSync(resolve(electronPackageDir, 'path.txt'), 'utf8')
@@ -182,6 +204,10 @@ async function installElectronPackageBinary(sharedEntry) {
process.exit(1)
}
+ // Why: a worktree sharing this slot may have installed it during the download; never swap out a dist it runs.
+ if (isSharedPackageDir(electronPackageDir, projectDir) && electronPackageIsUsable()) {
+ return
+ }
moveExtractedElectronDist(extractDir, electronDistDir)
if (sharedEntry !== null) {
publishElectronDistForSiblingWorktrees(sharedEntry, electronDistDir)
diff --git a/config/scripts/layout-save-divergence-summary.mjs b/config/scripts/layout-save-divergence-summary.mjs
new file mode 100644
index 000000000000..ea5e2519b3bd
--- /dev/null
+++ b/config/scripts/layout-save-divergence-summary.mjs
@@ -0,0 +1,148 @@
+// Counts a layout-owner run's save-divergence reports (ORCA_LAYOUT_SAVE_DIVERGENCE_LOG) by kind
+// and by the spec that was running when each was written.
+// Usage: node layout-save-divergence-summary.mjs [playwright.json...]
+import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { basename } from 'node:path'
+import process from 'node:process'
+import { pathToFileURL } from 'node:url'
+
+const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g
+// Saves land a little after the action that caused them, sometimes after the test ends.
+const AFTER_END_MS = 15_000
+const BEFORE_START_MS = 2_000
+
+/** A report path with its record id (a uuid or a file path) dropped: `group changed: tabOrder`. */
+export function divergenceKind(path) {
+ const named =
+ /^(editor view|pane tree|group|tab|file|browser|leaf|terminal) (.+?) (added|removed|changed: .*|tab-bar entry removed|kept over .*)$/s.exec(
+ path
+ )
+ const kind = named ? `${named[1]} ${named[3]}` : path.replace(/^\d+ /, ' ')
+ return kind.replace(UUID, '')
+}
+
+/** Every test attempt's time span, from Playwright JSON reports. */
+export function testSpans(reports) {
+ const spans = []
+ const visit = (suite, file) => {
+ const suiteFile = suite.file ?? file
+ for (const spec of suite.specs ?? []) {
+ for (const test of spec.tests ?? []) {
+ for (const result of test.results ?? []) {
+ const start = Date.parse(result.startTime)
+ if (Number.isFinite(start)) {
+ spans.push({
+ // The title too: a loop declares many tests on one line.
+ spec: `${spec.file ?? suiteFile}:${spec.line} ${spec.title}`,
+ start,
+ end: start + (result.duration ?? 0)
+ })
+ }
+ }
+ }
+ }
+ for (const child of suite.suites ?? []) {
+ visit(child, suiteFile)
+ }
+ }
+ for (const report of reports) {
+ for (const suite of report.suites ?? []) {
+ visit(suite)
+ }
+ }
+ return spans.sort((left, right) => left.start - right.start)
+}
+
+function specAt(at, spans) {
+ const inside = spans.findLast((span) => span.start <= at && at <= span.end)
+ if (inside) {
+ return inside.spec
+ }
+ const near = spans.findLast((span) => span.end < at && at - span.end <= AFTER_END_MS)
+ const next = spans.find((span) => span.start > at && span.start - at <= BEFORE_START_MS)
+ return (near ?? next)?.spec ?? 'unattributed'
+}
+
+const increment = (counts, key) => {
+ counts[key] = (counts[key] ?? 0) + 1
+}
+
+export function summarizeDivergence(lines, reports) {
+ const spans = testSpans(reports)
+ const summary = {
+ reports: 0,
+ paths: 0,
+ unreadable: 0,
+ byKind: {},
+ bySpec: {},
+ bySpecAndKind: {}
+ }
+ for (const line of lines) {
+ if (!line.trim()) {
+ continue
+ }
+ let report
+ try {
+ report = JSON.parse(line)
+ } catch {
+ // A line cut short by a killed app.
+ summary.unreadable++
+ continue
+ }
+ const spec = specAt(report.at, spans)
+ summary.reports++
+ increment(summary.bySpec, spec)
+ for (const path of report.paths ?? []) {
+ const kind = divergenceKind(path)
+ summary.paths++
+ increment(summary.byKind, kind)
+ summary.bySpecAndKind[spec] ??= {}
+ increment(summary.bySpecAndKind[spec], kind)
+ }
+ }
+ return summary
+}
+
+export function divergenceMarkdown(summary, title) {
+ const rows = (counts) =>
+ Object.entries(counts)
+ .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
+ .map(([key, count]) => `| ${count} | \`${key.replaceAll('|', '\\|')}\` |`)
+ const lines = [
+ `### ${title}`,
+ '',
+ `${summary.reports} reports, ${summary.paths} differences, ${summary.unreadable} unreadable lines.`,
+ ''
+ ]
+ if (summary.reports === 0) {
+ return `${lines.join('\n')}\n`
+ }
+ lines.push('| Count | Kind |', '| ---: | --- |', ...rows(summary.byKind), '')
+ for (const [spec, kinds] of Object.entries(summary.bySpecAndKind).sort()) {
+ lines.push(`${spec}: ${summary.bySpec[spec]} reports`, '')
+ lines.push('| Count | Kind |', '| ---: | --- |', ...rows(kinds), '', '', '')
+ }
+ return `${lines.join('\n')}\n`
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [logPath, summaryPath, ...reportPaths] = process.argv.slice(2)
+ const lines = existsSync(logPath) ? readFileSync(logPath, 'utf8').split('\n') : []
+ const reports = reportPaths
+ .filter((path) => existsSync(path))
+ .flatMap((path) => {
+ try {
+ return [JSON.parse(readFileSync(path, 'utf8'))]
+ } catch {
+ // A report cut short by a killed run: its divergences count as unattributed.
+ return []
+ }
+ })
+ const summary = summarizeDivergence(lines, reports)
+ writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`)
+ const markdown = divergenceMarkdown(summary, `Layout save divergence: ${basename(logPath)}`)
+ process.stdout.write(markdown)
+ if (process.env.GITHUB_STEP_SUMMARY) {
+ appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown)
+ }
+}
diff --git a/config/scripts/layout-save-divergence-summary.test.mjs b/config/scripts/layout-save-divergence-summary.test.mjs
new file mode 100644
index 000000000000..b6386578d52d
--- /dev/null
+++ b/config/scripts/layout-save-divergence-summary.test.mjs
@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest'
+import { divergenceKind, summarizeDivergence } from './layout-save-divergence-summary.mjs'
+
+describe('layout save divergence summary', () => {
+ it('counts reports by kind and by the spec running when each was written', () => {
+ const id = 'c47bcbc2-ec0c-458b-9d54-a2252e8f747a'
+ const report = {
+ suites: [
+ {
+ file: 'a.spec.ts',
+ specs: [
+ {
+ line: 8,
+ title: 'opens',
+ tests: [{ results: [{ startTime: '2026-10-19T00:00:00.000Z', duration: 1000 }] }]
+ }
+ ]
+ }
+ ]
+ }
+ const start = Date.parse('2026-10-19T00:00:00.000Z')
+ const lines = [
+ JSON.stringify({
+ at: start + 500,
+ paths: [`group ${id} changed: tabOrder`, 'workspace added']
+ }),
+ // Written after the test ended, as delayed saves are.
+ JSON.stringify({ at: start + 5000, paths: [`terminal ${id} tab-bar entry removed`] }),
+ JSON.stringify({ at: start + 60_000, paths: ['1 pane(s) rebound'] }),
+ '{"at":',
+ ''
+ ]
+ expect(summarizeDivergence(lines, [report])).toEqual({
+ reports: 3,
+ paths: 4,
+ unreadable: 1,
+ byKind: {
+ 'group changed: tabOrder': 1,
+ 'workspace added': 1,
+ 'terminal tab-bar entry removed': 1,
+ ' pane(s) rebound': 1
+ },
+ bySpec: { 'a.spec.ts:8 opens': 2, unattributed: 1 },
+ bySpecAndKind: {
+ 'a.spec.ts:8 opens': {
+ 'group changed: tabOrder': 1,
+ 'workspace added': 1,
+ 'terminal tab-bar entry removed': 1
+ },
+ unattributed: { ' pane(s) rebound': 1 }
+ }
+ })
+ expect(divergenceKind('tab picker-agent-tab changed: createdAt, terminal')).toBe(
+ 'tab changed: createdAt, terminal'
+ )
+ expect(divergenceKind('editor view /tmp/a b/c.ts changed: viewState')).toBe(
+ 'editor view changed: viewState'
+ )
+ expect(divergenceKind('file /tmp/a b/c.ts added')).toBe('file added')
+ })
+})
diff --git a/config/scripts/locale-cross-locale-key-overrides.mjs b/config/scripts/locale-cross-locale-key-overrides.mjs
index 7ea4059481ed..db056fd2b5e2 100644
--- a/config/scripts/locale-cross-locale-key-overrides.mjs
+++ b/config/scripts/locale-cross-locale-key-overrides.mjs
@@ -1,4 +1,13 @@
export const CROSS_LOCALE_KEY_OVERRIDES = {
+ // The live status describes ongoing reasoning; the composer option names the feature.
+ 'components.native-chat.status.thinking': {
+ zh: '思考中',
+ ja: '思考中',
+ ko: '생각 중',
+ fr: 'Réflexion en cours',
+ es: 'Pensando'
+ },
+ 'auto.components.feature.wall.WorkbenchAnimatedVisual.633a91e358': { zh: '思考中…' },
'auto.components.GitLabItemDialog.4168eb2c51': {
zh: '解决',
ja: '解決'
diff --git a/config/scripts/localization-extraction-change-scope.mjs b/config/scripts/localization-extraction-change-scope.mjs
index 66667ef710bd..fb9e80bcf92e 100644
--- a/config/scripts/localization-extraction-change-scope.mjs
+++ b/config/scripts/localization-extraction-change-scope.mjs
@@ -7,7 +7,7 @@ const TOOLCHAIN_FILES = new Set([
'pnpm-workspace.yaml',
'tsconfig.json',
'.npmrc',
- '.pnpmfile.cjs',
+ 'config/pnpmfile.cjs',
'.github/workflows/pr.yml'
])
diff --git a/config/scripts/mac-notarization-builder-contract.test.mjs b/config/scripts/mac-notarization-builder-contract.test.mjs
new file mode 100644
index 000000000000..a3df7b30afd6
--- /dev/null
+++ b/config/scripts/mac-notarization-builder-contract.test.mjs
@@ -0,0 +1,150 @@
+import { createRequire } from 'node:module'
+import { readFileSync } from 'node:fs'
+import { setImmediate } from 'node:timers/promises'
+import { describe, expect, it } from 'vitest'
+import { parse } from 'yaml'
+
+const builderRequire = createRequire(createRequire(import.meta.url).resolve('electron-builder'))
+const require = createRequire(builderRequire.resolve('app-builder-lib'))
+// Initialize the package root before submodules to preserve its CJS dependency order.
+const { Platform } = require('app-builder-lib')
+const { PlatformPackager } = require('app-builder-lib/out/platformPackager')
+const { ArchiveTarget } = require('app-builder-lib/out/targets/ArchiveTarget')
+const { DmgTarget } = require('dmg-builder/out/dmg')
+const { AsyncTaskManager } = require('builder-util')
+const { CancellationToken } = require('builder-util-runtime')
+const workflow = (name) => parse(readFileSync(`.github/workflows/${name}-mac-build.yml`, 'utf8'))
+
+it('the pinned builder reaches both signed slices before either real archive target passes its gate', async () => {
+ let release
+ const barrier = new Promise((resolve) => {
+ release = resolve
+ })
+ const events = []
+ const packager = {
+ platform: Platform.MAC,
+ config: {},
+ appInfo: { version: '1.0.0' },
+ platformSpecificBuildOptions: {},
+ info: {
+ cancellationToken: new CancellationToken(),
+ emitArtifactBuildStarted: async ({ arch }) => {
+ events.push(`wait:${arch}`)
+ await barrier
+ events.push(`pass:${arch}`)
+ throw new Error('STOP_BEFORE_ARTIFACT_IO')
+ }
+ },
+ expandArtifactNamePattern: (_options, extension, arch) => `fixture-${arch}.${extension}`
+ }
+ const archive = new ArchiveTarget('zip', '/unused', packager)
+ const targets = [archive, new DmgTarget(packager, '/unused')]
+ const manager = new AsyncTaskManager(packager.info.cancellationToken)
+ for (const arch of [1, 3]) {
+ events.push(`signed:${arch}`)
+ PlatformPackager.prototype.packageInDistributableFormat.call(
+ packager,
+ '/unused.app',
+ arch,
+ targets,
+ manager
+ )
+ await setImmediate()
+ }
+ expect(events.filter((event) => event.startsWith('signed:'))).toEqual(['signed:1', 'signed:3'])
+ expect(events.filter((event) => event.startsWith('wait:'))).toHaveLength(4)
+ expect(events.some((event) => event.startsWith('pass:'))).toBe(false)
+ release()
+ manager.addTask(archive.finishBuild())
+ await expect(manager.awaitTasks()).rejects.toThrow('STOP_BEFORE_ARTIFACT_IO')
+ expect(events.filter((event) => event.startsWith('pass:'))).toHaveLength(4)
+})
+
+describe('macOS build workflow policy', () => {
+ // The fork's release channel ships arm64 only, so it never takes the dual-arch overlap builder.
+ it('release builds arm64 directly on a hosted runner, keeping retries and signing secrets', () => {
+ const job = workflow('release').jobs['build-mac']
+ expect(job['runs-on']).toBe('macos-15')
+ const step = job.steps.find((step) => step.name === 'Publish release artifacts (macOS)')
+ expect(step.with.command).not.toContain('build-mac-notarization-overlap')
+ expect(step.with.command).toContain(
+ 'ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac'
+ )
+ expect(step.with.max_attempts).toBeGreaterThan(1)
+ for (const key of [
+ 'CSC_LINK',
+ 'CSC_KEY_PASSWORD',
+ 'APPLE_ID',
+ 'APPLE_APP_SPECIFIC_PASSWORD',
+ 'APPLE_TEAM_ID'
+ ]) {
+ expect(step.env[key]).toContain('secrets.')
+ }
+ })
+
+ it.each(['adhoc', 'daily', 'hourly'])(
+ '%s uses the guarded builder, preserving the serial fallback for older source refs',
+ (channel) => {
+ const job = Object.values(workflow(channel).jobs).find((job) =>
+ job.steps?.some((step) => step.with?.command?.includes('electron-builder --config'))
+ )
+ expect(job['runs-on']).toBe('blacksmith-6vcpu-macos-15')
+ const step = job.steps.find((step) =>
+ step.with?.command?.includes('electron-builder --config')
+ )
+ const command = step.with.command
+ expect(command).toContain('if [ -f config/scripts/build-mac-notarization-overlap.mjs ]')
+ expect(command).toContain(
+ `ORCA_MAC_${channel.toUpperCase()}=1 node config/scripts/build-mac-notarization-overlap.mjs`
+ )
+ expect(command).toContain(
+ `else ORCA_MAC_${channel.toUpperCase()}=1 pnpm exec electron-builder`
+ )
+ expect(step.with.max_attempts).toBeGreaterThan(1)
+ for (const key of [
+ 'CSC_LINK',
+ 'CSC_KEY_PASSWORD',
+ 'APPLE_ID',
+ 'APPLE_APP_SPECIFIC_PASSWORD',
+ 'APPLE_TEAM_ID'
+ ]) {
+ expect(step.env[key]).toContain('secrets.')
+ }
+ }
+ )
+
+ it('qualification signs the workflow commit, gates its complete template, and cannot publish', () => {
+ const source = workflow('release')
+ expect(source.on.workflow_dispatch.inputs.qualification_only.default).toBe(false)
+ expect(source.concurrency).toEqual({
+ group:
+ "release-mac-build-${{ inputs.tag }}${{ inputs.qualification_only && '-qualification' || '' }}",
+ 'cancel-in-progress': false
+ })
+ expect(source.jobs['qualification-template'].if).toBe('inputs.qualification_only')
+ expect(source.jobs['qualification-template'].with).toEqual({
+ ref: '${{ github.sha }}',
+ build_template: true
+ })
+ const job = source.jobs['build-mac']
+ expect(job.needs).toBe('qualification-template')
+ expect(job.if).toContain('!cancelled()')
+ expect(job.if).toContain("needs.qualification-template.result == 'success'")
+ const publish = job.steps.find((step) => step.name === 'Publish release artifacts (macOS)')
+ expect(publish.env.ORCA_MAC_PUBLISH).toBe(
+ "${{ inputs.qualification_only && 'never' || 'always' }}"
+ )
+ expect(publish.with.command.match(/--publish "\$ORCA_MAC_PUBLISH"/g)).toHaveLength(1)
+ for (const name of [
+ 'Abort if the parent release-cut run was cancelled',
+ 'Verify release remains draft after artifact upload'
+ ]) {
+ expect(job.steps.find((step) => step.name === name).if).toBe(
+ '${{ !inputs.qualification_only }}'
+ )
+ }
+ const upload = job.steps.find((step) => step.name === 'Upload qualification artifacts')
+ expect(upload.if).toBe('inputs.qualification_only')
+ expect(upload.with['if-no-files-found']).toBe('error')
+ })
+})
diff --git a/config/scripts/macos-shipit-scheduling.cjs b/config/scripts/macos-shipit-scheduling.cjs
new file mode 100644
index 000000000000..027c7b171e89
--- /dev/null
+++ b/config/scripts/macos-shipit-scheduling.cjs
@@ -0,0 +1,33 @@
+const { execFileSync } = require('node:child_process')
+const { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync } = require('node:fs')
+const { tmpdir } = require('node:os')
+const { join, resolve } = require('node:path')
+
+const SHIPIT_SCHEDULING_ADDON = 'orca_shipit_scheduling.node'
+
+function buildMacShipItScheduling(resourcesDir, arch) {
+ if (process.platform !== 'darwin' || !['x64', 'arm64'].includes(arch)) {
+ throw new Error(`Cannot build macOS installer scheduling for ${process.platform}/${arch}`)
+ }
+ const root = resolve(__dirname, '../..')
+ const workDir = mkdtempSync(join(tmpdir(), `orca-shipit-scheduling-${arch}-`))
+ try {
+ cpSync(join(root, 'native/macos-shipit-scheduling'), workDir, { recursive: true })
+ execFileSync(
+ process.execPath,
+ [require.resolve('node-gyp/bin/node-gyp.js'), 'rebuild', `--arch=${arch}`],
+ { cwd: workDir, stdio: 'inherit' }
+ )
+ const built = join(workDir, 'build/Release', SHIPIT_SCHEDULING_ADDON)
+ const expectedArch = arch === 'x64' ? 'x86_64' : 'arm64'
+ execFileSync('/usr/bin/lipo', [built, '-verify_arch', expectedArch])
+ mkdirSync(resourcesDir, { recursive: true })
+ const target = join(resourcesDir, SHIPIT_SCHEDULING_ADDON)
+ copyFileSync(built, target)
+ return target
+ } finally {
+ rmSync(workDir, { recursive: true, force: true })
+ }
+}
+
+module.exports = { buildMacShipItScheduling, SHIPIT_SCHEDULING_ADDON }
diff --git a/config/scripts/macos-shipit-scheduling.test.mjs b/config/scripts/macos-shipit-scheduling.test.mjs
new file mode 100644
index 000000000000..38025b3201ac
--- /dev/null
+++ b/config/scripts/macos-shipit-scheduling.test.mjs
@@ -0,0 +1,92 @@
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { homedir, tmpdir } from 'node:os'
+import { dirname, join, resolve } from 'node:path'
+import { describe, expect, it } from 'vitest'
+
+const require = createRequire(import.meta.url)
+const { buildMacShipItScheduling } = require('./macos-shipit-scheduling.cjs')
+const root = resolve(import.meta.dirname, '../..')
+
+describe.skipIf(process.platform !== 'darwin')('native ShipIt scheduling', () => {
+ it('changes only ProcessType in the real Electron job, retaining native install and retry settings', () => {
+ const work = mkdtempSync(join(tmpdir(), 'orca-shipit-contract-'))
+ const bundleId = `com.orca.shipit-contract.${work.split('-').at(-1).toLowerCase()}`
+ try {
+ for (const arch of ['x64', 'arm64']) {
+ buildMacShipItScheduling(join(work, arch), arch)
+ }
+ const addon = join(work, process.arch, 'orca_shipit_scheduling.node')
+ // Loading outside Electron must fail safely when Squirrel is absent.
+ expect(require(addon).configure()).toBe(false)
+ const compatibilityProbe = join(work, 'unsupported-api')
+ execFileSync('/usr/bin/clang', [
+ '-fobjc-arc',
+ '-framework',
+ 'Foundation',
+ join(root, 'native/macos-shipit-scheduling/probe-unsupported-api.m'),
+ '-o',
+ compatibilityProbe
+ ])
+ for (const mode of [
+ 'return-type',
+ 'object',
+ 'payload',
+ 'exception',
+ 'thrown-object',
+ 'supported'
+ ]) {
+ execFileSync(compatibilityProbe, [addon, mode], { timeout: 20_000 })
+ }
+ const contents = join(work, 'Probe.app', 'Contents')
+ const executable = join(contents, 'MacOS', 'probe')
+ mkdirSync(dirname(executable), { recursive: true })
+ writeFileSync(
+ join(contents, 'Info.plist'),
+ `
+ CFBundleIdentifier${bundleId}
+ CFBundleExecutableprobe`
+ )
+ execFileSync('/usr/bin/clang', [
+ '-fobjc-arc',
+ '-framework',
+ 'Foundation',
+ join(root, 'native/macos-shipit-scheduling/probe-job-dictionary.m'),
+ '-o',
+ executable
+ ])
+ const electron = require('electron')
+ expect(
+ execFileSync(electron, ['-e', 'console.log(require(process.argv[1]).configure())', addon], {
+ encoding: 'utf8',
+ timeout: 20_000,
+ env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1', ELECTRON_RUN_AS_NODE: '1' }
+ }).trim()
+ ).toBe('true')
+ const frameworks = resolve(dirname(electron), '../Frameworks')
+ const job = JSON.parse(
+ execFileSync(executable, [frameworks, addon], {
+ encoding: 'utf8',
+ timeout: 20_000,
+ env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
+ })
+ )
+ expect(job.ProcessType).toBe('Interactive')
+ expect(job.Label).toBe(`${bundleId}.ShipIt`)
+ expect(job.ProgramArguments).toEqual([
+ join(frameworks, 'Squirrel.framework/Resources/ShipIt'),
+ job.Label,
+ join(homedir(), 'Library/Caches', job.Label, 'ShipItState.plist')
+ ])
+ expect(job.KeepAlive).toEqual({ SuccessfulExit: false })
+ expect(job.MachServices).toEqual({ [job.Label]: true })
+ } finally {
+ rmSync(work, { recursive: true, force: true })
+ rmSync(join(homedir(), 'Library/Caches', `${bundleId}.ShipIt`), {
+ recursive: true,
+ force: true
+ })
+ }
+ }, 90_000)
+})
diff --git a/config/scripts/mobile-web-bundle-manifest.mjs b/config/scripts/mobile-web-bundle-manifest.mjs
index 2e9888f69c79..3951d0fc1b6c 100644
--- a/config/scripts/mobile-web-bundle-manifest.mjs
+++ b/config/scripts/mobile-web-bundle-manifest.mjs
@@ -1,8 +1,11 @@
import { createHash } from 'node:crypto'
import { readFile, mkdir, rm, writeFile } from 'node:fs/promises'
+import { createRequire } from 'node:module'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
+const { darkBuildMarkers } = createRequire(import.meta.url)('../dark-build-switches.cjs')
+
const projectDir = fileURLToPath(new URL('../..', import.meta.url))
export const MOBILE_WEB_BUNDLE_SCHEMA_VERSION = 1
@@ -118,7 +121,8 @@ export async function writeMobileWebBundleTree({
protocolWindow,
// Empty for a tree with no route list at all: a shell reading it finds no screen listed and
// renders every route natively.
- routes = []
+ routes = [],
+ env = process.env
}) {
const assets = written
.map(({ path, sha256, byteLength, contentType }) => ({ path, sha256, byteLength, contentType }))
@@ -143,5 +147,9 @@ export async function writeMobileWebBundleTree({
await writeFile(join(outDir, asset.path), asset.bytes)
}
await writeFile(join(outDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8')
+ // Outside the manifest, so the page's bytes and buildId stay what the switch compiled.
+ for (const marker of darkBuildMarkers(env)) {
+ await writeFile(join(outDir, marker.fileName), marker.text, 'utf8')
+ }
return { manifest, outDir }
}
diff --git a/config/scripts/mobile-web-bundle-manifest.test.mjs b/config/scripts/mobile-web-bundle-manifest.test.mjs
index 81c447878a8a..b6bf8f7e65d6 100644
--- a/config/scripts/mobile-web-bundle-manifest.test.mjs
+++ b/config/scripts/mobile-web-bundle-manifest.test.mjs
@@ -1,3 +1,4 @@
+import { existsSync } from 'node:fs'
import { mkdtemp, readFile, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -87,4 +88,34 @@ describe('writeMobileWebBundleTree', () => {
await rm(outDir, { recursive: true, force: true })
}
})
+
+ it('marks a dark build outside the manifest, so its buildId is the same tree’s', async () => {
+ const outDir = await mkdtemp(join(tmpdir(), 'mobile-web-bundle-manifest-'))
+ try {
+ const bytes = Buffer.from('\n', 'utf8')
+ const tree = (env) =>
+ writeMobileWebBundleTree({
+ outDir,
+ written: [
+ {
+ path: 'index.html',
+ bytes,
+ sha256: sha256Hex(bytes),
+ byteLength: bytes.byteLength,
+ contentType: contentTypeForExtension('html')
+ }
+ ],
+ desktopVersion: '1.0.0',
+ protocolWindow: { runtimeProtocolVersion: 3, minCompatibleRuntimeProtocolVersion: 3 },
+ env
+ })
+ const dark = await tree({ ORCA_SIDEBAR_INPUTS: 'main' })
+ expect(existsSync(join(outDir, 'ORCA_SIDEBAR_INPUTS_MAIN'))).toBe(true)
+ const off = await tree({})
+ expect(existsSync(join(outDir, 'ORCA_SIDEBAR_INPUTS_MAIN'))).toBe(false)
+ expect(dark.manifest.buildId).toBe(off.manifest.buildId)
+ } finally {
+ await rm(outDir, { recursive: true, force: true })
+ }
+ })
})
diff --git a/config/scripts/mobile-web-page-routes.mjs b/config/scripts/mobile-web-page-routes.mjs
index 9d0f912f9fcb..e75ea674f17e 100644
--- a/config/scripts/mobile-web-page-routes.mjs
+++ b/config/scripts/mobile-web-page-routes.mjs
@@ -145,7 +145,7 @@ export const MOBILE_WEB_PAGE_ROUTES = [
// is why the fourth verb this list carried is gone). Ruling 4's degradation is retired with them:
// the page no longer falls back to the vendored module's denied microphone.
//
- // `externalNavigation` is the one optional grant in this list, and it is C8.1's. The HTML preview
+ // `externalNavigation` is the first optional grant in this list, and it is C8.1's. The HTML preview
// renders an agent's artifact in a sealed frame, and a tap on a link inside it becomes a top-frame
// navigation only the shell can cancel and open. Without the grant the preview renders the
// artifact with its links as text: the document paints, the Preview/Source toggle works, and
@@ -170,6 +170,8 @@ export const MOBILE_WEB_PAGE_ROUTES = [
'native.audio.read',
'native.audio.stop'
],
- optionalGrants: ['externalNavigation']
+ // `native.media.name` gives a file the composer attaches its real name; an older shell lacks it
+ // and the page names the file from its type, so the attach still works.
+ optionalGrants: ['externalNavigation', 'native.media.name']
}
]
diff --git a/config/scripts/native-build-entry-probe.cjs b/config/scripts/native-build-entry-probe.cjs
new file mode 100644
index 000000000000..f628f8b7a268
--- /dev/null
+++ b/config/scripts/native-build-entry-probe.cjs
@@ -0,0 +1,53 @@
+'use strict'
+
+// Loads one native build dir in a fresh Node or Electron process: ` this.cjs `.
+// Why a child: a failed addon load can poison the process that attempted it.
+
+const { accessSync, constants, existsSync } = require('node:fs')
+const { join } = require('node:path')
+const { assertNodePtyJobOwnership } = require('./node-pty-job-ownership.cjs')
+const { assertWindowsProcessTreeCreationTime } = require('./windows-process-tree-creation-time.cjs')
+const { nodePtyNativeModuleName } = require('./native-build-entry.cjs')
+
+const NODE_PTY_CONPTY_RUNTIME_FILES = ['conpty.dll', 'OpenConsole.exe']
+
+function probeNativeBuildDir(moduleName, dir) {
+ if (moduleName === 'node-pty') {
+ const nativeName = nodePtyNativeModuleName()
+ const addonPath = join(dir, `${nativeName}.node`)
+ const native = { dir, module: require(addonPath) }
+ assertNodePtyJobOwnership({ nativeName, native, addonPath })
+ if (process.platform === 'darwin') {
+ accessSync(join(dir, 'spawn-helper'), constants.X_OK)
+ }
+ if (nativeName === 'conpty') {
+ for (const filename of NODE_PTY_CONPTY_RUNTIME_FILES) {
+ if (!existsSync(join(dir, 'conpty', filename))) {
+ throw new Error(
+ `node-pty ConPTY runtime file is missing: ${join(dir, 'conpty', filename)}`
+ )
+ }
+ }
+ }
+ return
+ }
+ if (moduleName === '@vscode/windows-process-tree') {
+ assertWindowsProcessTreeCreationTime({
+ module: require(join(dir, 'windows_process_tree.node'))
+ })
+ return
+ }
+ throw new Error(`no probe for ${moduleName}`)
+}
+
+if (require.main === module) {
+ const [moduleName, dir] = process.argv.slice(2)
+ try {
+ probeNativeBuildDir(moduleName, dir)
+ } catch (error) {
+ console.error(`${moduleName}: ${error instanceof Error ? error.message : String(error)}`)
+ process.exit(1)
+ }
+}
+
+module.exports = { probeNativeBuildDir }
diff --git a/config/scripts/native-build-entry.cjs b/config/scripts/native-build-entry.cjs
new file mode 100644
index 000000000000..c2e1a11b9bb2
--- /dev/null
+++ b/config/scripts/native-build-entry.cjs
@@ -0,0 +1,141 @@
+'use strict'
+
+const { existsSync, realpathSync } = require('node:fs')
+const { release } = require('node:os')
+const { join, relative, isAbsolute } = require('node:path')
+
+/**
+ * Per-ABI native build entries inside a package dir that other worktrees share.
+ *
+ * pnpm's global virtual store gives every worktree the same physical package dir, so an
+ * in-place `build/Release` rebuild in one worktree replaces the binary another is running.
+ * A shared package instead gets one immutable `build/` per ABI, published atomically
+ * and never rebuilt. The patched loaders (config/patches/node-pty@1.1.0.patch and
+ * @vscode__windows-process-tree@0.8.0.patch) try that dir before build/Release, and must
+ * compute the same name -- `native-build-entry.test.mjs` holds them to it.
+ */
+
+const ENTRY_PREFIX = 'orca-'
+
+/** How each shared native package is built and which runtime its binary is tied to. */
+const NATIVE_BUILD_PACKAGES = {
+ // Both are node-addon-api (N-API): one binary loads in Node and Electron alike.
+ 'node-pty': { abi: 'napi' },
+ '@vscode/windows-process-tree': { abi: 'napi' }
+}
+
+/**
+ * The entry dir name for `moduleName` on this runtime. N-API binaries are keyed by
+ * platform-arch only; anything else is tied to NODE_MODULE_VERSION, which already differs
+ * between Node and every Electron major.
+ */
+function nativeBuildEntryName(
+ moduleName,
+ { platform = process.platform, arch = process.arch, modules = process.versions.modules } = {}
+) {
+ const descriptor = NATIVE_BUILD_PACKAGES[moduleName]
+ if (!descriptor) {
+ throw new Error(`${moduleName} has no shared native build descriptor`)
+ }
+ const abi = descriptor.abi === 'napi' ? 'napi' : `abi${modules}`
+ return `${ENTRY_PREFIX}${abi}-${platform}-${arch}`
+}
+
+function nativeBuildEntryDir(packageDir, moduleName, target) {
+ return join(packageDir, 'build', nativeBuildEntryName(moduleName, target))
+}
+
+/** The dir the patched loader reads: the published entry when there is one, else build/Release. */
+function loadedNativeBuildDir(packageDir, moduleName, target) {
+ const entryDir = nativeBuildEntryDir(packageDir, moduleName, target)
+ return existsSync(entryDir) ? entryDir : join(packageDir, 'build', 'Release')
+}
+
+/** node-pty's own choice: ConPTY from Windows build 18309, winpty below it. */
+function nodePtyNativeModuleName(platform = process.platform, osRelease = release()) {
+ if (platform !== 'win32') {
+ return 'pty'
+ }
+ const match = /(\d+)\.(\d+)\.(\d+)/.exec(osRelease)
+ return match && Number.parseInt(match[3], 10) >= 18309 ? 'conpty' : 'pty'
+}
+
+/** Every published, staging and quarantined entry name, for packaging to strip. */
+function isNativeBuildEntryName(name) {
+ return name.startsWith(ENTRY_PREFIX)
+}
+
+/** electron-builder ignore globs for in-flight staging and quarantine trees (shared-tree-publish.mjs). */
+const TRANSIENT_NATIVE_BUILD_ENTRY_FILTER = [`!build/${ENTRY_PREFIX}*.{staging,unusable}-*{,/**/*}`]
+
+/**
+ * Whether `packageDir` physically lives outside this project, i.e. another worktree can load it.
+ * Why realpath: under the global virtual store node_modules/ is a link into the store.
+ */
+function isSharedPackageDir(packageDir, projectDir) {
+ try {
+ const rel = relative(realpathSync(projectDir), realpathSync(packageDir))
+ return rel.startsWith('..') || isAbsolute(rel)
+ } catch {
+ return false
+ }
+}
+
+/** Whether `moduleName` is installed in a package dir other worktrees share and can be built beside. */
+function isSharedNativePackage(moduleName, projectDir) {
+ if (!NATIVE_BUILD_PACKAGES[moduleName]) {
+ return false
+ }
+ try {
+ return isSharedPackageDir(resolveNativePackageDir(moduleName, projectDir), projectDir)
+ } catch {
+ return false // Not installed: an optional dependency off its platform.
+ }
+}
+
+/** The project's own top-level link, the same path every Orca native script already reads. */
+function resolveNativePackageDir(moduleName, projectDir) {
+ const packageDir = join(projectDir, 'node_modules', ...moduleName.split('/'))
+ if (!existsSync(join(packageDir, 'package.json'))) {
+ throw new Error(`${moduleName} is not installed in ${projectDir}`)
+ }
+ return packageDir
+}
+
+/** The files a node-pty source build must leave in whichever dir the loader reads. */
+function nodePtyBuildArtifactPaths(nodePtyDir, platform = process.platform) {
+ const buildDir = loadedNativeBuildDir(nodePtyDir, 'node-pty')
+ if (platform === 'win32') {
+ return ['conpty.node', 'conpty/conpty.dll', 'conpty/OpenConsole.exe'].map((file) =>
+ join(buildDir, file)
+ )
+ }
+ // Why: node-pty only builds spawn-helper on macOS; Linux builds only pty.node.
+ return ['pty.node', ...(platform === 'darwin' ? ['spawn-helper'] : [])].map((file) =>
+ join(buildDir, file)
+ )
+}
+
+/** A `loadNativeModule` dir holding Orca's source build rather than an upstream prebuild. */
+function isSourceBuildDir(nativeDir) {
+ if (typeof nativeDir !== 'string') {
+ return false
+ }
+ const dir = nativeDir.replace(/\\/g, '/')
+ return dir.includes('build/Release/') || dir.includes(`build/${ENTRY_PREFIX}`)
+}
+
+module.exports = {
+ NATIVE_BUILD_PACKAGES,
+ TRANSIENT_NATIVE_BUILD_ENTRY_FILTER,
+ isNativeBuildEntryName,
+ isSharedNativePackage,
+ isSharedPackageDir,
+ isSourceBuildDir,
+ loadedNativeBuildDir,
+ nativeBuildEntryDir,
+ nativeBuildEntryName,
+ nodePtyBuildArtifactPaths,
+ nodePtyNativeModuleName,
+ resolveNativePackageDir
+}
diff --git a/config/scripts/native-build-entry.test.mjs b/config/scripts/native-build-entry.test.mjs
new file mode 100644
index 000000000000..2aea450cdfd9
--- /dev/null
+++ b/config/scripts/native-build-entry.test.mjs
@@ -0,0 +1,188 @@
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+
+const require = createRequire(import.meta.url)
+const {
+ isNativeBuildEntryName,
+ isSharedPackageDir,
+ isSourceBuildDir,
+ loadedNativeBuildDir,
+ nativeBuildEntryName,
+ nodePtyNativeModuleName
+} = require('./native-build-entry.cjs')
+const { adoptPackagedNativeBuildEntries } = require('../packaged-native-build-entries.cjs')
+
+const PATCHES = path.join(import.meta.dirname, '..', 'patches')
+const roots = []
+
+afterEach(() => {
+ while (roots.length > 0) {
+ rmSync(roots.pop(), { recursive: true, force: true })
+ }
+})
+
+function makeRoot() {
+ const root = mkdtempSync(path.join(tmpdir(), 'orca-native-entry-'))
+ roots.push(root)
+ return root
+}
+
+function addedPatchLine(patchFile, pattern) {
+ const line = readFileSync(path.join(PATCHES, patchFile), 'utf8')
+ .split('\n')
+ .find((candidate) => candidate.startsWith('+') && pattern.test(candidate))
+ expect(line, `${patchFile} has no added line matching ${pattern}`).toBeDefined()
+ return line
+}
+
+describe('nativeBuildEntryName', () => {
+ it('keys an N-API addon by platform and arch only, so Node and Electron share it', () => {
+ const node = nativeBuildEntryName('node-pty', {
+ platform: 'darwin',
+ arch: 'arm64',
+ modules: '137'
+ })
+ const electron = nativeBuildEntryName('node-pty', {
+ platform: 'darwin',
+ arch: 'arm64',
+ modules: '148'
+ })
+ expect(node).toBe('orca-napi-darwin-arm64')
+ expect(electron).toBe(node)
+ expect(
+ nativeBuildEntryName('@vscode/windows-process-tree', { platform: 'win32', arch: 'x64' })
+ ).toBe('orca-napi-win32-x64')
+ })
+
+ it('separates arches and refuses a package with no descriptor', () => {
+ expect(nativeBuildEntryName('node-pty', { platform: 'darwin', arch: 'x64' })).not.toBe(
+ nativeBuildEntryName('node-pty', { platform: 'darwin', arch: 'arm64' })
+ )
+ expect(() => nativeBuildEntryName('cpu-features')).toThrow(/no shared native build/)
+ })
+
+ // The patched loaders cannot require this module, so they spell the name out themselves.
+ it('matches the dir the patched node-pty loader tries first', () => {
+ const line = addedPatchLine('node-pty@1.1.0.patch', /var dirs = \[/)
+ const firstDir = /\[([^,]+),/.exec(line)[1]
+ const evaluate = new Function('process', `return ${firstDir}`)
+ for (const [platform, arch] of [
+ ['darwin', 'arm64'],
+ ['linux', 'x64'],
+ ['win32', 'arm64']
+ ]) {
+ expect(evaluate({ platform, arch })).toBe(
+ `build/${nativeBuildEntryName('node-pty', { platform, arch })}`
+ )
+ }
+ })
+
+ it('matches the dir the patched windows-process-tree loader prefers', () => {
+ const line = addedPatchLine('@vscode__windows-process-tree@0.8.0.patch', /orcaNativeBuild = /)
+ const template = /= `(.*)`;/.exec(line)[1]
+ for (const arch of ['x64', 'arm64']) {
+ const resolved = template.replace('${__dirname}', '/pkg/lib').replace('${process.arch}', arch)
+ expect(path.posix.dirname(path.posix.normalize(resolved))).toBe(
+ `/pkg/build/${nativeBuildEntryName('@vscode/windows-process-tree', { platform: 'win32', arch })}`
+ )
+ }
+ })
+})
+
+describe('isSharedPackageDir', () => {
+ it('treats a package linked in from outside the project as shared', () => {
+ const root = makeRoot()
+ const store = path.join(root, 'store', 'node-pty')
+ const project = path.join(root, 'project')
+ mkdirSync(store, { recursive: true })
+ mkdirSync(path.join(project, 'node_modules'), { recursive: true })
+ symlinkSync(store, path.join(project, 'node_modules', 'node-pty'), 'junction')
+ expect(isSharedPackageDir(path.join(project, 'node_modules', 'node-pty'), project)).toBe(true)
+ })
+
+ it('treats the per-project virtual store as private', () => {
+ const root = makeRoot()
+ const real = path.join(
+ root,
+ 'node_modules',
+ '.pnpm',
+ 'node-pty@1.1.0',
+ 'node_modules',
+ 'node-pty'
+ )
+ mkdirSync(real, { recursive: true })
+ symlinkSync(real, path.join(root, 'node_modules', 'node-pty'), 'junction')
+ expect(isSharedPackageDir(path.join(root, 'node_modules', 'node-pty'), root)).toBe(false)
+ })
+
+ it('treats a missing package as private', () => {
+ const root = makeRoot()
+ expect(isSharedPackageDir(path.join(root, 'nope'), root)).toBe(false)
+ })
+})
+
+describe('loadedNativeBuildDir', () => {
+ it('prefers a published entry and otherwise reads build/Release', () => {
+ const root = makeRoot()
+ const target = { platform: 'linux', arch: 'x64' }
+ expect(loadedNativeBuildDir(root, 'node-pty', target)).toBe(path.join(root, 'build', 'Release'))
+ mkdirSync(path.join(root, 'build', 'orca-napi-linux-x64'), { recursive: true })
+ expect(loadedNativeBuildDir(root, 'node-pty', target)).toBe(
+ path.join(root, 'build', 'orca-napi-linux-x64')
+ )
+ })
+})
+
+describe('isSourceBuildDir', () => {
+ it('accepts build/Release and a published entry, never a prebuild', () => {
+ expect(isSourceBuildDir('../build/Release/')).toBe(true)
+ expect(isSourceBuildDir('..\\build\\orca-napi-win32-x64\\')).toBe(true)
+ expect(isSourceBuildDir('../prebuilds/darwin-arm64/')).toBe(false)
+ expect(isSourceBuildDir(undefined)).toBe(false)
+ })
+})
+
+describe('nodePtyNativeModuleName', () => {
+ it('follows node-pty: ConPTY from Windows build 18309', () => {
+ expect(nodePtyNativeModuleName('darwin', '25.3.0')).toBe('pty')
+ expect(nodePtyNativeModuleName('win32', '10.0.26100')).toBe('conpty')
+ expect(nodePtyNativeModuleName('win32', '10.0.17763')).toBe('pty')
+ })
+})
+
+describe('adoptPackagedNativeBuildEntries', () => {
+ function packagedNodePty(root, dirs) {
+ const buildDir = path.join(root, 'node_modules', 'node-pty', 'build')
+ for (const [dir, payload] of Object.entries(dirs)) {
+ mkdirSync(path.join(buildDir, dir), { recursive: true })
+ writeFileSync(path.join(buildDir, dir, 'pty.node'), payload)
+ }
+ return buildDir
+ }
+
+ it('ships the target entry as build/Release and drops every other entry', () => {
+ const root = makeRoot()
+ const buildDir = packagedNodePty(root, {
+ Release: 'pnpm-install-build',
+ 'orca-napi-darwin-arm64': 'entry-arm64',
+ 'orca-napi-darwin-x64': 'entry-x64',
+ 'orca-napi-darwin-arm64.staging-1-x': 'partial'
+ })
+ adoptPackagedNativeBuildEntries(root, 'darwin', 'x64')
+ expect(readFileSync(path.join(buildDir, 'Release', 'pty.node'), 'utf8')).toBe('entry-x64')
+ expect(
+ ['Release', 'orca-napi-darwin-arm64'].map((name) => isNativeBuildEntryName(name))
+ ).toEqual([false, true])
+ expect(require('node:fs').readdirSync(buildDir)).toEqual(['Release'])
+ })
+
+ it('leaves a CI layout with no entries untouched', () => {
+ const root = makeRoot()
+ const buildDir = packagedNodePty(root, { Release: 'ci-build' })
+ adoptPackagedNativeBuildEntries(root, 'linux', 'x64')
+ expect(readFileSync(path.join(buildDir, 'Release', 'pty.node'), 'utf8')).toBe('ci-build')
+ })
+})
diff --git a/config/scripts/node-server-change-scope.mjs b/config/scripts/node-server-change-scope.mjs
index 28292fc29c55..80067c7d43b6 100644
--- a/config/scripts/node-server-change-scope.mjs
+++ b/config/scripts/node-server-change-scope.mjs
@@ -34,7 +34,7 @@ const ALWAYS_FILES = new Set([
'pnpm-lock.yaml',
'pnpm-workspace.yaml',
'.npmrc',
- '.pnpmfile.cjs',
+ 'config/pnpmfile.cjs',
'tsconfig.json',
'.github/workflows/node-server-tests.yml',
'config/scripts/node-server-change-scope.mjs',
diff --git a/config/scripts/node-server-change-scope.test.mjs b/config/scripts/node-server-change-scope.test.mjs
index 00f2abcd414c..5e717890c58f 100644
--- a/config/scripts/node-server-change-scope.test.mjs
+++ b/config/scripts/node-server-change-scope.test.mjs
@@ -571,11 +571,11 @@ it('keeps every platform job and runs them when detection is skipped or fails',
expect(workflow.jobs.persistence.strategy.matrix.os).toContain('needs.changes.outputs.runners')
for (const jobName of ['linux_glibc_floor', 'linux_musl']) {
const job = workflow.jobs[jobName]
- expect(job.needs).toEqual(['changes', 'persistence', 'mobile_web_bundle'])
+ expect(job.needs).toEqual(['changes', 'mobile_web_bundle'])
// The containers never install mobile/, and orcad refuses to build without the page.
expect(job.steps.some((step) => step.with?.name === 'orcad-mobile-web-bundle')).toBe(true)
expect(job.if).toContain("needs.mobile_web_bundle.result == 'success'")
- expect(job.if).toContain("needs.persistence.result == 'success'")
+ expect(job.if).not.toContain('needs.persistence')
expect(job.if).toContain("needs.changes.outputs.qualification != 'false'")
expect(job.if).toContain("needs.changes.outputs.should_run != 'false'")
expect(job.strategy.matrix.os).toEqual(['ubuntu-22.04', 'ubuntu-24.04-arm'])
diff --git a/config/scripts/node-server-native-compat-scope.test.mjs b/config/scripts/node-server-native-compat-scope.test.mjs
index d2fa11118114..5792ee908989 100644
--- a/config/scripts/node-server-native-compat-scope.test.mjs
+++ b/config/scripts/node-server-native-compat-scope.test.mjs
@@ -327,7 +327,7 @@ it('builds server glibc slots on glibc 2.28 and the compat slot on glibc 2.17 (d
expect(floor.env).toMatchObject({ CC: 'gcc', CXX: 'g++' })
const compat = workflow.jobs.linux_glibc217_compat
- expect(compat.needs).toEqual(['changes', 'persistence'])
+ expect(compat.needs).toBe('changes')
// Only the floor builds orcad, so only it waits for the page.
expect(compat.if).toContain(
floor.if.slice(0, -3).replace(" && needs.mobile_web_bundle.result == 'success'", '')
@@ -366,15 +366,19 @@ it.each([
{ expected: true, native: 'false', event: 'workflow_dispatch' },
{ expected: true, native: 'false', template: true },
{ expected: true, native: 'false', ref: 'other-commit' },
- { expected: false, native: 'true', persistence: 'failure' },
- { expected: false, native: 'true', persistence: 'cancelled' },
- { expected: false, native: 'true', persistence: 'skipped' },
+ { expected: true, native: 'true', persistence: 'failure' },
+ { expected: true, native: 'true', persistence: 'cancelled' },
+ { expected: true, native: 'true', persistence: 'skipped' },
+ { expected: false, native: 'true', draft: true },
{ expected: false, native: 'true', qualification: 'false' },
{ expected: false, native: 'true', shouldRun: 'false' },
{ expected: false, native: 'true', cancelled: true }
])('evaluates the hosted native job guard with %j', (scenario) => {
const context = {
- github: { event_name: scenario.event ?? 'pull_request' },
+ github: {
+ event_name: scenario.event ?? 'pull_request',
+ event: { pull_request: { draft: scenario.draft ?? false } }
+ },
inputs: { build_template: scenario.template ?? false, ref: scenario.ref ?? '' },
needs: {
persistence: { result: scenario.persistence ?? 'success' },
diff --git a/config/scripts/node-server-release-parallelism.test.mjs b/config/scripts/node-server-release-parallelism.test.mjs
new file mode 100644
index 000000000000..480b81f405ed
--- /dev/null
+++ b/config/scripts/node-server-release-parallelism.test.mjs
@@ -0,0 +1,113 @@
+import { readFileSync } from 'node:fs'
+import { runInNewContext } from 'node:vm'
+import { expect, it } from 'vitest'
+import { parse } from 'yaml'
+import { compileAggregateExpression } from './pr-aggregate-gate-context.mjs'
+
+const workflow = parse(readFileSync('.github/workflows/node-server-tests.yml', 'utf8'))
+const persistence = workflow.jobs.persistence
+const shouldPersist = compileAggregateExpression(persistence.if, ['needs', 'github'])
+
+function context({ qualification, producer = 'success', event = 'pull_request', draft = false }) {
+ return {
+ github: { event_name: event, event: { pull_request: { draft } } },
+ needs: {
+ changes: { outputs: { qualification } },
+ mobile_web_bundle: { result: producer }
+ }
+ }
+}
+
+it.each([
+ { qualification: 'true', producer: 'success', expected: true },
+ { qualification: '', producer: 'success', expected: true },
+ { qualification: 'true', producer: 'failure', expected: false },
+ { qualification: 'true', producer: 'cancelled', expected: false },
+ { qualification: 'true', producer: 'skipped', expected: false },
+ { qualification: '', producer: 'skipped', expected: false },
+ { qualification: 'false', producer: 'skipped', expected: true },
+ { qualification: 'false', producer: 'skipped', draft: true, expected: false },
+ { producer: 'success', event: 'workflow_dispatch', expected: true },
+ { producer: 'success', event: 'schedule', expected: true }
+])('preserves mobile producer failure and test-only behavior: %j', (scenario) => {
+ const ctx = context(scenario)
+ expect(persistence.needs).toEqual(['changes', 'mobile_web_bundle'])
+ expect(shouldPersist(ctx).coerceString()).toBe(String(scenario.expected))
+ expect(shouldPersist(ctx, true).coerceString()).toBe('false')
+})
+
+it('uses the shared page for qualification and builds locally only when its producer is skipped', () => {
+ const download = persistence.steps.find((step) => step.with?.name === 'orcad-mobile-web-bundle')
+ const build = persistence.steps.find((step) => step.uses?.includes('build-mobile-web-bundle'))
+ const serverBuild = persistence.steps.find((step) => step.run === 'pnpm build:orcad')
+ expect(download.uses).toBe('actions/download-artifact@v8')
+ expect(download.with.path).toBe('out/mobile-web')
+ expect(persistence.steps.indexOf(download)).toBeLessThan(persistence.steps.indexOf(serverBuild))
+ expect(persistence.steps.indexOf(build)).toBeLessThan(persistence.steps.indexOf(serverBuild))
+ for (const qualification of ['true', '', undefined, 'false']) {
+ const ctx = context({ qualification })
+ expect(compileAggregateExpression(download.if)(ctx).coerceString()).toBe(
+ String(qualification !== 'false')
+ )
+ expect(compileAggregateExpression(build.if)(ctx).coerceString()).toBe(
+ String(qualification === 'false')
+ )
+ }
+ expect(serverBuild['continue-on-error']).toBeUndefined()
+})
+
+it('starts independent Linux qualification without waiting for another platform', () => {
+ for (const name of ['linux_glibc_floor', 'linux_glibc217_compat', 'linux_musl']) {
+ const job = workflow.jobs[name]
+ expect([job.needs].flat()).not.toContain('persistence')
+ expect(job.if).not.toContain('needs.persistence')
+ const evaluate = compileAggregateExpression(job.if, ['needs', 'github', 'inputs'])
+ const ctx = { ...context({ qualification: 'true' }), inputs: { build_template: true } }
+ expect(evaluate(ctx).coerceString()).toBe('true')
+ expect(evaluate(ctx, true).coerceString()).toBe('false')
+ const draft = {
+ ...context({ qualification: '', draft: true, producer: 'skipped' }),
+ inputs: { build_template: false }
+ }
+ expect(evaluate(draft).coerceString()).toBe('false')
+ }
+})
+
+it('requires every platform and compatibility result before publishing a desktop template', () => {
+ const template = workflow.jobs.desktop_template
+ const required = [
+ 'persistence',
+ 'mobile_web_bundle',
+ 'linux_glibc_floor',
+ 'linux_glibc217_compat',
+ 'linux_musl'
+ ]
+ expect(template.needs).toEqual(required)
+ const evaluate = compileAggregateExpression(template.if, ['needs', 'inputs'])
+ const needs = Object.fromEntries(required.map((name) => [name, { result: 'success' }]))
+ const ctx = { needs, inputs: { build_template: true } }
+ expect(evaluate(ctx).coerceString()).toBe('true')
+ expect(evaluate(ctx, true).coerceString()).toBe('false')
+ for (const name of required) {
+ for (const result of ['failure', 'cancelled', 'skipped']) {
+ expect(evaluate({ ...ctx, needs: { ...needs, [name]: { result } } }).coerceString()).toBe(
+ 'false'
+ )
+ }
+ }
+})
+
+it('avoids template download-cache transfers while retaining native and verification caches', () => {
+ const install = persistence.steps.find((step) => step.id === 'dependencies')
+ expect(install.with['native-runtime']).toContain("runner.os == 'Windows' && 'node'")
+ expect(install.with['cache-pnpm-verification']).toBeUndefined()
+ expect(install.with['persist-native-cache']).toBeUndefined()
+ const source = install.with['cache-pnpm-store'].slice(3, -2)
+ for (const os of ['Linux', 'macOS', 'Windows']) {
+ for (const buildTemplate of [true, false]) {
+ expect(
+ runInNewContext(source, { inputs: { build_template: buildTemplate }, runner: { os } })
+ ).toBe(!buildTemplate && os !== 'Windows')
+ }
+ }
+})
diff --git a/config/scripts/orcad-e2e-change-scope.test.mjs b/config/scripts/orcad-e2e-change-scope.test.mjs
index e1f90ff537e7..76e0ede6a06c 100644
--- a/config/scripts/orcad-e2e-change-scope.test.mjs
+++ b/config/scripts/orcad-e2e-change-scope.test.mjs
@@ -64,7 +64,8 @@ const cases = {
'src/main/wsl/wsl-orcad-deploy.test.ts',
'src/main/wsl/nested/wsl-orcad-deploy.test.ts',
'src/main/ssh/orcad-activation-outcome-log.test.ts',
- 'src/main/managed-server/install/orcad-remote-shell-commands.integration.test.ts'
+ 'src/main/managed-server/install/orcad-remote-shell-commands.integration.test.ts',
+ 'src/packages/process-host/src/run-process.test.ts'
],
retained: [
'src/main/wsl/wsl-orcad-real-distro-lane.test.ts',
@@ -76,6 +77,7 @@ const cases = {
'src/main/ssh/orcad-launch-test-harness.ts',
'src/main/ssh/orcad-launch.test-fixture.ts',
'src/main/managed-server/install/remote-install-transport.ts',
+ 'src/packages/process-host/src/run-process.ts',
'.github/workflows/wsl-orcad-e2e.yml'
],
product: 'src/main/wsl/wsl-orcad-deploy.ts'
@@ -148,7 +150,7 @@ it.each([
it('re-includes the real WSL distro root after excluding unit tests', () => {
const paths = workflows.wsl.on.pull_request.paths
- const exclude = paths.indexOf('!src/main/wsl/**/*.test.ts')
+ const exclude = paths.indexOf('!**/*.test.ts')
expect(exclude).toBeGreaterThan(paths.indexOf('src/main/wsl/**'))
expect(paths.indexOf('src/main/wsl/wsl-orcad-real-distro-lane.test.ts')).toBeGreaterThan(exclude)
const steps = workflows.wsl.jobs.wsl2.steps
@@ -161,8 +163,24 @@ it('re-includes the real WSL distro root after excluding unit tests', () => {
// Each remote host lane executes these sources, so a change to one must start the lane.
describe.each([
- ['hostileHosts', ['src/main/ssh/ssh-connection.ts', 'src/main/ssh/ssh-connection-utils.ts']],
- ['windowsHosts', ['src/main/ssh/ssh-connection.ts', 'src/main/ssh/ssh-connection-utils.ts']],
+ [
+ 'hostileHosts',
+ [
+ 'src/main/ssh/connection/ssh-connection.ts',
+ 'src/main/ssh/connection/ssh-connect-config.ts',
+ 'src/main/ssh/ssh-posix-command-wrapper.ts',
+ 'src/shared/shell-argument-quoting.ts'
+ ]
+ ],
+ [
+ 'windowsHosts',
+ [
+ 'src/main/ssh/connection/ssh-connection.ts',
+ 'src/main/ssh/connection/ssh-connect-config.ts',
+ 'src/main/ssh/ssh-posix-command-wrapper.ts',
+ 'src/shared/shell-argument-quoting.ts'
+ ]
+ ],
['wsl', ['src/main/orcad/orcad-wsl-host-login-environment.ts']]
])('%s remote transport routing', (lane, sources) => {
const paths = workflows[lane].on.pull_request.paths
diff --git a/config/scripts/orcad-mobile-web-bundle.mjs b/config/scripts/orcad-mobile-web-bundle.mjs
index 2aad10ad967c..27fea82fcf01 100644
--- a/config/scripts/orcad-mobile-web-bundle.mjs
+++ b/config/scripts/orcad-mobile-web-bundle.mjs
@@ -10,14 +10,19 @@ import {
const { MOBILE_WEB_BUNDLE_DIR, assertMobileWebBundleBuilt } = createRequire(import.meta.url)(
'./verify-packaged-mobile-web-bundle.cjs'
)
+const { assertNoDarkOutputOfOffSwitches } = createRequire(import.meta.url)(
+ '../dark-build-switches.cjs'
+)
/**
* The built page (pnpm build:mobile-web) into an orcad package. Fatal when it is missing or stale,
* as electron-builder's beforePack is: a server without it serves phones no page.
*/
export function stageOrcadMobileWebBundle(outputDir, bundleDir = MOBILE_WEB_BUNDLE_DIR) {
+ // The release template refuses the variable itself.
+ assertNoDarkOutputOfOffSwitches([bundleDir])
assertMobileWebBundleBuilt(bundleDir)
- // The guard refuses unlisted files, so the tree is exactly the manifest and its assets.
+ // The guard refuses unlisted files, so the tree is the manifest, its assets and a dark marker.
cpSync(bundleDir, join(outputDir, ...ORCAD_MOBILE_WEB_DIR.split('/')), { recursive: true })
}
diff --git a/config/scripts/orcad-mobile-web-bundle.test.mjs b/config/scripts/orcad-mobile-web-bundle.test.mjs
index 27079a8c0ef9..44a5c417013a 100644
--- a/config/scripts/orcad-mobile-web-bundle.test.mjs
+++ b/config/scripts/orcad-mobile-web-bundle.test.mjs
@@ -23,6 +23,7 @@ beforeEach(() => {
afterEach(() => {
vi.restoreAllMocks()
+ vi.unstubAllEnvs()
rmSync(scratch, { recursive: true, force: true })
})
@@ -42,6 +43,18 @@ it('stages the manifest and every asset it lists, byte for byte', async () => {
)
})
+it('stages a dark page into a dark orcad, and refuses it in any other', async () => {
+ vi.stubEnv('ORCA_SIDEBAR_INPUTS', 'main')
+ await writeMobileWebBundleFixtureTree({ outDir: bundleDir })
+ stageOrcadMobileWebBundle(outputDir, bundleDir)
+ expect(existsSync(join(outputDir, ORCAD_MOBILE_WEB_MANIFEST))).toBe(true)
+
+ vi.unstubAllEnvs()
+ rmSync(outputDir, { recursive: true, force: true })
+ expect(() => stageOrcadMobileWebBundle(outputDir, bundleDir)).toThrow('ORCA_SIDEBAR_INPUTS=main')
+ expect(existsSync(join(outputDir, ORCAD_MOBILE_WEB_MANIFEST))).toBe(false)
+})
+
it('fails the build when the page was never built', () => {
expect(() => stageOrcadMobileWebBundle(outputDir, bundleDir)).toThrow('pnpm build:mobile-web')
expect(existsSync(join(outputDir, ORCAD_MOBILE_WEB_MANIFEST))).toBe(false)
diff --git a/config/scripts/orcad-prebuild-slot-contents.mjs b/config/scripts/orcad-prebuild-slot-contents.mjs
index 8a6482e6e854..a74ffdcdd066 100644
--- a/config/scripts/orcad-prebuild-slot-contents.mjs
+++ b/config/scripts/orcad-prebuild-slot-contents.mjs
@@ -116,6 +116,8 @@ export function findPostBaselineNodeApiNames(binary) {
return [...names].filter((name) => !NODE_API_EXPORTS.has(name)).sort()
}
+const WINDOWS_WINPTY_FILES = ['pty.node', 'winpty.dll', 'winpty-agent.exe']
+
/**
* Files a slot ships, as `[slot-relative path, source path]`.
*
@@ -127,18 +129,20 @@ export function findPostBaselineNodeApiNames(binary) {
* lib/conpty_console_list_agent.js loads it, but only on the non-DLL kill path, which Orca
* never takes (it always passes useConptyDll on win32). Shipped so that path fails soft
* rather than on a missing module.
+ * `pty.node`, `winpty.dll` and `winpty-agent.exe` are upstream's winpty prebuilds, also kept by
+ * the desktop: node-pty selects winpty below Windows build 18309 (Windows 10 1809 / Server 2019),
+ * so without them orcad cannot open a terminal there (#30906).
*/
export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) {
if (platform === 'win32') {
const conptyRuntime = windowsConptyRuntimeDir(nodePtyDir, arch)
+ const upstream = join(nodePtyDir, 'prebuilds', `win32-${arch}`)
return [
['conpty.node', join(buildDir, 'conpty.node')],
- [
- 'conpty_console_list.node',
- join(nodePtyDir, 'prebuilds', `win32-${arch}`, 'conpty_console_list.node')
- ],
+ ['conpty_console_list.node', join(upstream, 'conpty_console_list.node')],
['conpty/conpty.dll', join(conptyRuntime, 'conpty.dll')],
- ['conpty/OpenConsole.exe', join(conptyRuntime, 'OpenConsole.exe')]
+ ['conpty/OpenConsole.exe', join(conptyRuntime, 'OpenConsole.exe')],
+ ...WINDOWS_WINPTY_FILES.map((file) => [file, join(upstream, file)])
]
}
// Why spawn-helper on macOS only: node-pty posix_spawns it there, and binding.gyp builds it
diff --git a/config/scripts/orcad-prebuild-slot-contents.test.mjs b/config/scripts/orcad-prebuild-slot-contents.test.mjs
index af6f3b45f5e4..bd79fffb2080 100644
--- a/config/scripts/orcad-prebuild-slot-contents.test.mjs
+++ b/config/scripts/orcad-prebuild-slot-contents.test.mjs
@@ -150,8 +150,16 @@ describe('slotSourceFiles', () => {
'conpty.node',
'conpty_console_list.node',
'conpty/conpty.dll',
- 'conpty/OpenConsole.exe'
+ 'conpty/OpenConsole.exe',
+ 'pty.node',
+ 'winpty.dll',
+ 'winpty-agent.exe'
])
+ // Upstream's winpty prebuilds: Orca's patch removes winpty from the source build.
+ const sources = Object.fromEntries(files)
+ for (const file of ['pty.node', 'winpty.dll', 'winpty-agent.exe']) {
+ expect(sources[file]).toBe(join(nodePtyDir, 'prebuilds', 'win32-arm64', file))
+ }
expect(files[2][1]).toBe(
join(nodePtyDir, 'third_party', 'conpty', '1.23.251008001', 'win10-arm64', 'conpty.dll')
)
diff --git a/config/scripts/orcad-prebuild-smoke-child.cjs b/config/scripts/orcad-prebuild-smoke-child.cjs
index 2fbb2e45e454..86cbdf7a4d97 100644
--- a/config/scripts/orcad-prebuild-smoke-child.cjs
+++ b/config/scripts/orcad-prebuild-smoke-child.cjs
@@ -22,44 +22,58 @@ if (process.platform === 'win32') {
}
const token = 'orca-prebuild-smoke-ok'
-let output = ''
-let settled = false
-// Why the pinned node as the child: no shell, so no platform quoting and no cmd.exe /c.
-const child = pty.spawn(
- process.execPath,
- ['-e', `process.stdout.write(${JSON.stringify(token)})`],
- {
- name: 'xterm-color',
- cols: 80,
- rows: 24,
- cwd: tmpdir(),
- env: process.env,
- ...(process.platform === 'win32' ? { useConptyDll: true } : {})
- }
-)
-
-const timeout = setTimeout(() => {
- settled = true
- child.kill()
- process.stderr.write(`prebuild smoke timed out; output so far: ${JSON.stringify(output)}\n`)
- process.exit(1)
-}, 20_000)
-child.onData((data) => {
- output = `${output}${data}`.slice(-4096)
-})
-child.onExit(({ exitCode }) => {
- if (settled) {
- return
- }
- settled = true
- clearTimeout(timeout)
- if (exitCode !== 0 || !output.includes(token)) {
- process.stderr.write(
- `prebuild smoke failed: exit=${exitCode} output=${JSON.stringify(output)}\n`
+function smokeSpawn(backend, windowsOptions) {
+ return new Promise((resolve) => {
+ let output = ''
+ let settled = false
+ // Why the pinned node as the child: no shell, so no platform quoting and no cmd.exe /c.
+ const child = pty.spawn(
+ process.execPath,
+ ['-e', `process.stdout.write(${JSON.stringify(token)})`],
+ {
+ name: 'xterm-color',
+ cols: 80,
+ rows: 24,
+ cwd: tmpdir(),
+ env: process.env,
+ ...(process.platform === 'win32' ? windowsOptions : {})
+ }
)
- process.exit(1)
+ const timeout = setTimeout(() => {
+ settled = true
+ child.kill()
+ process.stderr.write(
+ `${backend} prebuild smoke timed out; output so far: ${JSON.stringify(output)}\n`
+ )
+ process.exit(1)
+ }, 20_000)
+ child.onData((data) => {
+ output = `${output}${data}`.slice(-4096)
+ })
+ child.onExit(({ exitCode }) => {
+ if (settled) {
+ return
+ }
+ settled = true
+ clearTimeout(timeout)
+ if (exitCode !== 0 || !output.includes(token)) {
+ process.stderr.write(
+ `${backend} prebuild smoke failed: exit=${exitCode} output=${JSON.stringify(output)}\n`
+ )
+ process.exit(1)
+ }
+ resolve()
+ })
+ })
+}
+
+void (async () => {
+ await smokeSpawn('pty', { useConptyDll: true })
+ if (process.platform === 'win32') {
+ // node-pty picks winpty below build 18309 (#30906); `useConpty: false` forces it on this runner.
+ await smokeSpawn('winpty', { useConpty: false })
}
process.stdout.write(`${token} node=${process.version} napi=${process.versions.napi}\n`)
process.exit(0)
-})
+})()
diff --git a/config/scripts/orcad-windows-prebuild-cache.test.mjs b/config/scripts/orcad-windows-prebuild-cache.test.mjs
index 25da9dbfedb6..3a96a6cce7f7 100644
--- a/config/scripts/orcad-windows-prebuild-cache.test.mjs
+++ b/config/scripts/orcad-windows-prebuild-cache.test.mjs
@@ -50,11 +50,14 @@ function fixture(arch = 'x64') {
['conpty.node', Buffer.concat([peImage({ arch }), CYGWIN_BREAKAWAY_MARKER])],
['conpty_console_list.node', peImage({ arch })],
['conpty/conpty.dll', peImage({ arch })],
- ['conpty/OpenConsole.exe', peImage({ arch })]
+ ['conpty/OpenConsole.exe', peImage({ arch })],
+ ['pty.node', peImage({ arch })],
+ ['winpty.dll', peImage({ arch })],
+ ['winpty-agent.exe', peImage({ arch })]
]) {
write(join(slotDir, file), bytes)
files[file] = sha256Of(join(slotDir, file))
- if (file === 'conpty_console_list.node') {
+ if (file !== 'conpty.node' && !file.startsWith('conpty/')) {
write(join(sourceDir, 'prebuilds', slot, file), bytes)
}
if (file.startsWith('conpty/')) {
@@ -119,7 +122,7 @@ describe('Windows server prebuild cache validation', () => {
f.save()
expect(() => validateWindowsPrebuildCache(f)).toThrow(/exactly the current ConPTY payload/)
const extra = fixture()
- writeFileSync(join(extra.slotDir, 'pty.node'), 'upstream fallback')
+ writeFileSync(join(extra.slotDir, 'conpty.pdb'), 'upstream symbols')
expect(() => validateWindowsPrebuildCache(extra)).toThrow(/exactly the current ConPTY payload/)
const missing = fixture()
rmSync(join(missing.slotDir, 'conpty/OpenConsole.exe'))
diff --git a/config/scripts/packaged-cloud-source-exclusion.test.mjs b/config/scripts/packaged-cloud-source-exclusion.test.mjs
new file mode 100644
index 000000000000..e53b4be4954d
--- /dev/null
+++ b/config/scripts/packaged-cloud-source-exclusion.test.mjs
@@ -0,0 +1,53 @@
+import { existsSync } from 'node:fs'
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { createRequire } from 'node:module'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+
+const require = createRequire(import.meta.url)
+const electronBuilderConfig = require('../electron-builder.config.cjs')
+const { copyFiles, FileMatcher } = require('app-builder-lib/out/fileMatcher')
+
+describe('packaged cloud source exclusion', () => {
+ it('omits cloud service sources during traversal while preserving compiled cloud clients', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'orca-packaging-cloud-'))
+ const source = join(root, 'app')
+ const destination = join(root, 'selected')
+ const cloudPaths = [
+ 'cloud/apps/relay/src/index.ts',
+ 'cloud/apps/push/src/app.test.ts',
+ 'cloud/infra/main.tf',
+ 'cloud/dev/scripts/deploy.mjs',
+ 'cloud/.env',
+ 'cloud/packages/push-contract/src/push-host-proof-vector.json'
+ ]
+ const runtimePaths = [
+ 'package.json',
+ 'out/main/index.js',
+ 'out/cli/index.js',
+ 'out/main/runtime/mobile-cloud-relay/index.js',
+ 'out/main/runtime/push/push-host-proof.js',
+ 'out/shared/mobile-push-contract.js',
+ 'out/renderer/index.html',
+ 'out/renderer/assets/cloud/client.js',
+ 'resources/cloud/icon.png'
+ ]
+ try {
+ for (const fixturePath of [...cloudPaths, ...runtimePaths]) {
+ const file = join(source, fixturePath)
+ await mkdir(dirname(file), { recursive: true })
+ await writeFile(file, fixturePath)
+ }
+ await copyFiles([
+ new FileMatcher(source, destination, (value) => value, electronBuilderConfig.files)
+ ])
+ expect(existsSync(join(destination, 'cloud'))).toBe(false)
+ for (const runtimePath of runtimePaths) {
+ expect(await readFile(join(destination, runtimePath), 'utf8')).toBe(runtimePath)
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+})
diff --git a/config/scripts/packaged-orcad-template.cjs b/config/scripts/packaged-orcad-template.cjs
index 3041934a3f92..b7deb42fe33d 100644
--- a/config/scripts/packaged-orcad-template.cjs
+++ b/config/scripts/packaged-orcad-template.cjs
@@ -23,6 +23,10 @@ const {
ORCAD_TEMPLATE_TARGETS_DIR
} = require('../../src/shared/orcad-artifacts.ts')
const { verifyPackagedOrcadTemplate } = require('./verify-packaged-orcad-template.cjs')
+const {
+ compactPackagedOrcadRipgrep,
+ resealSignedPackagedRipgrep
+} = require('./compact-packaged-orcad-ripgrep.cjs')
const ORCAD_TEMPLATE_RESOURCE_DIR = 'orcad-template'
const orcadTemplateExtraResource = { from: 'out/orcad-template', to: ORCAD_TEMPLATE_RESOURCE_DIR }
@@ -135,15 +139,16 @@ async function finalizePackagedOrcadTemplate(resourcesDir, options) {
}
resealOrcadTemplateManifest(templateDir, machO)
}
- verifyPackagedOrcadTemplate(resourcesDir)
+ compactPackagedOrcadRipgrep(resourcesDir)
}
/** `inner-signing-list.txt` lines are app-relative Windows paths; keep the template's. */
function resealSignedWindowsApp(appDir, signedListFile) {
const prefix = `resources/${ORCAD_TEMPLATE_RESOURCE_DIR}/`
- const signed = readFileSync(signedListFile, 'utf8')
+ const signedAppPaths = readFileSync(signedListFile, 'utf8')
.split(/\r?\n/)
.map((line) => line.trim().replaceAll('\\', '/'))
+ const signed = signedAppPaths
.filter((line) => line.startsWith(prefix))
.map((line) => line.slice(prefix.length))
const resourcesDir = join(appDir, 'resources')
@@ -155,6 +160,10 @@ function resealSignedWindowsApp(appDir, signedListFile) {
return
}
resealOrcadTemplateManifest(join(resourcesDir, ORCAD_TEMPLATE_RESOURCE_DIR), signed)
+ resealSignedPackagedRipgrep(
+ resourcesDir,
+ signedAppPaths.filter((path) => path.startsWith('resources/')).map((path) => path.slice(10))
+ )
verifyPackagedOrcadTemplate(resourcesDir)
console.log(`[packaged-orcad-template] resealed ${signed.length} signed template file(s)`)
}
diff --git a/config/scripts/patched-dependencies-frozen-install.test.mjs b/config/scripts/patched-dependencies-frozen-install.test.mjs
index 071b58240af2..d435769cd40c 100644
--- a/config/scripts/patched-dependencies-frozen-install.test.mjs
+++ b/config/scripts/patched-dependencies-frozen-install.test.mjs
@@ -63,6 +63,8 @@ describe('patched dependencies', () => {
'package.json',
'pnpm-lock.yaml',
'pnpm-workspace.yaml',
+ // The lockfile pins its checksum, so a frozen install fails without it.
+ 'config/pnpmfile.cjs',
...workspacePackageManifests(PROJECT_DIR)
]) {
mkdirSync(resolve(scratch, file, '..'), { recursive: true })
diff --git a/config/scripts/pnpmfile-global-virtual-store.test.mjs b/config/scripts/pnpmfile-global-virtual-store.test.mjs
new file mode 100644
index 000000000000..71e2ed3142b6
--- /dev/null
+++ b/config/scripts/pnpmfile-global-virtual-store.test.mjs
@@ -0,0 +1,39 @@
+import { createRequire } from 'node:module'
+import { describe, expect, it } from 'vitest'
+
+const require = createRequire(import.meta.url)
+const {
+ FORCE_IN_CI_ENV,
+ GLOBAL_VIRTUAL_STORE_ENV: STORE_ENV,
+ usesGlobalVirtualStore,
+ hooks
+} = require('../pnpmfile.cjs')
+
+describe('usesGlobalVirtualStore', () => {
+ const cases = [
+ // [env, expected]; platform-independent, so Windows standard users get it too.
+ [{}, true],
+ [{ [STORE_ENV]: '1' }, true],
+ [{ [STORE_ENV]: '0' }, false],
+ [{ [STORE_ENV]: 'yes' }, true],
+ [{ CI: 'true' }, false],
+ [{ CI: '1' }, false],
+ [{ GITHUB_ACTIONS: 'true' }, false],
+ [{ CI: 'true', [STORE_ENV]: '1' }, false],
+ [{ CI: 'true', [FORCE_IN_CI_ENV]: '1' }, true],
+ [{ GITHUB_ACTIONS: 'true', [FORCE_IN_CI_ENV]: '1' }, true],
+ [{ [FORCE_IN_CI_ENV]: '1', [STORE_ENV]: '0' }, false],
+ [{ CI: 'false' }, true],
+ [{ CI: '0' }, true]
+ ]
+ it.each(cases)('env %j -> %s', (env, expected) => {
+ expect(usesGlobalVirtualStore({ env })).toBe(expected)
+ })
+})
+
+describe('config/pnpmfile.cjs updateConfig', () => {
+ it('sets the store from the policy, so pnpm-workspace.yaml never has to', () => {
+ const config = hooks.updateConfig({})
+ expect(config.enableGlobalVirtualStore).toBe(usesGlobalVirtualStore())
+ })
+})
diff --git a/config/scripts/pr-code-change-scope-format-route.test.mjs b/config/scripts/pr-code-change-scope-format-route.test.mjs
new file mode 100644
index 000000000000..3430e0f86939
--- /dev/null
+++ b/config/scripts/pr-code-change-scope-format-route.test.mjs
@@ -0,0 +1,28 @@
+import { readFileSync } from 'node:fs'
+import { expect, it } from 'vitest'
+import { parse } from 'yaml'
+import { classifyPrJobs } from './pr-code-change-scope.mjs'
+
+// The formatting check reads the whole tree, so a diff that skipped it would land drift that then
+// fails every later PR's merge ref.
+it.each(['README.md', '.github/ISSUE_TEMPLATE/bug.yml', '.github/workflows/mobile.yml'])(
+ 'runs static analysis, and only it, for a %s diff the formatter reads',
+ (file) => {
+ expect(classifyPrJobs([file])).toMatchObject({
+ should_run: false,
+ static_analysis: true,
+ typecheck: false,
+ test: false
+ })
+ }
+)
+
+it('leaves paths the formatter ignores out of static analysis', () => {
+ expect(classifyPrJobs(['cloud/apps/relay/README.md']).static_analysis).toBe(false)
+ expect(classifyPrJobs(['docs/assets/logo.png']).static_analysis).toBe(false)
+})
+
+it('checks out the formatter config the detector reads its ignores from', () => {
+ const workflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8'))
+ expect(workflow.jobs.code_paths.steps[0].with['sparse-checkout']).toContain('/.oxfmtrc.json')
+})
diff --git a/config/scripts/pr-code-change-scope.mjs b/config/scripts/pr-code-change-scope.mjs
index 2f772be3caeb..0e6076bb1679 100644
--- a/config/scripts/pr-code-change-scope.mjs
+++ b/config/scripts/pr-code-change-scope.mjs
@@ -33,7 +33,8 @@ export const PR_CHECK_JOBS = [
'managed_hook_node18',
'package',
'package_windows',
- 'layout_oracle'
+ 'layout_oracle',
+ 'dependency_isolation'
]
const ALWAYS_ON_CODE_JOBS = new Set(['static_analysis', 'typecheck', 'test'])
@@ -158,6 +159,18 @@ const ORCAD_BROWSER_PREFIXES = [
// The page bundle the desktop packages: the builder and verifier, the manifest writer and the
// packaging guard they share, the entry, the route tree it mounts, the mobile source those routes
// import, and the shell policy the render check runs the page under.
+// What decides which deps each package sees; package.json and the lockfile already force every job.
+const DEPENDENCY_ISOLATION_PREFIXES = [
+ 'pnpm-workspace.yaml',
+ 'config/pnpmfile.cjs',
+ 'config/scripts/undeclared-dependency-resolution',
+ 'config/build-plugins/markdown-parser-exports',
+ 'config/tsconfig',
+ 'src/packages/',
+ 'native/windows-registry/package.json',
+ 'native/windows-security/package.json'
+]
+
const MOBILE_WEB_APP_PREFIXES = [
'config/scripts/build-mobile-web-app',
'config/scripts/run-mobile-web-app-checks',
@@ -281,6 +294,8 @@ const NATIVE_RUNTIME_PREFIXES = [
'config/scripts/windows-process-tree-creation-time',
'config/scripts/windows-process-tree-gyp-rebuild',
'config/scripts/electron-builder-native-rebuild',
+ 'config/scripts/native-build-entry',
+ 'config/scripts/shared-native-build',
'config/patches/node-pty@',
'config/patches/@vscode__windows-process-tree'
]
@@ -292,7 +307,7 @@ const NATIVE_CACHE_FILES = new Set([
'.github/actions/prepare-native-runtime/action.yml',
'pnpm-workspace.yaml',
'.npmrc',
- '.pnpmfile.cjs',
+ 'config/pnpmfile.cjs',
'config/scripts/ensure-native-runtime.mjs',
'config/scripts/initialize-windows-native-toolchain.mjs',
'config/scripts/capture-windows-native-environment.cmd',
@@ -306,7 +321,11 @@ const NATIVE_CACHE_FILES = new Set([
'config/scripts/zip-extractor-command.mjs',
'src/shared/zip-extractor-command.ts',
'config/scripts/shared-electron-dist-cache.mjs',
- 'config/scripts/space-sharing-copy.mjs'
+ 'config/scripts/shared-tree-publish.mjs',
+ 'config/scripts/space-sharing-copy.mjs',
+ 'config/scripts/native-build-entry.cjs',
+ 'config/scripts/native-build-entry-probe.cjs',
+ 'config/scripts/shared-native-build.mjs'
])
const NATIVE_CACHE_PREFIXES = [
@@ -321,6 +340,7 @@ const SHARED_PACKAGE_PREFIXES = [
'electron.vite.config.ts',
'config/electron-builder',
'config/packaged-runtime',
+ 'config/packaged-native-build-entries',
'config/build-plugins/',
'config/scripts/build-',
'config/scripts/smoke-packaged',
@@ -425,6 +445,25 @@ export const STATIC_ANALYSIS_SCAN_ROOTS = readStaticAnalysisScanRoots()
const STATIC_ANALYSIS_SCAN_PREFIXES = STATIC_ANALYSIS_SCAN_ROOTS.map((root) => `${root}/`)
+// Over-inclusive on purpose: a type oxfmt skips only costs this PR a static run.
+const FORMAT_CHECKED_EXTENSION =
+ /\.(?:[cm]?[jt]sx?|json[c5]?|css|less|scss|md|mdx|ya?ml|vue|html|graphql|gql|toml)$/i
+
+function readFormatIgnoredPrefixes() {
+ let config
+ try {
+ config = JSON.parse(readFileSync(join(import.meta.dirname, '../../.oxfmtrc.json'), 'utf8'))
+ } catch {
+ // Fail closed: with no ignores known, every formatted type keeps the gate.
+ return []
+ }
+ return (config.ignorePatterns ?? [])
+ .filter((pattern) => pattern.endsWith('/**'))
+ .map((pattern) => pattern.slice(0, -2))
+}
+
+const FORMAT_IGNORED_PREFIXES = readFormatIgnoredPrefixes()
+
export function isDocsOnlyPath(file) {
if (DOCS_ONLY_FILES.has(file)) {
return true
@@ -494,7 +533,11 @@ export function classifyPrJobs(
// Why outside should_run: a mobile-only diff is desktop-irrelevant and skips every job above,
// but the repo-wide audits lint mobile/, and skipping them lands the violation on main, where
// it then fails this same gate on every later PR's merge ref.
- jobs.static_analysis = jobs.static_analysis || changedFiles.some(isStaticAnalysisScannedPath)
+ // Why formatted paths too: the formatting check reads the whole tree, so a docs-only diff that
+ // skipped it would land drift that then fails every later PR.
+ jobs.static_analysis =
+ jobs.static_analysis ||
+ changedFiles.some((file) => isStaticAnalysisScannedPath(file) || isFormatCheckedPath(file))
// Why outside should_run, for the same reason: a mobile-only diff is desktop-irrelevant, and
// that is exactly the diff that changes the page this job builds. Gated on should_run it would
// skip on every PR that can break it and run on none.
@@ -546,6 +589,8 @@ function jobDetector(job) {
return (files) => files.some(isWindowsPackagePath)
case 'layout_oracle':
return (files) => files.some(isLayoutOracleSource)
+ case 'dependency_isolation':
+ return (files) => files.some((file) => matchesPrefix(file, DEPENDENCY_ISOLATION_PREFIXES))
default:
return () => true
}
@@ -597,6 +642,10 @@ function isStaticAnalysisScannedPath(file) {
)
}
+function isFormatCheckedPath(file) {
+ return FORMAT_CHECKED_EXTENSION.test(file) && !matchesPrefix(file, FORMAT_IGNORED_PREFIXES)
+}
+
function isNativeCacheInputPath(file) {
return NATIVE_CACHE_FILES.has(file) || matchesPrefix(file, NATIVE_CACHE_PREFIXES)
}
diff --git a/config/scripts/pr-e2e-managed-ssh-harness-route.test.mjs b/config/scripts/pr-e2e-managed-ssh-harness-route.test.mjs
index ca34a2575982..4b6c7f085d14 100644
--- a/config/scripts/pr-e2e-managed-ssh-harness-route.test.mjs
+++ b/config/scripts/pr-e2e-managed-ssh-harness-route.test.mjs
@@ -2,14 +2,14 @@ import { describe, expect, it } from 'vitest'
import { readdirSync } from 'node:fs'
import { join, resolve, sep } from 'node:path'
import { testImportClosure } from './e2e-test-import-closure.mjs'
-import { selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
+import { MANUAL_ONLY_E2E_SPECS, selectPrE2eSpecs } from './pr-e2e-source-routing.mjs'
const projectDir = resolve(import.meta.dirname, '../..')
const MANAGED_SSH_SPEC = /^tests\/e2e\/ssh-orcad-.*\.spec\.ts$/
const allSpecs = readdirSync(join(projectDir, 'tests/e2e'), { encoding: 'utf8', recursive: true })
.map((name) => `tests/e2e/${name.split(sep).join('/')}`)
- .filter((file) => file.endsWith('.spec.ts'))
+ .filter((file) => file.endsWith('.spec.ts') && !MANUAL_ONLY_E2E_SPECS.includes(file))
/** Test files only managed-SSH specs import: nothing else runs when they change. */
function managedSshOnlyHarness() {
diff --git a/config/scripts/pr-e2e-source-routing.mjs b/config/scripts/pr-e2e-source-routing.mjs
index 59797779c16e..c6d4057af4be 100644
--- a/config/scripts/pr-e2e-source-routing.mjs
+++ b/config/scripts/pr-e2e-source-routing.mjs
@@ -5,6 +5,7 @@ import {
isDaemonInitTestSupportChange,
isUnitTestSupportSource
} from './pr-code-change-scope.mjs'
+import { MANAGED_SSH_E2E_SPEC_PREFIX } from './ci-e2e-job-selection.mjs'
import { LAYOUT_ORACLE_E2E_SPECS } from './pr-layout-oracle-route.mjs'
const isProductSource = (file) => !/\.test\.tsx?$/.test(file) && !isUnitTestSupportSource(file)
@@ -20,10 +21,10 @@ const NATIVE_IME_PRODUCT_SOURCE =
const NATIVE_IME_HARNESS =
/^(?:config\/scripts\/focus-nested-wayland-terminal\.sh$|config\/scripts\/(?:run-terminal-ibus-hangul-e2e|terminal-ime-engagement-receipt)\.mjs$|tests\/e2e\/terminal-ime-(?:boundary-probe|byte-reader|engagement-receipt)\.ts$|tests\/e2e\/terminal-(?:ibus-hangul|hangul-terminating-digit|macos-2set-korean)-native\.spec\.ts$)/
-/**
- * Managed-SSH harness helpers; every managed-SSH spec runs in one job, so any route may claim them.
- * `pr-e2e-managed-ssh-harness-route.test.mjs` fails when a helper only these specs import is missing.
- */
+// This spec requires ORCA_SIDEBAR_INPUTS=main; ordinary PR builds can only skip it.
+export const MANUAL_ONLY_E2E_SPECS = ['tests/e2e/ssh-orcad-phone-feed-list.spec.ts']
+
+// Shared fixtures and workflow changes must exercise every managed-SSH consumer.
const MANAGED_SSH_HARNESS =
/^tests\/e2e\/helpers\/(?:orcad-managed-workspace|orcad-convert-(?:flow|host|teardown)|orcad-template-variant|orcad-upgrade-(?:profile|retained-state)|docker-ssh-host-target|reserve-local-port|ssh-remote-only-browser-fixture|windows-host-orcad-processes)\.ts$/
@@ -151,7 +152,7 @@ export const PR_E2E_SOURCE_ROUTES = [
specs: ['tests/e2e/ssh-orcad-browser-capabilities.spec.ts'],
matches: (file) =>
isProductSource(file) &&
- /^src\/main\/(?:runtime\/(?:runtime-browser-commands-factory|orca-runtime-get-status)|host\/electron-browser-commands|orcad\/orcad-browser-)/.test(
+ /^src\/main\/(?:runtime\/(?:runtime-browser-commands-factory|orca-runtime-get-status|browser\/browser-client-page-opening)|host\/electron-browser-commands|orcad\/orcad-browser-)/.test(
file
)
},
@@ -248,7 +249,18 @@ export const PR_E2E_SOURCE_ROUTES = [
file
) ||
(isProductSource(file) &&
- /^src\/(?:main\/(?:ssh|managed-server\/[a-z-]+)\/orcad-managed-|main\/ipc\/(?:ssh\/)?ssh-port-forward|main\/(?:ipc\/(?:(?:ssh\/)?ssh-(?:connect-flow|connection-handlers|host-user-disconnect|session-teardown)|(?:runtime-environments\/)?runtime-environment-(?:connectivity-handlers|manual-disconnect))|ssh\/ssh-(?:connection-manager|user-disconnect))\.ts$|renderer\/src\/startup\/(?:ssh-startup-reconnect|startup-ssh-connection-restore)\.ts$|renderer\/src\/runtime\/(?:web-runtime-session|web-session-close-|runtime-terminal-artifact-read\.ts$)|renderer\/src\/components\/terminal-pane\/(?:remote-runtime-|terminal-host-(?:workspace-file|file-link-existence)\.ts$))/.test(
+ /^src\/(?:main\/(?:ssh|managed-server\/[a-z-]+)\/orcad-managed-|main\/ipc\/(?:ssh\/)?ssh-port-forward|main\/(?:ipc\/(?:(?:ssh\/)?ssh-(?:connect-flow|connection-handlers|host-user-disconnect|session-teardown)|(?:runtime-environments\/)?runtime-environment-(?:connectivity-handlers|manual-disconnect))|ssh\/(?:connection\/ssh-connection-manager|ssh-user-disconnect))\.ts$|renderer\/src\/startup\/(?:ssh-startup-reconnect|startup-ssh-connection-restore)\.ts$|renderer\/src\/runtime\/(?:web-runtime-session|web-session-close-|runtime-terminal-artifact-read\.ts$)|renderer\/src\/components\/terminal-pane\/(?:remote-runtime-|terminal-host-(?:workspace-file|file-link-existence)\.ts$))/.test(
+ file
+ ))
+ },
+ {
+ // Unsaved drafts stay on the desktop and are renamed to the server when a move commits.
+ id: 'ssh.orcad-editor-drafts-migration',
+ specs: ['tests/e2e/ssh-orcad-editor-drafts-migration.spec.ts'],
+ matches: (file) =>
+ MANAGED_SSH_HARNESS.test(file) ||
+ (isProductSource(file) &&
+ /^src\/main\/(?:persistence\/migrating-orcad-catalog\/orcad-source-(?:draft-rename|workspace-session|session-copy-resolution|export)|ssh\/orcad-migration-source-retention|ipc\/ssh\/ssh)\.ts$/.test(
file
))
},
@@ -257,6 +269,7 @@ export const PR_E2E_SOURCE_ROUTES = [
id: 'ssh.orcad-remote-sources',
specs: [
'tests/e2e/ssh-orcad-paired-phone-projects.spec.ts',
+ 'tests/e2e/ssh-orcad-pairing-repair.spec.ts',
'tests/e2e/ssh-orcad-retained-upgrade-state.spec.ts',
'tests/e2e/ssh-orcad-shared-home-machines.spec.ts',
'tests/e2e/ssh-orcad-shared-login-migration.spec.ts',
@@ -265,7 +278,7 @@ export const PR_E2E_SOURCE_ROUTES = [
],
matches: (file) =>
isProductSource(file) &&
- /^src\/(?:main\/(?:ssh|orcad|managed-server)\/|main\/ipc\/(?:ssh|runtime-environment|orcad-)|cli\/runtime\/(?:launch-)?serve|shared\/(?:remote-runtime|orcad-|protocol-version\.ts$|runtime-capability)|renderer\/src\/store\/repos\/(?:repo-removal|runtime-project-terminal-stop)|renderer\/src\/components\/sidebar\/(?:sleep-|SleepTwin|WorkspaceSleep|use-worktree-sleep|ssh-host-remove|host-rename-remove|HostRemove))/.test(
+ /^src\/(?:main\/(?:ssh|orcad|managed-server)\/|main\/ipc\/(?:ssh|runtime-environment|orcad-)|cli\/runtime\/(?:launch-)?serve|shared\/(?:remote-runtime|orcad-|protocol-version\.ts$|runtime-capability)|renderer\/src\/store\/repos\/(?:repo-removal|runtime-project-terminal-stop)|renderer\/src\/components\/sidebar\/(?:sleep-|SleepTwin|WorkspaceSleep|use-worktree-sleep|ssh-host-remove|host-rename-remove|HostRemove)|renderer\/src\/components\/terminal-pane\/(?:TerminalRemoteRuntimeReconnectBanner|use-managed-pairing-refused|TerminalPaneRuntimePortals))/.test(
file
)
},
@@ -288,7 +301,7 @@ export const PR_E2E_SOURCE_ROUTES = [
],
matches: (file) =>
isProductSource(file) &&
- /^(?:config\/scripts\/(?:verify-wsl-e2e-participation|verify-playwright-participation)\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|terminal-inline-images-windows\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test(
+ /^(?:config\/scripts\/(?:verify-wsl-e2e-participation|verify-playwright-participation)\.mjs$|src\/main\/(?:wsl[/-]|pty\/.*wsl|providers\/wsl)|src\/shared\/(?:wsl-|windows-terminal-shell|shell-argument-quoting)|src\/renderer\/src\/.*(?:terminal-paste|pty-paste)|tests\/e2e\/(?:golden-tab-bar-agent-launch\.spec|terminal-windows-shell-paste-ownership\.spec|terminal-inline-images-windows\.spec|helpers\/(?:wsl-golden-stub-agent|golden-stub-agent))|\.github\/(?:actions\/setup-wsl-test-runtime\/|workflows\/windows-wsl-e2e\.yml))/.test(
file
)
},
@@ -463,7 +476,10 @@ export function selectPrE2eSpecs(
// The required layout_oracle job owns the oracle specs; running them here too would double the cost.
const specs = new Set(
changedPaths.filter(
- (file) => /^tests\/e2e\/.*\.spec\.ts$/.test(file) && !LAYOUT_ORACLE_E2E_SPECS.includes(file)
+ (file) =>
+ /^tests\/e2e\/.*\.spec\.ts$/.test(file) &&
+ !LAYOUT_ORACLE_E2E_SPECS.includes(file) &&
+ !MANUAL_ONLY_E2E_SPECS.includes(file)
)
)
for (const route of PR_E2E_SOURCE_ROUTES) {
@@ -474,6 +490,17 @@ export function selectPrE2eSpecs(
route.specs.forEach((spec) => specs.add(spec))
reportRoute(`[pr-e2e] ${route.id}: ${route.specs.join(', ')}`)
}
+ if (
+ routePaths.some(
+ (file) =>
+ (file.startsWith('tests/e2e/helpers/') &&
+ [...specs].some((spec) => spec.startsWith(MANAGED_SSH_E2E_SPEC_PREFIX))) ||
+ file === '.github/workflows/e2e.yml' ||
+ file === 'config/scripts/ci-e2e-job-selection.mjs'
+ )
+ ) {
+ specs.add(MANAGED_SSH_E2E_SPEC_PREFIX)
+ }
return [...specs].sort((left, right) => left.localeCompare(right))
}
diff --git a/config/scripts/pr-package-test-paths.mjs b/config/scripts/pr-package-test-paths.mjs
index 14f0f83de5e9..fb5e11da5da4 100644
--- a/config/scripts/pr-package-test-paths.mjs
+++ b/config/scripts/pr-package-test-paths.mjs
@@ -1,4 +1,7 @@
export const LINUX_PACKAGE_TESTS = [
+ 'config/scripts/packaged-cloud-source-exclusion.test.mjs',
+ 'src/main/browser/popup-origin-chrome.electron.test.ts',
+ 'src/main/browser/browser-link-materialization.electron.test.ts',
'src/main/browser/browser-client-page-renderer-lifecycle.electron.test.ts',
'src/main/browser/browser-route-tcp-egress.electron.test.ts',
'src/main/browser/browser-route-webrtc-egress.electron.test.ts',
@@ -6,6 +9,7 @@ export const LINUX_PACKAGE_TESTS = [
'src/main/browser/browser-route-dns-prefetch.electron.test.ts',
'src/main/persistence/profile-state/profile-state-writer-stall.electron.test.ts',
'src/main/browser/browser-popup-compatibility.electron.test.ts',
+ 'src/main/browser/browser-popup-adapter-compatibility.electron.test.ts',
'src/main/browser/browser-popup-native-compatibility.electron.test.ts',
'src/main/browser/browser-popup-background-compatibility.electron.test.ts'
]
diff --git a/config/scripts/pr-unit-support-routing.test.mjs b/config/scripts/pr-unit-support-routing.test.mjs
index bdef0ec15797..238a55114e52 100644
--- a/config/scripts/pr-unit-support-routing.test.mjs
+++ b/config/scripts/pr-unit-support-routing.test.mjs
@@ -23,7 +23,9 @@ const unitHarnesses = [
]
const supportBridges = new Set([
'src/renderer/src/runtime/web-runtime-browser-creation-placement-test-rig.ts',
- 'src/renderer/src/components/editor/editor-model-lifetime-fixture.ts'
+ 'src/renderer/src/components/editor/editor-model-lifetime-fixture.ts',
+ // Autosave suites' store rig; it takes the editor harness's tab-layout maps.
+ 'src/renderer/src/components/editor/editor-autosave-controller-test-fixture.ts'
])
function productConsumers(file, graph, unitFiles) {
diff --git a/config/scripts/project-renderer-web-client.mjs b/config/scripts/project-renderer-web-client.mjs
index a3df2d9b1b56..50fe78376ad7 100644
--- a/config/scripts/project-renderer-web-client.mjs
+++ b/config/scripts/project-renderer-web-client.mjs
@@ -181,6 +181,11 @@ try {
rmSync(webOutput, { force: true, recursive: true })
renameSync(stagingOutput, webOutput)
+ // The installed browser client shares renderer bytes; only its selected files may be served.
+ writeFileSync(
+ join(rendererOutput, '.orca-web-client-files.json'),
+ JSON.stringify([...selectedFiles].sort())
+ )
} finally {
rmSync(stagingOutput, { force: true, recursive: true })
}
diff --git a/config/scripts/project-renderer-web-client.test.mjs b/config/scripts/project-renderer-web-client.test.mjs
index 0b7525fb378b..55ca40ce6a78 100644
--- a/config/scripts/project-renderer-web-client.test.mjs
+++ b/config/scripts/project-renderer-web-client.test.mjs
@@ -103,6 +103,23 @@ describe('renderer web client projection', () => {
expect(existsSync(join(root, 'out/web/assets/logo.png'))).toBe(true)
expect(existsSync(join(root, 'out/web/assets/desktop-entry.js'))).toBe(false)
expect(existsSync(join(root, 'out/web/stale.js'))).toBe(false)
+ const sharedFiles = JSON.parse(
+ readFileSync(join(root, 'out/renderer/.orca-web-client-files.json'), 'utf8')
+ )
+ expect(sharedFiles).toEqual(
+ expect.arrayContaining([
+ 'web-index.html',
+ 'assets/web-entry.js',
+ 'assets/web-shared.js',
+ 'assets/editor.worker-fixture.js',
+ 'assets/web.css',
+ 'assets/lazy.js',
+ 'assets/logo.png'
+ ])
+ )
+ expect(sharedFiles).not.toContain('assets/desktop-entry.js')
+ expect(sharedFiles).not.toContain('.vite/manifest.json')
+ expect(sharedFiles).not.toContain('index.html')
expect(readFileSync(join(root, 'out/web/assets/web.css'), 'utf8')).toBe('.root{color:red}\n')
})
diff --git a/config/scripts/ratchet-baseline.mjs b/config/scripts/ratchet-baseline.mjs
index 5acad73836df..6458083e3289 100644
--- a/config/scripts/ratchet-baseline.mjs
+++ b/config/scripts/ratchet-baseline.mjs
@@ -172,3 +172,40 @@ export function pruneCountBaseline(root, baselinePath, header, current) {
fs.writeFileSync(file, formatCountBaseline(pruned, header, parseCountNotes(text)))
return totalCount(pruned)
}
+
+/**
+ * Gate for a count ratchet `{ name, baselinePath, header, collect(root), fix, unit }`: a file above
+ * its baseline fails with `fix`, one below it fails until pruned. `--prune` lowers rows only.
+ */
+export function runCountRatchet(ratchet, root, arg) {
+ const current = ratchet.collect(root)
+ if (arg === '--prune') {
+ const total = pruneCountBaseline(root, ratchet.baselinePath, ratchet.header, current)
+ console.log(`${ratchet.name} baseline pruned to ${total}.`)
+ return 0
+ }
+ const baselineFile = path.join(root, ratchet.baselinePath)
+ const baseline = parseCountBaseline(
+ fs.existsSync(baselineFile) ? fs.readFileSync(baselineFile, 'utf8') : ''
+ )
+ const { grown, shrunk } = diffCounts(current, baseline)
+ for (const { file, now, allowed } of grown) {
+ console.error(`${file}: ${now} ${ratchet.unit}, baseline allows ${allowed}.`)
+ }
+ if (grown.length > 0) {
+ console.error(ratchet.fix.join('\n'))
+ }
+ for (const { file, now, allowed } of shrunk) {
+ console.error(`${file}: now ${now}, baseline ${allowed}.`)
+ }
+ if (shrunk.length > 0) {
+ console.error(
+ `The baseline may only shrink. Fix it: pnpm check:${ratchet.name}-ratchet --prune`
+ )
+ }
+ if (grown.length + shrunk.length > 0) {
+ return 1
+ }
+ console.log(`${ratchet.name} ratchet OK — ${totalCount(current)} grandfathered ${ratchet.unit}.`)
+ return 0
+}
diff --git a/config/scripts/ratchet-baseline.test.mjs b/config/scripts/ratchet-baseline.test.mjs
index 170034f05109..922c4e51968f 100644
--- a/config/scripts/ratchet-baseline.test.mjs
+++ b/config/scripts/ratchet-baseline.test.mjs
@@ -10,6 +10,7 @@ import {
parseCountBaseline,
parseCountNotes,
parseEntryBaseline,
+ runCountRatchet,
runEntryRatchet
} from './ratchet-baseline.mjs'
@@ -67,6 +68,68 @@ describe('runEntryRatchet', () => {
})
})
+describe('runCountRatchet', () => {
+ let root
+ afterEach(() => {
+ vi.restoreAllMocks()
+ rmSync(root, { recursive: true, force: true })
+ })
+
+ function ratchet(current) {
+ root = mkdtempSync(join(tmpdir(), 'orca-count-ratchet-'))
+ writeFileSync(join(root, 'baseline.txt'), '# header\n2 a.ts # why\n1 b.ts\n')
+ return {
+ name: 'demo',
+ baselinePath: 'baseline.txt',
+ header: ['# header'],
+ collect: () => new Map(current),
+ unit: 'thing(s)',
+ fix: ['Fix the file.']
+ }
+ }
+
+ it('fails growth and an unpruned shrink, and passes the baseline as-is', () => {
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ expect(
+ runCountRatchet(
+ ratchet([
+ ['a.ts', 3],
+ ['b.ts', 1]
+ ]),
+ root
+ )
+ ).toBe(1)
+ expect(error.mock.calls.join('\n')).toContain('a.ts: 3 thing(s), baseline allows 2.')
+ expect(runCountRatchet(ratchet([['a.ts', 2]]), root)).toBe(1)
+ expect(error.mock.calls.join('\n')).toContain('b.ts: now 0, baseline 1.')
+ expect(
+ runCountRatchet(
+ ratchet([
+ ['b.ts', 1],
+ ['a.ts', 2]
+ ]),
+ root
+ )
+ ).toBe(0)
+ })
+
+ it('prunes rows down only, keeping their notes', () => {
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ expect(
+ runCountRatchet(
+ ratchet([
+ ['a.ts', 1],
+ ['c.ts', 4]
+ ]),
+ root,
+ '--prune'
+ )
+ ).toBe(0)
+ expect(readFileSync(join(root, 'baseline.txt'), 'utf8')).toBe('# header\n1 a.ts # why\n')
+ })
+})
+
describe('count baseline', () => {
it('round-trips through format and parse, dropping zero rows', () => {
const counts = new Map([
diff --git a/config/scripts/rebuild-native-deps.mjs b/config/scripts/rebuild-native-deps.mjs
index ac9cf10bc962..519ece7a6e8a 100644
--- a/config/scripts/rebuild-native-deps.mjs
+++ b/config/scripts/rebuild-native-deps.mjs
@@ -31,6 +31,12 @@ import {
windowsProcessTreeAddonPath
} from './windows-process-tree-gyp-rebuild.mjs'
import { disableMsbuildFileTrackingOnWindows } from './msbuild-file-tracking.mjs'
+import { warnAboutNodeModulesAboveSharedStore } from './store-ancestor-node-modules.mjs'
+import {
+ isSharedNativePackage,
+ nodePtyBuildArtifactPaths,
+ resolveNativePackageDir
+} from './native-build-entry.cjs'
import {
copyFileSync,
existsSync,
@@ -89,12 +95,18 @@ const NATIVE_MODULES = [
: [])
]
const onlyModules = NATIVE_MODULES.filter((m) => !ignoreModules.includes(m))
+// Why: these package dirs are shared with other worktrees (pnpm's global virtual store), so they
+// get a published per-ABI build beside build/Release instead of an in-place rebuild.
+const sharedModules = new Set(onlyModules.filter((m) => isSharedNativePackage(m, projectDir)))
+// Loaded only when needed, so a normal install runs exactly the code it always did.
+const sharedNativeBuild = sharedModules.size > 0 ? await import('./shared-native-build.mjs') : null
/** Whether this rebuild targets something other than the machine running it. */
const isCrossHostRebuild = rebuildPlatform !== osPlatform() || rebuildArch !== process.arch
const forceRebuild =
process.env.ORCA_FORCE_NATIVE_REBUILD === '1' || cliOptions.force || isCrossHostRebuild
let modulesToRebuild = onlyModules
+warnAboutNodeModulesAboveSharedStore(projectDir)
ensureElectronPackageInstalled()
restoreNodePtyWindowsConptyRuntime()
@@ -159,6 +171,10 @@ try {
// the catch below is written for. Outside, it aborted `pnpm install` with a
// raw stack instead of the "close running Orca/Electron processes" message.
assertNodePtyConptySourceDeniesMsysBreakaway()
+ for (const moduleName of modulesToRebuild.filter((m) => sharedModules.has(m))) {
+ buildSharedElectronModule(moduleName)
+ }
+ modulesToRebuild = modulesToRebuild.filter((m) => !sharedModules.has(m))
if (
rebuildPlatform === 'win32' &&
modulesToRebuild.includes('@vscode/windows-process-tree') &&
@@ -170,20 +186,22 @@ try {
}
}
disableMsbuildFileTrackingOnWindows()
- await rebuild({
- buildPath: projectDir,
- electronVersion,
- platform: rebuildPlatform,
- arch: rebuildArch,
- ignoreModules,
- onlyModules: modulesToRebuild,
- // Why: without force, @electron/rebuild skips modules it considers
- // "already built" — even when they were compiled for the wrong ABI
- // (e.g., system Node instead of Electron's embedded Node). This is
- // common after pnpm install, which compiles native modules for system
- // Node before postinstall runs this script.
- force: true
- })
+ if (modulesToRebuild.length > 0) {
+ await rebuild({
+ buildPath: projectDir,
+ electronVersion,
+ platform: rebuildPlatform,
+ arch: rebuildArch,
+ ignoreModules,
+ onlyModules: modulesToRebuild,
+ // Why: without force, @electron/rebuild skips modules it considers
+ // "already built" — even when they were compiled for the wrong ABI
+ // (e.g., system Node instead of Electron's embedded Node). This is
+ // common after pnpm install, which compiles native modules for system
+ // Node before postinstall runs this script.
+ force: true
+ })
+ }
restoreNodePtyWindowsConptyRuntime()
assertWindowsProcessTreeAddonIsPatched()
assertNodePtyConptyDeniesMsysBreakaway()
@@ -282,8 +300,29 @@ function assertNodePtyConptySourceDeniesMsysBreakaway() {
})
}
+function buildSharedElectronModule(moduleName) {
+ const target = { platform: rebuildPlatform, arch: rebuildArch }
+ // An N-API build serves Electron and Node alike, so either can vouch for it on this host.
+ const executable = electronPackageIsUsable() ? getElectronExecutablePath() : process.execPath
+ sharedNativeBuild.ensureSharedNativeBuild({
+ moduleName,
+ packageDir: resolveNativePackageDir(moduleName, projectDir),
+ ...target,
+ checkBuildDir: (dir) =>
+ isCrossHostRebuild
+ ? sharedNativeBuild.checkNativeBuildDirArtifacts(moduleName, dir, target)
+ : sharedNativeBuild.checkNativeBuildDirLoads(moduleName, dir, { executable }),
+ log: (message) => console.log(`[rebuild] ${message}`)
+ })
+}
+
function restoreNodePtyWindowsConptyRuntime() {
- if (rebuildPlatform !== 'win32' || !onlyModules.includes('node-pty')) {
+ // A shared node-pty's published build carries its own ConPTY runtime.
+ if (
+ rebuildPlatform !== 'win32' ||
+ !onlyModules.includes('node-pty') ||
+ sharedModules.has('node-pty')
+ ) {
return
}
@@ -534,21 +573,10 @@ function getPatchedNodePtyRebuildReason() {
// Why: Orca patches node-pty's native Unix spawn path and Windows job-object
// exports; upstream prebuilds can load while missing those patches.
- const nodePtyDir = resolve(projectDir, 'node_modules', 'node-pty')
- const artifactPaths =
- rebuildPlatform === 'win32'
- ? [
- resolve(nodePtyDir, 'build', 'Release', 'conpty.node'),
- ...NODE_PTY_CONPTY_RUNTIME_FILES.map((filename) =>
- resolve(nodePtyDir, 'build', 'Release', 'conpty', filename)
- )
- ]
- : [
- resolve(nodePtyDir, 'build', 'Release', 'pty.node'),
- ...(osPlatform() === 'darwin'
- ? [resolve(nodePtyDir, 'build', 'Release', 'spawn-helper')]
- : [])
- ]
+ const artifactPaths = nodePtyBuildArtifactPaths(
+ resolve(projectDir, 'node_modules', 'node-pty'),
+ rebuildPlatform
+ )
const missingArtifact = artifactPaths.find((artifactPath) => !existsSync(artifactPath))
if (!missingArtifact) {
@@ -583,9 +611,11 @@ function probeElectronNativeModules(moduleNames) {
const probeSource = `
const { createRequire } = require('node:module')
const { existsSync } = require('node:fs')
-const { release } = require('node:os')
const { resolve } = require('node:path')
const projectRequire = createRequire(resolve(process.cwd(), 'package.json'))
+const { isSourceBuildDir, nodePtyNativeModuleName: getNodePtyNativeModuleName } = projectRequire(
+ './config/scripts/native-build-entry.cjs'
+)
const moduleNames = ${JSON.stringify(moduleNames)}
const requirePatchedNodePtySourceBuild = ${JSON.stringify(requiresPatchedNodePtySourceBuild())}
const failures = []
@@ -623,17 +653,14 @@ function loadNativeModule(moduleName) {
const { loadNativeModule } = projectRequire('node-pty/lib/utils')
const nativeName = getNodePtyNativeModuleName()
const native = loadNativeModule(nativeName)
- assertNodePtyWindowsConptyRuntime(native.dir)
- assertNodePtyJobOwnership({
- nativeName,
+ const addonPath = nodePtyAddonPath(
+ projectRequire.resolve('node-pty/lib/utils'),
native,
- addonPath: nodePtyAddonPath(
- projectRequire.resolve('node-pty/lib/utils'),
- native,
- nativeName
- )
- })
- if (requirePatchedNodePtySourceBuild && !isNodePtyReleaseBuildDir(native.dir)) {
+ nativeName
+ )
+ assertNodePtyWindowsConptyRuntime(native.dir, require('node:path').dirname(addonPath))
+ assertNodePtyJobOwnership({ nativeName, native, addonPath })
+ if (requirePatchedNodePtySourceBuild && !isSourceBuildDir(native.dir)) {
throw new Error(
'node-pty resolved to ' +
native.dir +
@@ -654,18 +681,11 @@ function loadNativeModule(moduleName) {
projectRequire(moduleName)
}
-function assertNodePtyWindowsConptyRuntime(nativeDir) {
- if (process.platform !== 'win32' || !isNodePtyReleaseBuildDir(nativeDir)) {
+function assertNodePtyWindowsConptyRuntime(nativeDir, loadedDir) {
+ if (process.platform !== 'win32' || !isSourceBuildDir(nativeDir)) {
return
}
- const runtimeDir = resolve(
- process.cwd(),
- 'node_modules',
- 'node-pty',
- 'build',
- 'Release',
- 'conpty'
- )
+ const runtimeDir = resolve(loadedDir, 'conpty')
for (const filename of ${JSON.stringify(NODE_PTY_CONPTY_RUNTIME_FILES)}) {
const runtimeFile = resolve(runtimeDir, filename)
if (!existsSync(runtimeFile)) {
@@ -674,18 +694,6 @@ function assertNodePtyWindowsConptyRuntime(nativeDir) {
}
}
-function isNodePtyReleaseBuildDir(nativeDir) {
- return typeof nativeDir === 'string' && nativeDir.replace(/\\\\/g, '/').includes('build/Release/')
-}
-
-function getNodePtyNativeModuleName() {
- if (process.platform !== 'win32') {
- return 'pty'
- }
- const match = /(\\d+)\\.(\\d+)\\.(\\d+)/g.exec(release())
- const buildNumber = match && match.length === 4 ? Number.parseInt(match[3], 10) : 0
- return buildNumber >= 18309 ? 'conpty' : 'pty'
-}
function formatError(error) {
return error instanceof Error ? error.message : String(error)
diff --git a/config/scripts/release-cut-token-permissions.test.mjs b/config/scripts/release-cut-token-permissions.test.mjs
index ded0d7f9579c..1cb41124b034 100644
--- a/config/scripts/release-cut-token-permissions.test.mjs
+++ b/config/scripts/release-cut-token-permissions.test.mjs
@@ -24,6 +24,21 @@ const EXPECTED_MATRIX = {
'.github/workflows/node-server-tests.yml#mobile_web_bundle': { contents: 'read' },
'.github/workflows/node-server-tests.yml#persistence': { contents: 'read' },
'.github/workflows/release-mac-build.yml#build-mac': { actions: 'read', contents: 'write' },
+ '.github/workflows/release-mac-build.yml#qualification-template': { contents: 'read' },
+ ...Object.fromEntries(
+ [
+ 'changes',
+ 'desktop_template',
+ 'linux_glibc217_compat',
+ 'linux_glibc_floor',
+ 'linux_musl',
+ 'mobile_web_bundle',
+ 'persistence'
+ ].map((job) => [
+ `.github/workflows/release-mac-build.yml#qualification-template -> .github/workflows/node-server-tests.yml#${job}`,
+ { contents: 'read' }
+ ])
+ ),
[`${RELEASE_WORKFLOW}#build`]: { actions: 'read', contents: 'write' },
[`${RELEASE_WORKFLOW}#build-mac`]: { actions: 'write', contents: 'read' },
[`${RELEASE_WORKFLOW}#verify-mac-build`]: { actions: 'read', contents: 'read' },
@@ -243,8 +258,12 @@ describe('release-cut token permissions', () => {
}
const macWorkflow = readWorkflow('.github/workflows/release-mac-build.yml')
- expect(macWorkflow.jobs['build-mac'].if).toBe("github.repository == 'zpyoung/orca'")
- expect(checkoutRef(macWorkflow.jobs['build-mac'])).toBe('refs/tags/${{ inputs.tag }}')
+ expect(macWorkflow.jobs['build-mac'].if).toBe(
+ "${{ !cancelled() && (!inputs.qualification_only || needs.qualification-template.result == 'success') && github.repository == 'zpyoung/orca' }}"
+ )
+ expect(checkoutRef(macWorkflow.jobs['build-mac'])).toBe(
+ "${{ inputs.qualification_only && github.sha || format('refs/tags/{0}', inputs.tag) }}"
+ )
const e2eWorkflow = readWorkflow('.github/workflows/e2e.yml')
for (const jobName of Object.keys(e2eWorkflow.jobs)) {
diff --git a/config/scripts/release-desktop-timing.mjs b/config/scripts/release-desktop-timing.mjs
new file mode 100644
index 000000000000..c6d222fbd52b
--- /dev/null
+++ b/config/scripts/release-desktop-timing.mjs
@@ -0,0 +1,168 @@
+import assert from 'node:assert/strict'
+import { readFileSync, writeFileSync } from 'node:fs'
+import { pathToFileURL } from 'node:url'
+
+function timestamp(value, label) {
+ const result = Date.parse(value)
+ assert(Number.isFinite(result), `Missing or invalid ${label}`)
+ return result
+}
+
+function startedStep(job) {
+ const starts = job.steps
+ .filter((step) => step.status !== 'skipped' && step.conclusion !== 'skipped')
+ .map((step) => Date.parse(step.startedAt ?? step.started_at))
+ .filter(Number.isFinite)
+ assert(starts.length > 0, `No started steps for ${job.name}`)
+ return Math.min(...starts)
+}
+
+function interval(job) {
+ const start = startedStep(job)
+ const end = timestamp(job.completedAt ?? job.completed_at, `${job.name} completion`)
+ assert(end >= start, `Completion precedes first step for ${job.name}`)
+ return { start, end }
+}
+
+function seconds(milliseconds) {
+ return Math.round(milliseconds) / 1000
+}
+
+function phaseSteps(jobs, pattern) {
+ const matches = jobs.flatMap((job) =>
+ job.steps
+ .filter((step) => step.conclusion === 'success' && pattern.test(step.name))
+ .map((step) => ({
+ job: job.name,
+ step: step.name,
+ seconds: seconds(
+ timestamp(step.completedAt ?? step.completed_at, `${step.name} completion`) -
+ timestamp(step.startedAt ?? step.started_at, `${step.name} start`)
+ )
+ }))
+ )
+ return {
+ seconds: matches.length ? matches.reduce((sum, step) => sum + step.seconds, 0) : null,
+ steps: matches
+ }
+}
+
+function overlapSeconds(mac, windows) {
+ const segments = windows
+ .map((win) => ({ start: Math.max(mac.start, win.start), end: Math.min(mac.end, win.end) }))
+ .filter(({ start, end }) => start < end)
+ .sort((left, right) => left.start - right.start)
+ const merged = []
+ for (const segment of segments) {
+ const previous = merged.at(-1)
+ if (previous && previous.end >= segment.start) {
+ previous.end = Math.max(previous.end, segment.end)
+ } else {
+ merged.push({ ...segment })
+ }
+ }
+ return seconds(merged.reduce((sum, { start, end }) => sum + end - start, 0))
+}
+
+export function summarizeReleaseDesktopTiming(run) {
+ assert(
+ run.status === 'completed' && run.conclusion === 'success',
+ 'Require a completed successful run'
+ )
+ const jobs = run.jobs.filter((job) => job.conclusion !== 'skipped')
+ const mac = jobs.find((job) => job.name === 'build-adhoc-mac')
+ const windows = jobs.filter((job) => /^(?:build|stage|publish)-adhoc-win(?:\b|\/)/.test(job.name))
+ const template = jobs.find((job) => job.name === 'orcad-template / desktop_template')
+ assert(mac && windows.length > 0 && template, 'Missing Mac, Windows, or qualified template job')
+ assert(
+ [mac, ...windows, template].every((job) => job.conclusion === 'success'),
+ 'Desktop or template job failed'
+ )
+ const qualification = jobs.filter((job) => job.name.startsWith('orcad-template / '))
+ const macInterval = interval(mac)
+ const windowsIntervals = windows.map(interval)
+ const desktopStart = Math.min(macInterval.start, ...windowsIntervals.map(({ start }) => start))
+ const desktopEnd = Math.max(macInterval.end, ...windowsIntervals.map(({ end }) => end))
+ const created = timestamp(run.createdAt ?? run.created_at, 'run creation')
+ const templateEnd = interval(template).end
+ return {
+ source: { url: run.url ?? run.html_url ?? null, headSha: run.headSha ?? run.head_sha ?? null },
+ measuredSeconds: {
+ creationToLastJob: seconds(Math.max(...jobs.map((job) => interval(job).end)) - created),
+ qualification: seconds(templateEnd - Math.min(...qualification.map(startedStep))),
+ desktop: seconds(desktopEnd - desktopStart),
+ macJob: seconds(macInterval.end - macInterval.start),
+ windowsJobTotal: seconds(
+ windowsIntervals.reduce((sum, { start, end }) => sum + end - start, 0)
+ ),
+ desktopJobOverlap: overlapSeconds(macInterval, windowsIntervals),
+ templateReadyToMacFirstStep: seconds(macInterval.start - templateEnd),
+ templateReadyToWindowsFirstStep: seconds(
+ Math.min(...windowsIntervals.map(({ start }) => start)) - templateEnd
+ )
+ },
+ windowsPhases: {
+ dependencies: phaseSteps(windows, /^Install dependencies$/),
+ appBuild: phaseSteps(windows, /^Build app$/),
+ packagingAndUpload: phaseSteps(
+ windows,
+ /^(?:Package Windows artifacts(?: without publishing)?|Publish Windows artifacts)$/
+ ),
+ artifactHandoff: phaseSteps(
+ windows,
+ /^(?:Stage Windows artifacts for gated publishing|Download the staged Windows artifacts)$/
+ )
+ },
+ desktopJobs: [mac, ...windows].map((job) => {
+ const { start, end } = interval(job)
+ return {
+ name: job.name,
+ firstStepAt: new Date(start).toISOString(),
+ completedAt: new Date(end).toISOString(),
+ reportedStartToFirstStepSeconds: seconds(
+ start - timestamp(job.startedAt ?? job.started_at, `${job.name} reported start`)
+ )
+ }
+ })
+ }
+}
+
+export function compareReleaseDesktopTiming(baseline, candidate) {
+ const before = summarizeReleaseDesktopTiming(baseline)
+ const after = summarizeReleaseDesktopTiming(candidate)
+ return {
+ methodology: [
+ 'Measured timestamps from two successful GitHub Actions runs; no modeled speedup.',
+ 'Job intervals begin at the first started step, not the reported job start.',
+ 'Creation-to-last-job includes scheduling and runner waits; desktop includes teardown and publication.',
+ 'Template-to-first-step gaps include dependency scheduling and runner waits, not queue time alone.',
+ 'Overlap is the union of Windows job intervals intersecting the Mac job, including setup and teardown.',
+ 'Different source commits, caches, and runner load can affect a two-run comparison.'
+ ],
+ baseline: before,
+ candidate: after,
+ measuredSavingsSeconds: Object.fromEntries(
+ Object.entries(before.measuredSeconds).map(([key, value]) => [
+ key,
+ seconds((value - after.measuredSeconds[key]) * 1000)
+ ])
+ )
+ }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [baselineFile, candidateFile, outputFile] = process.argv.slice(2)
+ assert(
+ baselineFile && candidateFile && process.argv.length <= 5,
+ 'Usage: node config/scripts/release-desktop-timing.mjs baseline.json candidate.json [report.json]'
+ )
+ const report = compareReleaseDesktopTiming(
+ JSON.parse(readFileSync(baselineFile, 'utf8')),
+ JSON.parse(readFileSync(candidateFile, 'utf8'))
+ )
+ const json = `${JSON.stringify(report, null, 2)}\n`
+ if (outputFile) {
+ writeFileSync(outputFile, json)
+ }
+ process.stdout.write(json)
+}
diff --git a/config/scripts/release-desktop-timing.test.mjs b/config/scripts/release-desktop-timing.test.mjs
new file mode 100644
index 000000000000..8ab947f46abe
--- /dev/null
+++ b/config/scripts/release-desktop-timing.test.mjs
@@ -0,0 +1,120 @@
+import { describe, expect, it } from 'vitest'
+import {
+ compareReleaseDesktopTiming,
+ summarizeReleaseDesktopTiming
+} from './release-desktop-timing.mjs'
+
+const at = (seconds) => new Date(Date.UTC(2026, 0, 1) + seconds * 1000).toISOString()
+
+function job(name, start, end, firstStep = start, steps = []) {
+ return {
+ name,
+ conclusion: 'success',
+ startedAt: at(start),
+ completedAt: at(end),
+ steps: [
+ {
+ name: 'Set up job',
+ conclusion: 'success',
+ startedAt: at(firstStep),
+ completedAt: at(firstStep + 1)
+ },
+ ...steps
+ ]
+ }
+}
+
+function run(windows, macStart = 20, macEnd = 100) {
+ return {
+ status: 'completed',
+ conclusion: 'success',
+ createdAt: at(0),
+ jobs: [
+ job('orcad-template-support', 2, 3),
+ job('orcad-template / desktop_template', 3, 10),
+ job('build-adhoc-mac', macStart, macEnd),
+ ...windows
+ ]
+ }
+}
+
+describe('release desktop timing evidence', () => {
+ it('measures the sequential baseline and actual candidate overlap without counting queue timestamps as execution', () => {
+ const baseline = run([job('build-adhoc-win / build-win', 100, 160, 110)])
+ const candidate = run([
+ job('build-adhoc-win / build-win', 10, 75, 25),
+ job('publish-adhoc-win / publish-win', 100, 110, 101)
+ ])
+ const report = compareReleaseDesktopTiming(baseline, candidate)
+ expect(report.baseline.measuredSeconds.desktopJobOverlap).toBe(0)
+ expect(report.candidate.measuredSeconds.desktopJobOverlap).toBe(50)
+ expect(report.candidate.measuredSeconds.desktop).toBe(90)
+ expect(report.measuredSavingsSeconds.desktop).toBe(50)
+ expect(report.candidate.desktopJobs[1].reportedStartToFirstStepSeconds).toBe(15)
+ expect(report.candidate.measuredSeconds.templateReadyToWindowsFirstStep).toBe(15)
+ })
+
+ it('unions intersecting Windows intervals and ignores skipped jobs and skipped step timestamps', () => {
+ const windows = job('build-adhoc-win / build-win', 10, 80, 30)
+ windows.steps.unshift({ name: 'Skipped', conclusion: 'skipped', startedAt: at(10) })
+ const input = run([windows, job('publish-adhoc-win / publish-win', 70, 90)])
+ input.jobs.push({ name: 'build-adhoc-win-old', conclusion: 'skipped', steps: [] })
+ const report = summarizeReleaseDesktopTiming(input)
+ expect(report.measuredSeconds.desktopJobOverlap).toBe(60)
+ expect(report.measuredSeconds.windowsJobTotal).toBe(70)
+ })
+
+ it('retains measured step evidence and leaves missing phases unknown', () => {
+ const input = run([
+ job('build-adhoc-win / build-win', 100, 170, 100, [
+ { name: 'Build app', conclusion: 'success', startedAt: at(101), completedAt: at(131) },
+ {
+ name: 'Publish Windows artifacts',
+ conclusion: 'success',
+ startedAt: at(131),
+ completedAt: at(160)
+ }
+ ])
+ ])
+ const report = summarizeReleaseDesktopTiming(input)
+ expect(report.windowsPhases.appBuild.seconds).toBe(30)
+ expect(report.windowsPhases.packagingAndUpload.seconds).toBe(29)
+ expect(report.windowsPhases.dependencies.seconds).toBeNull()
+ expect(report.windowsPhases.appBuild.steps[0].job).toBe('build-adhoc-win / build-win')
+ })
+
+ it('accounts for packaging and the extra artifact handoff in a split Windows build', () => {
+ const step = (name, start, end) => ({
+ name,
+ conclusion: 'success',
+ startedAt: at(start),
+ completedAt: at(end)
+ })
+ const report = summarizeReleaseDesktopTiming(
+ run([
+ job('stage-adhoc-win / build-win', 20, 80, 20, [
+ step('Package Windows artifacts without publishing', 30, 60),
+ step('Stage Windows artifacts for gated publishing', 60, 70)
+ ]),
+ job('publish-adhoc-win / publish-win', 100, 130, 100, [
+ step('Download the staged Windows artifacts', 101, 111),
+ step('Publish Windows artifacts', 111, 125)
+ ])
+ ])
+ )
+ expect(report.windowsPhases.packagingAndUpload.seconds).toBe(44)
+ expect(report.windowsPhases.artifactHandoff.seconds).toBe(20)
+ })
+
+ it('rejects incomplete runs, failed desktop jobs, and missing timing evidence', () => {
+ const input = run([job('build-adhoc-win / build-win', 100, 160)])
+ expect(() => summarizeReleaseDesktopTiming({ ...input, status: 'in_progress' })).toThrow(
+ 'completed successful run'
+ )
+ input.jobs.at(-1).conclusion = 'failure'
+ expect(() => summarizeReleaseDesktopTiming(input)).toThrow('Desktop or template job failed')
+ input.jobs.at(-1).conclusion = 'success'
+ input.jobs.at(-1).steps = []
+ expect(() => summarizeReleaseDesktopTiming(input)).toThrow('No started steps')
+ })
+})
diff --git a/config/scripts/release-javascript-artifact.mjs b/config/scripts/release-javascript-artifact.mjs
index 5a7f610594b5..a04cf2445ef4 100644
--- a/config/scripts/release-javascript-artifact.mjs
+++ b/config/scripts/release-javascript-artifact.mjs
@@ -24,6 +24,7 @@ const REQUIRED_FILES = [
'preload/index.js',
'renderer/index.html',
'renderer/.vite/manifest.json',
+ 'renderer/.orca-web-client-files.json',
'web/web-index.html',
'mobile-web/manifest.json',
'package.json'
diff --git a/config/scripts/release-javascript-artifact.test.mjs b/config/scripts/release-javascript-artifact.test.mjs
index 420311b90f89..c98efdb54563 100644
--- a/config/scripts/release-javascript-artifact.test.mjs
+++ b/config/scripts/release-javascript-artifact.test.mjs
@@ -13,7 +13,7 @@ const directories = []
const configuration = {
sourceSha: 'a'.repeat(40),
version: '1.2.3-rc.4',
- packageManager: 'pnpm@12.8.1',
+ packageManager: 'pnpm@12.14.0',
identity: 'rc',
diagnosticsTokenUrl: 'https://example.test/diagnostics',
telemetryKeySha256: 'b'.repeat(64)
@@ -34,6 +34,7 @@ function fixture() {
'preload/index.js',
'renderer/index.html',
'renderer/.vite/manifest.json',
+ 'renderer/.orca-web-client-files.json',
'web/web-index.html',
'mobile-web/manifest.json',
'package.json',
diff --git a/config/scripts/release-windows-signing-gate.test.mjs b/config/scripts/release-windows-signing-gate.test.mjs
new file mode 100644
index 000000000000..1c6f08902c8a
--- /dev/null
+++ b/config/scripts/release-windows-signing-gate.test.mjs
@@ -0,0 +1,115 @@
+import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { runInNewContext } from 'node:vm'
+import { runProcess } from '@orca/process-host'
+import { describe, expect, it } from 'vitest'
+import { parse } from 'yaml'
+
+const { jobs } = parse(readFileSync('.github/workflows/release-cut.yml', 'utf8'))
+const { steps } = jobs.build
+const gate = steps.find((step) => step.id === 'mac-signing-gate')
+const submissions = steps.filter((step) =>
+ step.uses?.startsWith('signpath/github-action-submit-signing-request@')
+)
+
+function signingAllowed(submission, outcome, attempt = 1, platform = 'win', uploaded = 'success') {
+ const expression = submission.if.replace(/steps\.([a-z][a-z0-9-]*)/g, 'steps["$1"]')
+ return runInNewContext(expression, {
+ matrix: { platform },
+ github: { run_attempt: attempt },
+ steps: {
+ 'mac-signing-gate': { outcome },
+ 'upload-unsigned-inner': { outcome: uploaded },
+ 'upload-unsigned-windows-installer': { outcome: uploaded }
+ }
+ })
+}
+
+describe('Windows release signing waits for macOS success', () => {
+ it('packages concurrently, then verifies the exact dispatched macOS run before signing', () => {
+ expect(jobs.build.needs).toContain('build-mac')
+ // the fork releases macOS only, so its Windows/Linux build job never runs
+ expect(jobs.build.if).toBe(false)
+ expect(jobs.build.needs).not.toContain('verify-mac-build')
+ expect(jobs['build-mac'].needs).not.toContain('build')
+ expect(jobs['verify-mac-build'].needs).toContain('build')
+ const packaging = steps.findIndex((step) => step.name === 'Build Windows release artifacts')
+ const gateIndex = steps.indexOf(gate)
+ expect(packaging).toBeGreaterThanOrEqual(0)
+ expect(gateIndex).toBeGreaterThan(packaging)
+ expect(gate.run).toBe('node config/scripts/run-release-mac-build-workflow.mjs wait')
+ expect(gate.if).toBe("matrix.platform == 'win' && github.run_attempt == 1")
+ expect(gate['continue-on-error']).toBeUndefined()
+ expect(gate.env).toEqual(
+ jobs['verify-mac-build'].steps.find((step) => step.run === gate.run).env
+ )
+ expect(gate.env.RELEASE_MAC_BUILD_RUN_ID).toBe('${{ needs.build-mac.outputs.run_id }}')
+ expect(gate.env.RELEASE_MAC_BUILD_WORKFLOW_SHA).toBe(
+ '${{ needs.build-mac.outputs.workflow_sha }}'
+ )
+ expect(gate.env.RELEASE_MAC_BUILD_RUN_ATTEMPT).toBe(
+ '${{ needs.build-mac.outputs.run_attempt }}'
+ )
+ expect(gate.env.RELEASE_MAC_BUILD_WAIT_DEADLINE_MS).toBe(
+ '${{ needs.build-mac.outputs.wait_deadline_ms }}'
+ )
+ expect(submissions).toHaveLength(2)
+ for (const submission of submissions) {
+ expect(steps.indexOf(submission)).toBeGreaterThan(gateIndex)
+ }
+ })
+
+ it('allows neither signing request without successful macOS verification and upload', () => {
+ for (const submission of submissions) {
+ expect(signingAllowed(submission, 'success')).toBe(true)
+ for (const outcome of ['failure', 'cancelled', 'skipped', '', undefined]) {
+ expect(signingAllowed(submission, outcome), `${submission.id}: ${outcome}`).toBe(false)
+ expect(signingAllowed(submission, 'success', 1, 'win', outcome ?? null)).toBe(false)
+ }
+ for (const attempt of [2, 3]) {
+ expect(signingAllowed(submission, 'success', attempt)).toBe(false)
+ }
+ for (const platform of ['linux-x64', 'linux-arm64']) {
+ expect(signingAllowed(submission, 'success', 1, platform)).toBe(false)
+ }
+ }
+ })
+
+ it('restores the verifier from the workflow commit when releasing older sources', () => {
+ const restore = steps.find((step) =>
+ step.run?.includes(
+ 'git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs'
+ )
+ )
+ expect(restore.env.WORKFLOW_SHA).toBe('${{ github.workflow_sha }}')
+ expect(restore.run).toContain('config/scripts/run-release-mac-build-workflow.mjs')
+ expect(steps.indexOf(restore)).toBeLessThan(steps.indexOf(gate))
+ })
+
+ it('runs the verifier CLI from paths requiring URL encoding, as Windows paths do', async () => {
+ const directory = realpathSync(mkdtempSync(join(tmpdir(), 'orca signing gate ')))
+ const script = join(directory, 'mac verifier.mjs')
+ writeFileSync(script, readFileSync('config/scripts/run-release-mac-build-workflow.mjs'))
+ try {
+ const result = await runProcess({
+ program: process.execPath,
+ args: [script, 'wait'],
+ env: {
+ ...process.env,
+ ORCA_BACKGROUND_LAUNCH: '1',
+ RELEASE_MAC_BUILD_REF: 'main',
+ GITHUB_REPOSITORY: 'stablyai/orca',
+ GITHUB_RUN_ID: '777',
+ GITHUB_TOKEN: 'test-token',
+ RELEASE_MAC_BUILD_TAG: 'v1.2.3',
+ RELEASE_MAC_BUILD_RUN_ID: ''
+ }
+ })
+ expect(result.code).toBe(1)
+ expect(result.stderr).toContain('RELEASE_MAC_BUILD_RUN_ID is required')
+ } finally {
+ rmSync(directory, { recursive: true, force: true })
+ }
+ })
+})
diff --git a/config/scripts/run-e2e-typecheck.mjs b/config/scripts/run-e2e-typecheck.mjs
new file mode 100644
index 000000000000..36ef063862b2
--- /dev/null
+++ b/config/scripts/run-e2e-typecheck.mjs
@@ -0,0 +1,58 @@
+import { availableParallelism } from 'node:os'
+import { runProcessSync } from '@orca/process-host'
+import { resolvePinnedBunExecutable } from './pinned-bun-executable.mjs'
+import { describeProcessFailure } from './process-failure-message.mjs'
+import { isDirectInvocation } from './script-entry-detection.mjs'
+import {
+ resolveTypecheckCommand,
+ selectTypecheckCompiler
+} from './run-typecheck-projects-in-parallel.mjs'
+
+const CONFIG = 'config/tsconfig.e2e.json'
+
+export function runE2eTypecheck({
+ root = process.cwd(),
+ compiler = 'bun',
+ execute = runProcessSync,
+ resolveBunExecutable = (directory) => resolvePinnedBunExecutable(directory, runProcessSync)
+} = {}) {
+ const selection = selectTypecheckCompiler(compiler, { root, resolveBunExecutable })
+ const command = resolveTypecheckCommand(CONFIG, {
+ ...selection,
+ root,
+ bunThreads: availableParallelism()
+ })
+ const result = execute({
+ ...command,
+ args: [...command.args, ...(compiler === 'native' ? ['--pretty', 'false'] : [])],
+ timeoutMs: null,
+ maxOutputBytes: 64 * 1024 * 1024
+ })
+ const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
+ const diagnosticExit = result.code === 1 || (compiler === 'native' && result.code === 2)
+ // Incomplete diagnostics must never look like fixes when pruning the baseline.
+ if (
+ result.signal ||
+ result.timedOut ||
+ result.outputTruncated ||
+ (result.code !== 0 && (!diagnosticExit || !/error TS\d+:/.test(output)))
+ ) {
+ throw new Error(`${compiler} ${CONFIG} failed to run: ${describeProcessFailure(result)}`)
+ }
+ return { code: result.code, output }
+}
+
+if (isDirectInvocation(import.meta.url, process.argv[1])) {
+ try {
+ const args = process.argv.slice(2)
+ if (args.length > 1 || (args.length === 1 && args[0] !== '--compiler=native')) {
+ throw new Error('Usage: run-e2e-typecheck.mjs [--compiler=native]')
+ }
+ const result = runE2eTypecheck({ compiler: args.length ? 'native' : 'bun' })
+ process.stdout.write(result.output)
+ process.exitCode = result.code
+ } catch (error) {
+ console.error(error.message)
+ process.exitCode = 1
+ }
+}
diff --git a/config/scripts/run-e2e-typecheck.test.mjs b/config/scripts/run-e2e-typecheck.test.mjs
new file mode 100644
index 000000000000..9c4fce4263ed
--- /dev/null
+++ b/config/scripts/run-e2e-typecheck.test.mjs
@@ -0,0 +1,119 @@
+import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { availableParallelism, tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { runProcessSync } from '@orca/process-host'
+import { afterEach, expect, it, vi } from 'vitest'
+import { countTestTypeErrors } from './check-e2e-typecheck-ratchet.mjs'
+import { resolvePinnedBunExecutable } from './pinned-bun-executable.mjs'
+import { runE2eTypecheck } from './run-e2e-typecheck.mjs'
+
+const ROOT = process.cwd()
+const resolveBunExecutable = () => resolvePinnedBunExecutable(ROOT, runProcessSync)
+const directories = []
+const diagnostic = 'tests/entry.ts(1,1): error TS2322: Type error.\n'
+
+afterEach(() => {
+ for (const directory of directories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
+function fixture() {
+ const root = mkdtempSync(join(tmpdir(), 'orca-e2e-check-'))
+ directories.push(root)
+ mkdirSync(join(root, 'config'))
+ mkdirSync(join(root, 'tests'))
+ symlinkSync(resolve('node_modules'), join(root, 'node_modules'), 'junction')
+ writeFileSync(join(root, 'package.json'), '{"type":"commonjs"}')
+ writeFileSync(
+ join(root, 'config', 'tsconfig.e2e.json'),
+ JSON.stringify({
+ compilerOptions: {
+ module: 'Node20',
+ moduleResolution: 'Node16',
+ target: 'ES2022',
+ strict: true,
+ types: []
+ },
+ include: ['../tests/**/*.ts']
+ })
+ )
+ writeFileSync(join(root, 'tests', 'exports.ts'), 'export const value = 1\n')
+ writeFileSync(
+ join(root, 'tests', 'entry.ts'),
+ "import { missing } from './exports'\nexport const value: number = 'wrong'\n"
+ )
+ return root
+}
+
+it('captures all diagnostics using the pinned Bun and all available CPU cores', () => {
+ const execute = vi.fn(() => ({ code: 1, stdout: diagnostic, stderr: 'summary' }))
+ expect(runE2eTypecheck({ execute, resolveBunExecutable: () => 'pinned-bun' })).toEqual({
+ code: 1,
+ output: `${diagnostic}summary`
+ })
+ expect(execute).toHaveBeenCalledWith({
+ program: 'pinned-bun',
+ args: [
+ 'check',
+ '-p',
+ 'config/tsconfig.e2e.json',
+ '--threads',
+ String(availableParallelism()),
+ '--no-pretty',
+ '--all'
+ ],
+ cwd: ROOT,
+ timeoutMs: null,
+ maxOutputBytes: 64 * 1024 * 1024
+ })
+})
+
+it.each([
+ { code: 1, stdout: '', stderr: 'failed to start' },
+ { code: null, stdout: diagnostic, signal: 'SIGKILL' },
+ { code: 0, stdout: diagnostic, timedOut: true },
+ { code: 1, stdout: diagnostic, outputTruncated: true },
+ { code: 99, stdout: diagnostic },
+ { code: null, stdout: diagnostic }
+])('rejects failed or incomplete checks before the ratchet can prune errors: %j', (result) => {
+ expect(() =>
+ runE2eTypecheck({
+ execute: () => result,
+ resolveBunExecutable: () => 'pinned-bun'
+ })
+ ).toThrow('failed to run')
+})
+
+it('does not run when Bun is missing or the project compiler version has drifted', () => {
+ const execute = vi.fn()
+ expect(() => runE2eTypecheck({ execute, resolveBunExecutable: () => null })).toThrow('missing')
+ const root = fixture()
+ mkdirSync(join(root, 'config', 'node_modules', 'typescript'), { recursive: true })
+ writeFileSync(
+ join(root, 'config', 'node_modules', 'typescript', 'package.json'),
+ JSON.stringify({ version: '0.0.0', bin: { tsc: './bin/tsc' } })
+ )
+ expect(() =>
+ runE2eTypecheck({
+ root,
+ execute,
+ resolveBunExecutable: () => 'pinned-bun'
+ })
+ ).toThrow('resolves 0.0.0')
+ expect(execute).not.toHaveBeenCalled()
+})
+
+it('keeps real missing-export and type-error diagnostics identical to TypeScript', () => {
+ const root = fixture()
+ const native = runE2eTypecheck({ root, compiler: 'native' })
+ const bun = runE2eTypecheck({ root, resolveBunExecutable })
+ const locations = (output) =>
+ [...output.matchAll(/^.+\(\d+,\d+\): error TS\d+:/gm)].map(([line]) => line).sort()
+ expect(locations(bun.output)).toEqual(locations(native.output))
+ expect(locations(bun.output)).toHaveLength(2)
+ expect(countTestTypeErrors(bun.output)).toEqual(new Map([['tests/entry.ts', 2]]))
+ writeFileSync(join(root, 'tests', 'entry.ts'), "export { value } from './exports'\n")
+ expect(runE2eTypecheck({ root, resolveBunExecutable }).code).toBe(0)
+ expect(runE2eTypecheck({ root, compiler: 'native' }).code).toBe(0)
+})
diff --git a/config/scripts/run-release-mac-build-workflow.mjs b/config/scripts/run-release-mac-build-workflow.mjs
index 4148db46071c..5123b81c3274 100644
--- a/config/scripts/run-release-mac-build-workflow.mjs
+++ b/config/scripts/run-release-mac-build-workflow.mjs
@@ -1,4 +1,5 @@
import { appendFileSync } from 'node:fs'
+import { pathToFileURL } from 'node:url'
const DEFAULT_API_VERSION = '2026-03-10'
const DEFAULT_TIMEOUT_MINUTES = 90
@@ -344,7 +345,7 @@ export async function executeReleaseMacBuildMode(mode, env = process.env, deps =
throw new Error(`Unknown mac release build mode: ${mode}`)
}
-if (import.meta.url === `file://${process.argv[1]}`) {
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
executeReleaseMacBuildMode(process.argv[2]).catch((error) => {
console.error(error)
process.exitCode = 1
diff --git a/config/scripts/run-sandboxed-test-shards.mjs b/config/scripts/run-sandboxed-test-shards.mjs
index f7cc387faedf..b6e866d9ddd1 100644
--- a/config/scripts/run-sandboxed-test-shards.mjs
+++ b/config/scripts/run-sandboxed-test-shards.mjs
@@ -49,6 +49,8 @@ const IMAGE_CONTEXT_ROOTS = [
'.npmrc',
'config/docker/test-sandbox',
'config/patches',
+ // pnpm-workspace.yaml names it, so install fails without it.
+ 'config/pnpmfile.cjs',
'config/scripts',
'package.json',
'pnpm-lock.yaml',
diff --git a/config/scripts/run-ssh-staged-upload-reliability.mjs b/config/scripts/run-ssh-staged-upload-reliability.mjs
index ca17b774bdbc..9262571d6984 100644
--- a/config/scripts/run-ssh-staged-upload-reliability.mjs
+++ b/config/scripts/run-ssh-staged-upload-reliability.mjs
@@ -5,7 +5,7 @@ const defaultFiles = [
'src/main/ssh/ssh-file-transfer-abort.test.ts',
'src/main/ssh/ssh-relay-install-namespace.test.ts',
'src/main/ssh/sftp-namespace-resolution.test.ts',
- 'src/main/ssh/ssh-connection-sftp-wire.test.ts',
+ 'src/main/ssh/connection/ssh-connection-sftp-wire.test.ts',
'src/main/ssh/ssh-remote-commands.test.ts'
]
diff --git a/config/scripts/run-typecheck-projects-in-parallel.mjs b/config/scripts/run-typecheck-projects-in-parallel.mjs
index ebf6389c20b0..267a9facf4ed 100644
--- a/config/scripts/run-typecheck-projects-in-parallel.mjs
+++ b/config/scripts/run-typecheck-projects-in-parallel.mjs
@@ -138,21 +138,27 @@ export function typecheckBatches(projects, { compiler, bunConcurrency, memoryByt
)
}
+export function resolveTypecheckCommand(config, { compiler, bunExecutable, bunThreads, root }) {
+ const installedCompiler = resolveProjectCompilerInfo(config, root)
+ if (compiler === 'bun' && installedCompiler.version !== BUN_TYPESCRIPT_VERSION) {
+ throw new Error(
+ `Bun CI compiler targets TypeScript ${BUN_TYPESCRIPT_VERSION}, but ${config} resolves ${installedCompiler.version}`
+ )
+ }
+ return {
+ program: compiler === 'bun' ? bunExecutable : process.execPath,
+ args:
+ compiler === 'bun'
+ ? ['check', '-p', config, '--threads', String(bunThreads), '--no-pretty', '--all']
+ : [installedCompiler.program, '--noEmit', '-p', config],
+ cwd: root
+ }
+}
+
function checkProject(config, { compiler, bunExecutable, bunThreads, root, spawnCompiler }) {
return new Promise((resolve, reject) => {
- const installedCompiler = resolveProjectCompilerInfo(config, root)
- if (compiler === 'bun' && installedCompiler.version !== BUN_TYPESCRIPT_VERSION) {
- throw new Error(
- `Bun CI compiler targets TypeScript ${BUN_TYPESCRIPT_VERSION}, but ${config} resolves ${installedCompiler.version}`
- )
- }
const child = spawnCompiler({
- program: compiler === 'bun' ? bunExecutable : process.execPath,
- args:
- compiler === 'bun'
- ? ['check', '-p', config, '--threads', String(bunThreads), '--no-pretty', '--all']
- : [installedCompiler.program, '--noEmit', '-p', config],
- cwd: root,
+ ...resolveTypecheckCommand(config, { compiler, bunExecutable, bunThreads, root }),
stdio: 'inherit'
})
child.on('error', reject)
diff --git a/config/scripts/shared-electron-dist-cache.mjs b/config/scripts/shared-electron-dist-cache.mjs
index 0562e926b954..e7757e3490cc 100644
--- a/config/scripts/shared-electron-dist-cache.mjs
+++ b/config/scripts/shared-electron-dist-cache.mjs
@@ -1,8 +1,10 @@
import { execFileSync } from 'node:child_process'
import { randomUUID } from 'node:crypto'
-import { existsSync, lstatSync, mkdirSync, readFileSync, renameSync, rmSync } from 'node:fs'
+import { existsSync, lstatSync, mkdirSync, readFileSync, rmSync } from 'node:fs'
import path from 'node:path'
import { makeTreeReadOnly, shareTree } from './space-sharing-copy.mjs'
+import { publishSharedTree, sharedTreeStagingPath } from './shared-tree-publish.mjs'
+import { isSharedPackageDir } from './native-build-entry.cjs'
const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/
// Sibling of path.txt, never inside dist: an install replaces dist wholesale.
@@ -23,6 +25,11 @@ export function resolveSharedElectronDistEntry(options) {
if (![version, targetPlatform, targetArch].every((part) => IDENTITY_PATTERN.test(part ?? ''))) {
return null
}
+ // Under pnpm's global virtual store every worktree already shares one electron package dir;
+ // adopting a cache entry would swap the dist out from under worktrees running it.
+ if ((options.isSharedPackageDir ?? isSharedPackageDir)(options.electronPackageDir, repoRoot)) {
+ return null
+ }
let gitCommonDir
try {
gitCommonDir = resolveGitCommonDir(repoRoot, options.execFile ?? execFileSync)
@@ -99,7 +106,7 @@ export function publishSharedElectronDist(distPath, entry, options = {}) {
return false
}
- const stagePath = `${entry.entryPath}.staging-${process.pid}-${uuid()}`
+ const stagePath = sharedTreeStagingPath(entry.entryPath, uuid)
try {
mkdirSync(entry.cacheRoot, { recursive: true })
;(options.share ?? shareTree)(distPath, stagePath)
@@ -111,65 +118,14 @@ export function publishSharedElectronDist(distPath, entry, options = {}) {
return false
}
- return swapInElectronDistEntry(entry, stagePath, {
- canValidate,
- version,
- platformPath,
+ // An entry we cannot judge counts as good: another worktree got there first, so keep theirs.
+ return publishSharedTree(stagePath, entry.entryPath, {
+ isUsable: () => !canValidate || isUsable(entry, version, platformPath),
uuid,
- rename: options.rename ?? renameSync
+ rename: options.rename
})
}
-/**
- * Replace the entry with a staged tree, re-checking first.
- *
- * Sharing the tree above takes seconds, and a sibling worktree can publish a perfectly good entry
- * in that time. Re-validating here, immediately before the destructive rename, keeps us from
- * discarding that entry -- and restoring the quarantine on a failed swap keeps a losing publisher
- * from leaving the cache empty.
- */
-function swapInElectronDistEntry(entry, stagePath, options) {
- const { canValidate, version, platformPath, uuid, rename } = options
- let quarantinePath = null
- if (existsSync(entry.entryPath)) {
- // Same rule as before staging: an entry we cannot judge, or one that is good, is never
- // displaced. Both mean another worktree got there first, so keep theirs.
- if (!canValidate || isUsable(entry, version, platformPath)) {
- rmSync(stagePath, { recursive: true, force: true })
- return false
- }
- quarantinePath = `${entry.entryPath}.unusable-${process.pid}-${uuid()}`
- try {
- renameSync(entry.entryPath, quarantinePath)
- } catch {
- rmSync(stagePath, { recursive: true, force: true })
- return false // Another worktree is already replacing it.
- }
- }
-
- try {
- rename(stagePath, entry.entryPath)
- } catch {
- rmSync(stagePath, { recursive: true, force: true })
- if (quarantinePath !== null) {
- // Put it back rather than leave no entry at all; a bad entry still beats an empty cache,
- // because the next publisher re-validates and replaces it.
- try {
- renameSync(quarantinePath, entry.entryPath)
- return false
- } catch {
- rmSync(quarantinePath, { recursive: true, force: true })
- }
- }
- return false
- }
-
- if (quarantinePath !== null) {
- rmSync(quarantinePath, { recursive: true, force: true })
- }
- return true
-}
-
function isUsable(entry, version, platformPath) {
return isUsableElectronDist(entry.entryPath, version, platformPath)
}
diff --git a/config/scripts/shared-electron-dist-cache.test.ts b/config/scripts/shared-electron-dist-cache.test.ts
index f53c68b7c9ad..1e0e8c323999 100644
--- a/config/scripts/shared-electron-dist-cache.test.ts
+++ b/config/scripts/shared-electron-dist-cache.test.ts
@@ -82,6 +82,12 @@ describe('resolveSharedElectronDistEntry', () => {
}
})
+ it('declines when the global virtual store already shares the electron package', () => {
+ expect(
+ resolveSharedElectronDistEntry({ ...baseOptions, isSharedPackageDir: () => true })
+ ).toBeNull()
+ })
+
it('declines on CI, where every job gets a fresh checkout', () => {
expect(resolveSharedElectronDistEntry({ ...baseOptions, env: { CI: '1' } })).toBeNull()
expect(resolveSharedElectronDistEntry({ ...baseOptions, env: { CI: 'true' } })).toBeNull()
diff --git a/config/scripts/shared-native-build.mjs b/config/scripts/shared-native-build.mjs
new file mode 100644
index 000000000000..0a9770172a9d
--- /dev/null
+++ b/config/scripts/shared-native-build.mjs
@@ -0,0 +1,212 @@
+/**
+ * Build a native addon for a package dir other worktrees share, without touching what they load.
+ *
+ * Under pnpm's global virtual store the package dir is one store slot for every worktree, so an
+ * in-place node-gyp rebuild swaps the binary out from under running processes. Instead the
+ * source is compiled in a private scratch copy, the outputs are validated in a fresh process,
+ * and they are published as the immutable `build/` from native-build-entry.cjs. A usable
+ * entry is reused as-is and never rebuilt, which is also why a forced rebuild cannot displace it.
+ */
+import { cpSync, existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { tmpdir } from 'node:os'
+import { dirname, join, relative } from 'node:path'
+import { runProcessSync } from '@orca/process-host'
+import { disableMsbuildFileTrackingOnWindows } from './msbuild-file-tracking.mjs'
+import { stageNodeAddonApi } from './orcad-prebuild-node-gyp.mjs'
+import { describeProcessFailure } from './process-failure-message.mjs'
+import { publishSharedTree, sharedTreeStagingPath } from './shared-tree-publish.mjs'
+import { nativeBuildEntryDir } from './native-build-entry.cjs'
+import {
+ ensureWindowsProcessTreeCommandLinePatch,
+ inspectWindowsProcessTreeAddon,
+ nodeGypRebuildInvocation,
+ nodeGypRebuildTimeoutMs,
+ stageWindowsProcessTreeNodeAddonApiHeaders
+} from './windows-process-tree-gyp-rebuild.mjs'
+
+const require = createRequire(import.meta.url)
+const { assertNodePtySourceDeniesMsysBreakaway } = require('./node-pty-job-ownership.cjs')
+const { PE_MACHINE, describePeMachine, readPeMachine } = require('./windows-pe-machine.cjs')
+const PROBE_SCRIPT = join(import.meta.dirname, 'native-build-entry-probe.cjs')
+// Prior in-place build outputs; the scratch copy compiles from source alone.
+const SCRATCH_SKIPPED_TOP_LEVEL = new Set(['build', 'bin', 'node_modules'])
+const NODE_PTY_CONPTY_RUNTIME_FILES = ['conpty.dll', 'OpenConsole.exe']
+
+/** The compiler environment every Orca node-gyp build of these addons uses. */
+export function nativeBuildEnvironment(env = process.env, platform = process.platform) {
+ return platform === 'linux'
+ ? { ...env, CXXFLAGS: `${env.CXXFLAGS ?? ''} -std=gnu++2a`.trim() }
+ : disableMsbuildFileTrackingOnWindows({ ...env })
+}
+
+/**
+ * Why `dir` cannot serve this runtime, or null when it can. Loads it in a fresh `executable`
+ * (Node, or Electron as Node), the only proof an addon is usable.
+ */
+export function checkNativeBuildDirLoads(moduleName, dir, { executable = process.execPath } = {}) {
+ if (
+ moduleName === '@vscode/windows-process-tree' &&
+ inspectWindowsProcessTreeAddon(join(dir, 'windows_process_tree.node')) !== 'clean'
+ ) {
+ return `${dir} has no addon built from the patched command-line reader`
+ }
+ let result
+ try {
+ result = runProcessSync({
+ program: executable,
+ args: [PROBE_SCRIPT, moduleName, dir],
+ env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' },
+ timeoutMs: 60_000
+ })
+ } catch (error) {
+ return `the load probe could not start: ${error.message}`
+ }
+ return result.code === 0
+ ? null
+ : (result.stderr || result.stdout || describeProcessFailure(result)).trim()
+}
+
+/** For a cross-host target nothing here can load: the artifacts must exist and match the arch. */
+export function checkNativeBuildDirArtifacts(moduleName, dir, { platform, arch }) {
+ const files =
+ moduleName === '@vscode/windows-process-tree'
+ ? ['windows_process_tree.node']
+ : platform === 'win32'
+ ? ['conpty.node', ...NODE_PTY_CONPTY_RUNTIME_FILES.map((name) => join('conpty', name))]
+ : ['pty.node', ...(platform === 'darwin' ? ['spawn-helper'] : [])]
+ const missing = files.find((file) => !existsSync(join(dir, file)))
+ if (missing) {
+ return `${join(dir, missing)} is missing`
+ }
+ const machine = platform === 'win32' ? readPeMachine(join(dir, files[0])) : undefined
+ if (machine !== undefined && machine !== PE_MACHINE[arch]) {
+ return `${join(dir, files[0])} is ${describePeMachine(machine)}, not ${arch}`
+ }
+ return null
+}
+
+/**
+ * Make sure `packageDir` holds a usable published build of `moduleName` for platform-arch.
+ *
+ * `checkBuildDir(dir)` returns why a dir is unusable, or null. Returns the entry dir and whether
+ * this call compiled. Throws when no usable entry exists afterwards.
+ */
+export function ensureSharedNativeBuild({
+ moduleName,
+ packageDir,
+ platform = process.platform,
+ arch = process.arch,
+ checkBuildDir,
+ nodeGypEntry,
+ log = console.warn
+}) {
+ // Why: pinned once, so a concurrent relink can't publish this source's build into another slot.
+ const physicalDir = realpathSync(packageDir)
+ const entryDir = nativeBuildEntryDir(physicalDir, moduleName, { platform, arch })
+ if (existsSync(entryDir)) {
+ const reason = checkBuildDir(entryDir)
+ if (reason === null) {
+ log(`Reusing the shared ${moduleName} build at ${entryDir}.`)
+ return { entryDir, built: false }
+ }
+ log(`The shared ${moduleName} build at ${entryDir} is unusable (${reason}); replacing it.`)
+ }
+ // node-gyp only compiles for the host OS; publishing that under another platform's name poisons the slot.
+ if (platform !== process.platform) {
+ throw new Error(
+ `cannot build ${moduleName} for ${platform} on ${process.platform}; build it on ${platform}`
+ )
+ }
+
+ const workDir = mkdtempSync(join(tmpdir(), 'orca-native-build-'))
+ try {
+ const releaseDir = compileInScratchCopy({
+ moduleName,
+ physicalDir,
+ sourceDir: join(workDir, 'package'),
+ arch,
+ nodeGypEntry,
+ log
+ })
+ const stagingDir = sharedTreeStagingPath(entryDir)
+ mkdirSync(dirname(entryDir), { recursive: true })
+ cpSync(releaseDir, stagingDir, {
+ recursive: true,
+ // windows-process-tree leaves MSBuild intermediates beside its one runtime file.
+ filter: (path) =>
+ moduleName !== '@vscode/windows-process-tree' ||
+ path === releaseDir ||
+ path.endsWith('.node')
+ })
+ const stagedReason = checkBuildDir(stagingDir)
+ if (stagedReason !== null) {
+ rmSync(stagingDir, { recursive: true, force: true })
+ throw new Error(`the freshly built ${moduleName} is unusable: ${stagedReason}`)
+ }
+ const published = publishSharedTree(stagingDir, entryDir, {
+ isUsable: (dir) => checkBuildDir(dir) === null
+ })
+ if (!published) {
+ const reason = checkBuildDir(entryDir)
+ if (reason !== null) {
+ throw new Error(`could not publish ${entryDir}, and what is there is unusable: ${reason}`)
+ }
+ log(`Another worktree published ${entryDir} first; using it.`)
+ } else {
+ log(`Published the shared ${moduleName} build at ${entryDir}.`)
+ }
+ return { entryDir, built: true }
+ } finally {
+ rmSync(workDir, { recursive: true, force: true })
+ }
+}
+
+function compileInScratchCopy({ moduleName, physicalDir, sourceDir, arch, nodeGypEntry, log }) {
+ cpSync(physicalDir, sourceDir, {
+ recursive: true,
+ dereference: true,
+ filter: (path) => !SCRATCH_SKIPPED_TOP_LEVEL.has(relative(physicalDir, path))
+ })
+ if (moduleName === 'node-pty') {
+ stageNodeAddonApi(physicalDir, sourceDir)
+ if (process.platform === 'win32') {
+ assertNodePtySourceDeniesMsysBreakaway({ nodePtyDir: sourceDir })
+ }
+ } else if (moduleName === '@vscode/windows-process-tree') {
+ // Repairs land in the private copy: the shared slot's source stays exactly as pnpm wrote it.
+ ensureWindowsProcessTreeCommandLinePatch(sourceDir)
+ stageWindowsProcessTreeNodeAddonApiHeaders(sourceDir, physicalDir)
+ }
+
+ log(`Building ${moduleName} for ${arch} in ${sourceDir}.`)
+ const env = { ...nativeBuildEnvironment(), npm_config_arch: arch }
+ const { args, cwd } = nodeGypRebuildInvocation(arch, sourceDir, nodeGypEntry)
+ runOrThrow(`node-gyp rebuild of ${moduleName}`, {
+ program: process.execPath,
+ args,
+ cwd,
+ env,
+ stdio: 'inherit',
+ timeoutMs: nodeGypRebuildTimeoutMs(moduleName)
+ })
+ if (moduleName === 'node-pty') {
+ // node-pty's own post-install trims build/Release to its runtime files and stages ConPTY.
+ runOrThrow('node-pty post-install', {
+ program: process.execPath,
+ args: [join(sourceDir, 'scripts', 'post-install.js')],
+ cwd: sourceDir,
+ env,
+ stdio: 'inherit',
+ timeoutMs: 60_000
+ })
+ }
+ return join(sourceDir, 'build', 'Release')
+}
+
+function runOrThrow(label, options) {
+ const result = runProcessSync(options)
+ if (result.code !== 0) {
+ throw new Error(`${label} failed in ${options.cwd}: ${describeProcessFailure(result)}`)
+ }
+}
diff --git a/config/scripts/shared-native-build.test.mjs b/config/scripts/shared-native-build.test.mjs
new file mode 100644
index 000000000000..70abe2f7e293
--- /dev/null
+++ b/config/scripts/shared-native-build.test.mjs
@@ -0,0 +1,97 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { checkNativeBuildDirArtifacts, ensureSharedNativeBuild } from './shared-native-build.mjs'
+
+const roots = []
+
+afterEach(() => {
+ while (roots.length > 0) {
+ rmSync(roots.pop(), { recursive: true, force: true })
+ }
+})
+
+function makePackage() {
+ const root = mkdtempSync(path.join(tmpdir(), 'orca-shared-native-'))
+ roots.push(root)
+ return realpathSync(root)
+}
+
+describe('ensureSharedNativeBuild', () => {
+ it('reuses a usable published entry without compiling, even when forced', () => {
+ const packageDir = makePackage()
+ const entryDir = path.join(packageDir, 'build', 'orca-napi-linux-x64')
+ mkdirSync(entryDir, { recursive: true })
+ const checked = []
+ const result = ensureSharedNativeBuild({
+ moduleName: 'node-pty',
+ packageDir,
+ platform: 'linux',
+ arch: 'x64',
+ checkBuildDir: (dir) => {
+ checked.push(dir)
+ return null
+ },
+ // No source in packageDir: any compile attempt would throw.
+ log: () => {}
+ })
+ expect(result).toEqual({ entryDir, built: false })
+ expect(checked).toEqual([entryDir])
+ })
+
+ it('publishes beside the physical package dir, not through the project link', () => {
+ const packageDir = makePackage()
+ const entryDir = path.join(packageDir, 'build', 'orca-napi-linux-x64')
+ mkdirSync(entryDir, { recursive: true })
+ const link = path.join(makePackage(), 'node-pty')
+ symlinkSync(packageDir, link, 'junction')
+ const result = ensureSharedNativeBuild({
+ moduleName: 'node-pty',
+ packageDir: link,
+ platform: 'linux',
+ arch: 'x64',
+ checkBuildDir: () => null,
+ log: () => {}
+ })
+ expect(result.entryDir).toBe(entryDir)
+ })
+
+ it('refuses to compile for another platform', () => {
+ const otherPlatform = process.platform === 'linux' ? 'darwin' : 'linux'
+ expect(() =>
+ ensureSharedNativeBuild({
+ moduleName: 'node-pty',
+ packageDir: makePackage(),
+ platform: otherPlatform,
+ arch: 'x64',
+ checkBuildDir: () => null,
+ log: () => {}
+ })
+ ).toThrow(`cannot build node-pty for ${otherPlatform}`)
+ })
+})
+
+describe('checkNativeBuildDirArtifacts', () => {
+ it('requires spawn-helper on macOS only', () => {
+ const dir = makePackage()
+ writeFileSync(path.join(dir, 'pty.node'), '')
+ expect(checkNativeBuildDirArtifacts('node-pty', dir, { platform: 'linux', arch: 'x64' })).toBe(
+ null
+ )
+ expect(
+ checkNativeBuildDirArtifacts('node-pty', dir, { platform: 'darwin', arch: 'x64' })
+ ).toMatch(/spawn-helper is missing/)
+ })
+
+ it('rejects a Windows addon built for another arch', () => {
+ const dir = makePackage()
+ mkdirSync(path.join(dir, 'conpty'))
+ for (const file of ['conpty.node', 'conpty/conpty.dll', 'conpty/OpenConsole.exe']) {
+ writeFileSync(path.join(dir, file), 'not-a-pe')
+ }
+ expect(
+ checkNativeBuildDirArtifacts('node-pty', dir, { platform: 'win32', arch: 'arm64' })
+ ).toMatch(/not arm64/)
+ })
+})
diff --git a/config/scripts/shared-tree-publish.mjs b/config/scripts/shared-tree-publish.mjs
new file mode 100644
index 000000000000..26597d7837c8
--- /dev/null
+++ b/config/scripts/shared-tree-publish.mjs
@@ -0,0 +1,85 @@
+import { randomUUID } from 'node:crypto'
+import { existsSync, renameSync, rmSync, statSync } from 'node:fs'
+
+/** A uniquely named sibling of `entryDir`, so concurrent publishers never share a staging tree. */
+export function sharedTreeStagingPath(entryDir, uuid = randomUUID) {
+ return `${entryDir}.staging-${process.pid}-${uuid()}`
+}
+
+/**
+ * Move a fully built, validated `stagingDir` into place as `entryDir`, lock-free.
+ *
+ * Every worktree that shares the entry may be reading it, so a usable entry is never replaced,
+ * and the entry is either absent, the old tree, or the new tree -- never half-written. A rename
+ * onto a populated directory fails, so of two racing publishers one wins and the other just
+ * removes its own staging tree. An unusable entry is quarantined first, and put back if the swap
+ * fails, because a bad entry the next publisher can re-judge beats an empty one. The quarantined
+ * tree must be the one judged unusable; a sibling's fresh publish that landed mid-judgement is
+ * put back untouched.
+ *
+ * Re-checks the entry here, immediately before the rename, because staging takes long enough for
+ * a sibling to publish a good one in the meantime. Returns true only when this call published.
+ */
+export function publishSharedTree(stagingDir, entryDir, options) {
+ const { isUsable } = options
+ const uuid = options.uuid ?? randomUUID
+ const rename = options.rename ?? renameSync
+ let quarantinePath = null
+ if (existsSync(entryDir)) {
+ const judged = treeIdentity(entryDir)
+ if (isUsable(entryDir)) {
+ rmSync(stagingDir, { recursive: true, force: true })
+ return false
+ }
+ // Why: the probe takes long enough for a sibling to publish; don't even briefly move theirs.
+ if (judged === null || treeIdentity(entryDir) !== judged) {
+ rmSync(stagingDir, { recursive: true, force: true })
+ return false
+ }
+ quarantinePath = `${entryDir}.unusable-${process.pid}-${uuid()}`
+ try {
+ renameSync(entryDir, quarantinePath)
+ } catch {
+ rmSync(stagingDir, { recursive: true, force: true })
+ return false // Another publisher is already replacing it, or (Windows) it is loaded.
+ }
+ if (judged === null || treeIdentity(quarantinePath) !== judged) {
+ rmSync(stagingDir, { recursive: true, force: true })
+ try {
+ renameSync(quarantinePath, entryDir)
+ } catch {
+ rmSync(quarantinePath, { recursive: true, force: true }) // A third publisher refilled it.
+ }
+ return false
+ }
+ }
+
+ try {
+ rename(stagingDir, entryDir)
+ } catch {
+ rmSync(stagingDir, { recursive: true, force: true })
+ if (quarantinePath !== null) {
+ try {
+ renameSync(quarantinePath, entryDir)
+ return false
+ } catch {
+ rmSync(quarantinePath, { recursive: true, force: true })
+ }
+ }
+ return false
+ }
+
+ if (quarantinePath !== null) {
+ rmSync(quarantinePath, { recursive: true, force: true })
+ }
+ return true
+}
+
+function treeIdentity(dir) {
+ try {
+ const { dev, ino } = statSync(dir, { bigint: true })
+ return `${dev}:${ino}`
+ } catch {
+ return null
+ }
+}
diff --git a/config/scripts/shared-tree-publish.test.mjs b/config/scripts/shared-tree-publish.test.mjs
new file mode 100644
index 000000000000..13995fff213e
--- /dev/null
+++ b/config/scripts/shared-tree-publish.test.mjs
@@ -0,0 +1,123 @@
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ renameSync,
+ rmSync,
+ writeFileSync
+} from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { publishSharedTree, sharedTreeStagingPath } from './shared-tree-publish.mjs'
+
+const roots = []
+
+afterEach(() => {
+ while (roots.length > 0) {
+ rmSync(roots.pop(), { recursive: true, force: true })
+ }
+})
+
+function makeRoot() {
+ const root = mkdtempSync(path.join(tmpdir(), 'orca-shared-tree-'))
+ roots.push(root)
+ return root
+}
+
+function writeTree(dir, content) {
+ mkdirSync(dir, { recursive: true })
+ writeFileSync(path.join(dir, 'payload'), content)
+ return dir
+}
+
+const readPayload = (dir) => readFileSync(path.join(dir, 'payload'), 'utf8')
+const isGood = (dir) => existsSync(path.join(dir, 'payload')) && readPayload(dir) !== 'bad'
+
+describe('publishSharedTree', () => {
+ it('names staging trees uniquely beside the entry', () => {
+ expect(sharedTreeStagingPath('/x/entry', () => 'u1')).toBe(`/x/entry.staging-${process.pid}-u1`)
+ })
+
+ it('renames a staged tree into an empty slot', () => {
+ const root = makeRoot()
+ const entry = path.join(root, 'entry')
+ const staging = writeTree(sharedTreeStagingPath(entry), 'new')
+ expect(publishSharedTree(staging, entry, { isUsable: isGood })).toBe(true)
+ expect(readPayload(entry)).toBe('new')
+ expect(readdirSync(root)).toEqual(['entry'])
+ })
+
+ it('never replaces a usable entry, and drops its own staging tree', () => {
+ const root = makeRoot()
+ const entry = writeTree(path.join(root, 'entry'), 'old')
+ const staging = writeTree(sharedTreeStagingPath(entry), 'new')
+ expect(publishSharedTree(staging, entry, { isUsable: isGood })).toBe(false)
+ expect(readPayload(entry)).toBe('old')
+ expect(readdirSync(root)).toEqual(['entry'])
+ })
+
+ it('lets exactly one of two racing publishers win', () => {
+ const root = makeRoot()
+ const entry = path.join(root, 'entry')
+ const first = writeTree(
+ sharedTreeStagingPath(entry, () => 'a'),
+ 'first'
+ )
+ const second = writeTree(
+ sharedTreeStagingPath(entry, () => 'b'),
+ 'second'
+ )
+ // The second passes its pre-check, then the first lands just before its own rename does.
+ const raced = publishSharedTree(second, entry, {
+ isUsable: isGood,
+ rename: (from, to) => {
+ expect(publishSharedTree(first, entry, { isUsable: isGood })).toBe(true)
+ renameSync(from, to)
+ }
+ })
+ expect(raced).toBe(false)
+ expect(readPayload(entry)).toBe('first')
+ expect(readdirSync(root)).toEqual(['entry'])
+ })
+
+ it('quarantines an unusable entry and publishes over it', () => {
+ const root = makeRoot()
+ const entry = writeTree(path.join(root, 'entry'), 'bad')
+ const staging = writeTree(sharedTreeStagingPath(entry), 'new')
+ expect(publishSharedTree(staging, entry, { isUsable: isGood })).toBe(true)
+ expect(readPayload(entry)).toBe('new')
+ expect(readdirSync(root)).toEqual(['entry'])
+ })
+
+ it('puts the quarantined entry back when the swap fails', () => {
+ const root = makeRoot()
+ const entry = writeTree(path.join(root, 'entry'), 'bad')
+ const staging = writeTree(sharedTreeStagingPath(entry), 'new')
+ const failingRename = () => {
+ throw new Error('EPERM')
+ }
+ expect(publishSharedTree(staging, entry, { isUsable: isGood, rename: failingRename })).toBe(
+ false
+ )
+ expect(readPayload(entry)).toBe('bad')
+ expect(readdirSync(root)).toEqual(['entry'])
+ })
+ it('never quarantines a good entry a sibling published while the bad one was judged', () => {
+ const root = makeRoot()
+ const entry = writeTree(path.join(root, 'entry'), 'bad')
+ const staging = writeTree(sharedTreeStagingPath(entry), 'mine')
+ const sibling = writeTree(path.join(root, 'sibling'), 'sibling')
+ const judgeThenRace = (dir) => {
+ const usable = isGood(dir)
+ rmSync(entry, { recursive: true, force: true })
+ renameSync(sibling, entry)
+ return usable
+ }
+ expect(publishSharedTree(staging, entry, { isUsable: judgeThenRace })).toBe(false)
+ expect(readPayload(entry)).toBe('sibling')
+ expect(readdirSync(root)).toEqual(['entry'])
+ })
+})
diff --git a/config/scripts/store-ancestor-node-modules.mjs b/config/scripts/store-ancestor-node-modules.mjs
new file mode 100644
index 000000000000..c424a2704ef3
--- /dev/null
+++ b/config/scripts/store-ancestor-node-modules.mjs
@@ -0,0 +1,46 @@
+import { existsSync, realpathSync } from 'node:fs'
+import { basename, dirname, join } from 'node:path'
+import { isSharedPackageDir } from './native-build-entry.cjs'
+
+/**
+ * node_modules dirs above the global virtual store's `links` root, given one store slot's path.
+ *
+ * Node resolves upward from a slot, so a stray `~/node_modules` (or `/tmp/node_modules`) above
+ * the store hands every package dependencies it never declared and hides exactly the bugs the
+ * isolated layout exists to surface.
+ */
+export function nodeModulesAboveStore(slotPackageDir, exists = existsSync) {
+ let storeRoot = slotPackageDir
+ while (basename(storeRoot) !== 'links') {
+ const parent = dirname(storeRoot)
+ if (parent === storeRoot) {
+ return []
+ }
+ storeRoot = parent
+ }
+ const found = []
+ for (let dir = dirname(storeRoot); ; dir = dirname(dir)) {
+ if (exists(join(dir, 'node_modules'))) {
+ found.push(join(dir, 'node_modules'))
+ }
+ if (dirname(dir) === dir) {
+ return found
+ }
+ }
+}
+
+/** Warns, never fails: the install itself is fine, only undeclared-dependency checks are blinded. */
+export function warnAboutNodeModulesAboveSharedStore(projectDir, probePackage = 'zod') {
+ const linked = join(projectDir, 'node_modules', probePackage)
+ if (!existsSync(linked) || !isSharedPackageDir(linked, projectDir)) {
+ return []
+ }
+ const found = nodeModulesAboveStore(realpathSync(linked))
+ for (const dir of found) {
+ console.warn(
+ `[gvs] ${dir} sits above the shared pnpm store, so packages there resolve dependencies ` +
+ 'they never declared and undeclared-dependency bugs stay hidden. Move or remove it.'
+ )
+ }
+ return found
+}
diff --git a/config/scripts/store-ancestor-node-modules.test.mjs b/config/scripts/store-ancestor-node-modules.test.mjs
new file mode 100644
index 000000000000..ab36a4dcfd02
--- /dev/null
+++ b/config/scripts/store-ancestor-node-modules.test.mjs
@@ -0,0 +1,49 @@
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { nodeModulesAboveStore } from './store-ancestor-node-modules.mjs'
+
+const slot = join(
+ '/',
+ 'home',
+ 'dev',
+ 'pnpm',
+ 'store',
+ 'v11',
+ 'links',
+ '@',
+ 'zod',
+ '4.6.5',
+ 'h',
+ 'node_modules',
+ 'zod'
+)
+
+describe('nodeModulesAboveStore', () => {
+ it('reports node_modules above the store root, not the slot-local ones', () => {
+ const present = new Set([
+ join('/', 'home', 'dev', 'node_modules'),
+ join(
+ '/',
+ 'home',
+ 'dev',
+ 'pnpm',
+ 'store',
+ 'v11',
+ 'links',
+ '@',
+ 'zod',
+ '4.6.5',
+ 'h',
+ 'node_modules'
+ )
+ ])
+ expect(nodeModulesAboveStore(slot, (path) => present.has(path))).toEqual([
+ join('/', 'home', 'dev', 'node_modules')
+ ])
+ })
+
+ it('reports nothing for a clean store or a path outside any store', () => {
+ expect(nodeModulesAboveStore(slot, () => false)).toEqual([])
+ expect(nodeModulesAboveStore(join('/', 'repo', 'node_modules', 'zod'), () => true)).toEqual([])
+ })
+})
diff --git a/config/scripts/terminal-perf-dependency-preparation.test.mjs b/config/scripts/terminal-perf-dependency-preparation.test.mjs
index ad8fa4d34740..82e2d31e97f7 100644
--- a/config/scripts/terminal-perf-dependency-preparation.test.mjs
+++ b/config/scripts/terminal-perf-dependency-preparation.test.mjs
@@ -13,7 +13,7 @@ const script = selector.run.trim().match(/^node <<'NODE'\n([\s\S]*)\nNODE$/)[1]
const supportedAction = readFileSync('.github/actions/install-node-dependencies/action.yml', 'utf8')
const supportedManifest = {
engines: { node: '24' },
- packageManager: 'pnpm@12.8.1',
+ packageManager: 'pnpm@12.14.0',
scripts: {
postinstall: 'pnpm run build:packages && node config/scripts/rebuild-native-deps.mjs'
}
@@ -62,7 +62,7 @@ function select(options = {}) {
it('selects the measured current root profile with the actual installer metadata', () => {
expect(select()).toBe('shared=true')
expect(
- select({ manifest: { ...supportedManifest, packageManager: 'pnpm@12.8.1+sha512.fixture' } })
+ select({ manifest: { ...supportedManifest, packageManager: 'pnpm@12.14.0+sha512.fixture' } })
).toBe('shared=true')
})
@@ -80,7 +80,7 @@ it('keeps the shared preparation route for the qualified older lifecycle', () =>
it.each([
['historical Node', { manifest: { ...supportedManifest, engines: { node: '22' } } }],
['historical pnpm', { manifest: { ...supportedManifest, packageManager: 'pnpm@10.0.0' } }],
- ['unmeasured pnpm', { manifest: { ...supportedManifest, packageManager: 'pnpm@12.8.10' } }],
+ ['unmeasured pnpm', { manifest: { ...supportedManifest, packageManager: 'pnpm@12.14.00' } }],
['missing toolchain', { manifest: {} }],
[
'extra lifecycle work',
diff --git a/config/scripts/undeclared-dependency-resolution.test.ts b/config/scripts/undeclared-dependency-resolution.test.ts
new file mode 100644
index 000000000000..5df467c56a35
--- /dev/null
+++ b/config/scripts/undeclared-dependency-resolution.test.ts
@@ -0,0 +1,73 @@
+import { existsSync, readFileSync, realpathSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import { parse } from 'yaml'
+import { z } from 'zod'
+import { markdownParserAliases } from '../build-plugins/markdown-parser-exports'
+
+const repoRoot = join(import.meta.dirname, '../..')
+const repoNodeModules = join(repoRoot, 'node_modules')
+
+const dependencyMap = z.record(z.string(), z.string()).optional()
+const workspace = z
+ .object({
+ packageExtensions: z
+ .record(
+ z.string(),
+ z.object({ dependencies: dependencyMap, peerDependencies: dependencyMap })
+ )
+ .optional(),
+ addMissingPeerTypes: z.boolean().optional()
+ })
+ .parse(parse(readFileSync(join(repoRoot, 'pnpm-workspace.yaml'), 'utf8')))
+
+// Every dependency pnpm-workspace.yaml adds, so a new entry is covered without editing this file.
+const extensionPairs = Object.entries(workspace.packageExtensions ?? {}).flatMap(
+ ([owner, extension]) =>
+ [
+ ...Object.keys(extension.dependencies ?? {}),
+ ...Object.keys(extension.peerDependencies ?? {})
+ ].map((dependency) => [owner, dependency] as const)
+)
+
+// These peer on react and ship .d.ts that import it; addMissingPeerTypes supplies @types/react.
+const peerTypePairs = [
+ '@dnd-kit/core',
+ '@dnd-kit/sortable',
+ '@dnd-kit/utilities',
+ '@monaco-editor/react',
+ 'cmdk',
+ 'emoji-picker-react',
+ 'lucide-react',
+ 'react-colorful',
+ 'react-i18next',
+ 'sonner'
+].map((owner) => [owner, '@types/react'] as const)
+
+// Node's own lookup from the package's real location, without tripping over "exports" maps.
+function isVisibleFrom(owner: string, dependency: string): boolean {
+ const ownerDir = realpathSync(join(repoNodeModules, owner))
+ const lookupDirs = createRequire(join(ownerDir, 'package.json')).resolve.paths(dependency) ?? []
+ return lookupDirs.some((dir) => existsSync(join(dir, dependency, 'package.json')))
+}
+
+describe('undeclared dependency resolution', () => {
+ it('covers every packageExtensions entry', () => {
+ const owners = new Set(extensionPairs.map(([owner]) => owner))
+ expect([...owners].sort()).toEqual(Object.keys(workspace.packageExtensions ?? {}).sort())
+ expect(workspace.addMissingPeerTypes).toBe(true)
+ })
+
+ // Why: the global virtual store links only declared deps; pnpm-workspace.yaml supplies these.
+ it.each([...extensionPairs, ...peerTypePairs])(
+ '%s sees %s from its own install location',
+ (owner, dependency) => {
+ expect(isVisibleFrom(owner, dependency)).toBe(true)
+ }
+ )
+
+ it('aliases the DOM-free decode-named-character-reference entry', () => {
+ expect(markdownParserAliases['decode-named-character-reference']).toMatch(/[\\/]index\.js$/)
+ })
+})
diff --git a/config/scripts/vendor-feature-wall-assets.mjs b/config/scripts/vendor-feature-wall-assets.mjs
index b1e9b061f8f8..c2c9a7781cd4 100644
--- a/config/scripts/vendor-feature-wall-assets.mjs
+++ b/config/scripts/vendor-feature-wall-assets.mjs
@@ -1,8 +1,11 @@
#!/usr/bin/env node
-import { copyFile, mkdir, writeFile } from 'node:fs/promises'
+import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { spawnSync } from 'node:child_process'
-import { homedir } from 'node:os'
+import { homedir, tmpdir } from 'node:os'
import path from 'node:path'
+import { runProcessSync } from '@orca/process-host'
+import { FEATURE_WALL_TILES } from '../../src/shared/feature-wall-tiles.ts'
+import { verifyLosslessFeatureWallMedia } from './verify-feature-wall-lossless-media.mjs'
const __dirname = import.meta.dirname
const ROOT = path.join(__dirname, '..', '..')
@@ -101,29 +104,66 @@ function gitRecordedAtSeconds(tile, relativePath) {
await mkdir(DEST_ROOT, { recursive: true })
-for (const tile of TILES) {
- const sourceRoot = sourceRootForTile(tile)
- const sourceGif = path.join(sourceRoot, ...tile.gifRelativePath.split('/'))
- const sourcePoster = path.join(sourceRoot, ...tile.posterRelativePath.split('/'))
- const destGif = path.join(DEST_ROOT, `${tile.id}.gif`)
- const destPoster = path.join(DEST_ROOT, `${tile.id}.poster.jpg`)
- const recordedAtSeconds = gitRecordedAtSeconds(tile, tile.gifRelativePath)
+const originals = new Map()
+const stagingRoot = await mkdtemp(path.join(tmpdir(), 'orca-feature-wall-media-'))
+try {
+ for (const tile of TILES) {
+ const sourceRoot = sourceRootForTile(tile)
+ const sourceGif = path.join(sourceRoot, ...tile.gifRelativePath.split('/'))
+ const sourcePoster = path.join(sourceRoot, ...tile.posterRelativePath.split('/'))
+ const media = FEATURE_WALL_TILES.find((item) => item.id === tile.id)
+ if (!media || media.kind !== 'media') {
+ throw new Error(`Unknown media tile: ${tile.id}`)
+ }
+ const destAnimation = path.join(stagingRoot, media.gifPath)
+ const destPoster = path.join(stagingRoot, `${tile.id}.poster.jpg`)
+ const recordedAtSeconds = gitRecordedAtSeconds(tile, tile.gifRelativePath)
- await copyFile(sourceGif, destGif)
- await copyFile(sourcePoster, destPoster)
- await writeFile(
- path.join(DEST_ROOT, `${tile.id}.recorded-at.json`),
- `${JSON.stringify(
- {
- recordedAtUnixSeconds: recordedAtSeconds,
- recordedAtIso: new Date(recordedAtSeconds * 1000).toISOString(),
- sourceGif: tile.gifRelativePath,
- sourcePoster: tile.posterRelativePath
- },
- null,
- 2
- )}\n`
- )
+ originals.set(tile.id, await readFile(sourceGif))
+ if (media.gifPath.endsWith('.gif')) {
+ await copyFile(sourceGif, destAnimation)
+ } else {
+ // gif2webp defaults to lossless; the verifier rejects changed pixels or timing.
+ const result = runProcessSync({
+ program: 'gif2webp',
+ args: ['-m', '6', '-min_size', '-metadata', 'all', sourceGif, '-o', destAnimation],
+ timeoutMs: 120_000
+ })
+ if (result.code !== 0) {
+ throw new Error(`gif2webp failed for ${tile.id}: ${result.stderr}`)
+ }
+ }
+ await copyFile(sourcePoster, destPoster)
+ await writeFile(
+ path.join(stagingRoot, `${tile.id}.recorded-at.json`),
+ `${JSON.stringify(
+ {
+ recordedAtUnixSeconds: recordedAtSeconds,
+ recordedAtIso: new Date(recordedAtSeconds * 1000).toISOString(),
+ sourceGif: tile.gifRelativePath,
+ sourcePoster: tile.posterRelativePath
+ },
+ null,
+ 2
+ )}\n`
+ )
- console.log(`Vendored ${tile.gifRelativePath} -> ${tile.id}`)
+ console.log(`Vendored ${tile.gifRelativePath} -> ${tile.id}`)
+ }
+
+ await verifyLosslessFeatureWallMedia(originals, stagingRoot)
+ for (const tile of TILES) {
+ const media = FEATURE_WALL_TILES.find((item) => item.id === tile.id)
+ if (media?.kind !== 'media') {
+ throw new Error(`Unknown media tile: ${tile.id}`)
+ }
+ for (const file of [media.gifPath, media.posterPath, media.recordedAtPath]) {
+ await copyFile(path.join(stagingRoot, file), path.join(DEST_ROOT, file))
+ }
+ if (media.gifPath.endsWith('.webp')) {
+ await rm(path.join(DEST_ROOT, `${tile.id}.gif`), { force: true })
+ }
+ }
+} finally {
+ await rm(stagingRoot, { recursive: true, force: true })
}
diff --git a/config/scripts/verify-feature-wall-lossless-media.mjs b/config/scripts/verify-feature-wall-lossless-media.mjs
new file mode 100644
index 000000000000..76274691721a
--- /dev/null
+++ b/config/scripts/verify-feature-wall-lossless-media.mjs
@@ -0,0 +1,198 @@
+#!/usr/bin/env node
+import assert from 'node:assert/strict'
+import { readFile } from 'node:fs/promises'
+import { createServer } from 'node:http'
+import path from 'node:path'
+import { chromium } from 'playwright-core'
+import { runProcessSync } from '@orca/process-host'
+import { FEATURE_WALL_TILES, isFeatureWallMediaTile } from '../../src/shared/feature-wall-tiles.ts'
+
+const ROOT = path.resolve(import.meta.dirname, '..', '..')
+const ASSET_DIR = 'resources/onboarding/feature-wall'
+
+function skipGifSubBlocks(bytes, offset) {
+ while (offset < bytes.length && bytes[offset] !== 0) {
+ offset += 1 + bytes[offset]
+ }
+ assert.ok(offset < bytes.length, 'Truncated GIF sub-block')
+ return offset + 1
+}
+
+function gifDurations(bytes) {
+ assert.match(bytes.subarray(0, 6).toString(), /^GIF8[79]a$/)
+ let offset = 13 + (bytes[10] & 128 ? 3 * 2 ** ((bytes[10] & 7) + 1) : 0)
+ let duration = 0
+ const durations = []
+ while (offset < bytes.length) {
+ const block = bytes[offset++]
+ if (block === 0x3b) {
+ return durations
+ }
+ if (block === 0x21) {
+ const label = bytes[offset++]
+ if (label === 0xf9) {
+ duration = bytes.readUInt16LE(offset + 2) * 10
+ }
+ offset = skipGifSubBlocks(bytes, offset)
+ } else {
+ assert.equal(block, 0x2c, 'Invalid GIF image descriptor')
+ const packed = bytes[offset + 8]
+ offset += 9 + (packed & 128 ? 3 * 2 ** ((packed & 7) + 1) : 0)
+ offset = skipGifSubBlocks(bytes, offset + 1)
+ durations.push(duration)
+ duration = 0
+ }
+ }
+ throw new Error('Missing GIF trailer')
+}
+
+function animationDurations(bytes) {
+ if (bytes.subarray(0, 3).toString() === 'GIF') {
+ return gifDurations(bytes)
+ }
+ assert.equal(bytes.subarray(0, 4).toString(), 'RIFF')
+ assert.equal(bytes.subarray(8, 12).toString(), 'WEBP')
+ const durations = []
+ for (let offset = 12; offset < bytes.length;) {
+ const length = bytes.readUInt32LE(offset + 4)
+ if (bytes.subarray(offset, offset + 4).toString() === 'ANMF') {
+ durations.push(bytes.readUIntLE(offset + 20, 3))
+ }
+ offset += 8 + length + (length % 2)
+ }
+ return durations
+}
+
+export async function verifyLosslessFeatureWallMedia(
+ originals,
+ candidateDirectory = path.join(ROOT, ASSET_DIR)
+) {
+ const candidates = new Map()
+ for (const tile of FEATURE_WALL_TILES.filter(isFeatureWallMediaTile)) {
+ const candidate = await readFile(path.join(candidateDirectory, tile.gifPath))
+ const original = originals.get(tile.id)
+ assert.ok(original, `${tile.id}: original GIF missing`)
+ assert.deepEqual(
+ animationDurations(candidate),
+ animationDurations(original),
+ `${tile.id}: encoded per-frame timing changed`
+ )
+ candidates.set(tile.id, candidate)
+ }
+ const server = createServer((request, response) => {
+ response.setHeader('Cache-Control', 'no-store')
+ if (request.url === '/') {
+ return response.end('Animation equality')
+ }
+ const [kind, id] = request.url.slice(1).split('/')
+ const bytes = (kind === 'original' ? originals : candidates).get(id)
+ response.statusCode = bytes ? 200 : 404
+ response.end(bytes)
+ })
+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
+ let browser
+ try {
+ browser = await chromium.launch({
+ headless: true,
+ executablePath: process.env.ORCA_ANIMATION_CHROMIUM_EXECUTABLE || undefined
+ })
+ const page = await browser.newPage()
+ await page.goto(`http://127.0.0.1:${server.address().port}`)
+ const results = []
+ for (const tile of FEATURE_WALL_TILES.filter(isFeatureWallMediaTile)) {
+ const result = await page.evaluate(async ({ id, gifPath }) => {
+ const makeDecoder = async (kind, type) =>
+ new ImageDecoder({
+ data: await (await fetch(`/${kind}/${id}`)).arrayBuffer(),
+ type,
+ preferAnimation: true
+ })
+ const original = await makeDecoder('original', 'image/gif')
+ const candidate = await makeDecoder(
+ 'candidate',
+ gifPath.endsWith('.gif') ? 'image/gif' : 'image/webp'
+ )
+ try {
+ await Promise.all([original.tracks.ready, candidate.tracks.ready])
+ const a = original.tracks.selectedTrack
+ const b = candidate.tracks.selectedTrack
+ if (a.frameCount !== b.frameCount || a.repetitionCount !== b.repetitionCount) {
+ throw new Error(`${id}: frame count or loop changed`)
+ }
+ let canvas, context
+ let durationUs = 0
+ for (let frameIndex = 0; frameIndex < a.frameCount; frameIndex++) {
+ const first = (await original.decode({ frameIndex })).image
+ const second = (await candidate.decode({ frameIndex })).image
+ try {
+ if (
+ first.displayWidth !== second.displayWidth ||
+ first.displayHeight !== second.displayHeight ||
+ first.duration !== second.duration ||
+ first.timestamp !== second.timestamp
+ ) {
+ throw new Error(`${id}: dimensions or playback timing changed at ${frameIndex}`)
+ }
+ if (!canvas) {
+ canvas = new OffscreenCanvas(first.displayWidth, first.displayHeight)
+ context = canvas.getContext('2d', { willReadFrequently: true })
+ }
+ context.clearRect(0, 0, canvas.width, canvas.height)
+ context.drawImage(first, 0, 0)
+ const left = context.getImageData(0, 0, canvas.width, canvas.height).data
+ context.clearRect(0, 0, canvas.width, canvas.height)
+ context.drawImage(second, 0, 0)
+ const right = context.getImageData(0, 0, canvas.width, canvas.height).data
+ const leftPixels = new Uint32Array(left.buffer)
+ const rightPixels = new Uint32Array(right.buffer)
+ for (let index = 0; index < leftPixels.length; index++) {
+ if (leftPixels[index] !== rightPixels[index]) {
+ throw new Error(`${id}: RGBA pixel ${index} changed at frame ${frameIndex}`)
+ }
+ }
+ durationUs += first.duration
+ } finally {
+ first.close()
+ second.close()
+ }
+ }
+ return { id, frames: a.frameCount, durationUs, exactRGBA: true }
+ } finally {
+ original.close()
+ candidate.close()
+ }
+ }, tile)
+ results.push(result)
+ console.log(JSON.stringify(result))
+ }
+ return results
+ } finally {
+ await browser?.close()
+ await new Promise((resolve) => server.close(resolve))
+ }
+}
+
+function readOriginalGif(baseRef, id) {
+ const result = runProcessSync({
+ program: 'git',
+ args: ['show', `${baseRef}:${ASSET_DIR}/${id}.gif`],
+ cwd: ROOT,
+ maxOutputBytes: 16 * 1024 * 1024,
+ captureStdoutAsBytes: true
+ })
+ assert.equal(result.code, 0, result.stderr)
+ assert.ok(result.stdoutBytes)
+ return result.stdoutBytes
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === import.meta.filename) {
+ const baseRef = process.argv[2]
+ assert.ok(baseRef, 'Usage: node config/scripts/verify-feature-wall-lossless-media.mjs ')
+ const originals = new Map(
+ FEATURE_WALL_TILES.filter(isFeatureWallMediaTile).map((tile) => [
+ tile.id,
+ readOriginalGif(baseRef, tile.id)
+ ])
+ )
+ await verifyLosslessFeatureWallMedia(originals)
+}
diff --git a/config/scripts/verify-packaged-mobile-web-bundle.cjs b/config/scripts/verify-packaged-mobile-web-bundle.cjs
index b28662a48d58..c630e642bf1d 100644
--- a/config/scripts/verify-packaged-mobile-web-bundle.cjs
+++ b/config/scripts/verify-packaged-mobile-web-bundle.cjs
@@ -2,6 +2,8 @@ const { createHash } = require('node:crypto')
const { readFileSync, readdirSync, statSync } = require('node:fs')
const { join, resolve } = require('node:path')
+const { darkBuildMarkerFileNames } = require('../dark-build-switches.cjs')
+
const projectDir = resolve(__dirname, '..', '..')
const MOBILE_WEB_BUNDLE_DIR = join(projectDir, 'out', 'mobile-web')
const REMEDY = 'Run pnpm build:mobile-web (build:desktop and build:release already do).'
@@ -141,7 +143,12 @@ function listBundleFiles(directory, prefix = '') {
* the installer; content-addressed names mean stale copies never get overwritten.
*/
function assertNoUnlistedFiles(bundleDir, manifest) {
- const listed = new Set(['manifest.json', ...manifest.assets.map((asset) => asset.path)])
+ // A dark build's marker is accounted for here; packaging refuses it through assertNotDarkBuild.
+ const listed = new Set([
+ 'manifest.json',
+ ...darkBuildMarkerFileNames(),
+ ...manifest.assets.map((asset) => asset.path)
+ ])
const strays = listBundleFiles(bundleDir).filter((path) => !listed.has(path))
if (strays.length > 0) {
throw failure(
diff --git a/config/scripts/verify-packaged-mobile-web-bundle.test.mjs b/config/scripts/verify-packaged-mobile-web-bundle.test.mjs
index 54f99afd324f..b2f32c58aa73 100644
--- a/config/scripts/verify-packaged-mobile-web-bundle.test.mjs
+++ b/config/scripts/verify-packaged-mobile-web-bundle.test.mjs
@@ -8,6 +8,7 @@ import { writeMobileWebBundleFixtureTree } from './mobile-web-bundle-fixture-tre
const require = createRequire(import.meta.url)
const { assertMobileWebBundleBuilt } = require('./verify-packaged-mobile-web-bundle.cjs')
const electronBuilderConfig = require('../electron-builder.config.cjs')
+const { darkBuildMarkers } = require('../dark-build-switches.cjs')
async function withBundle(run) {
const scratch = await mkdtemp(join(tmpdir(), 'orca-mobile-web-guard-'))
@@ -166,6 +167,20 @@ describe('electron-builder packaging wiring', () => {
expect(excludesBuiltBundle).toBe(false)
})
+ it('verifies a dark page build, and refuses to package it', async () => {
+ await withBundle(async ({ bundleDir }) => {
+ const [marker] = darkBuildMarkers({ ORCA_SIDEBAR_INPUTS: 'main' })
+ await writeFile(join(bundleDir, marker.fileName), marker.text, 'utf8')
+ expect(() => assertMobileWebBundleBuilt(bundleDir)).not.toThrow()
+ expect(() =>
+ electronBuilderConfig.beforePack(
+ { electronPlatformName: process.platform, arch: process.arch === 'arm64' ? 3 : 1 },
+ bundleDir
+ )
+ ).toThrow(`${bundleDir} holds an ORCA_SIDEBAR_INPUTS=main build`)
+ })
+ })
+
it('verifies the bundle root it is given, not the repo one', async () => {
// The seam exists so unit tests need no built out/; it would be worthless if the root were
// accepted and then ignored.
diff --git a/config/scripts/verify-packaged-orcad-template.cjs b/config/scripts/verify-packaged-orcad-template.cjs
index 8e5a0e6752bb..7930dabd7c4e 100644
--- a/config/scripts/verify-packaged-orcad-template.cjs
+++ b/config/scripts/verify-packaged-orcad-template.cjs
@@ -9,6 +9,7 @@ const {
ORCAD_TEMPLATE_TARGETS_DIR,
orcadMobileWebAssetFilenames,
orcadTemplateCommonFilenames,
+ orcadTemplatePackagedRipgrep,
orcadTemplateTargetFilenames
} = require('../../src/shared/orcad-artifacts.ts')
const {
@@ -97,9 +98,14 @@ function requireContent(path, expected, label) {
}
}
-function verifyTarget(templateDir, target, value) {
+function verifyTarget(resourcesDir, templateDir, target, value, schemaVersion) {
const targetManifest = requireRecord(value, `${target} manifest`)
const files = requireRecord(targetManifest.files, `${target} files`)
+ const packagedRipgrep = orcadTemplatePackagedRipgrep(
+ schemaVersion,
+ target,
+ targetManifest.packagedRipgrep
+ )
const expectedFiles = orcadTemplateTargetFilenames(target)
requireExactNames(Object.keys(files), expectedFiles, `${target} manifest inventory`)
const hasBrowserName = Object.hasOwn(targetManifest, 'browserName')
@@ -112,7 +118,7 @@ function verifyTarget(templateDir, target, value) {
const targetDir = join(templateDir, ORCAD_TEMPLATE_TARGETS_DIR, target)
for (const filename of expectedFiles) {
verifyFile(
- join(targetDir, ...filename.split('/')),
+ join(filename === packagedRipgrep ? resourcesDir : targetDir, ...filename.split('/')),
requireSha256(files[filename], `${target} ${filename} checksum`),
`${target} ${filename}`
)
@@ -124,7 +130,7 @@ function verifyTarget(templateDir, target, value) {
`${target} runtime reference`
)
- const inventory = [...expectedFiles]
+ const inventory = expectedFiles.filter((filename) => filename !== packagedRipgrep)
if (hasBrowserName) {
const browserName = targetManifest.browserName
if (
@@ -148,9 +154,9 @@ function verifyTarget(templateDir, target, value) {
function verifyPackagedOrcadTemplate(resourcesDir, targets = ORCAD_TEMPLATE_TARGETS) {
const templateDir = join(resourcesDir, 'orcad-template')
const manifest = requireRecord(readManifest(templateDir), 'manifest')
- if (manifest.schemaVersion !== TEMPLATE_SCHEMA_VERSION) {
+ if (![TEMPLATE_SCHEMA_VERSION, 4].includes(manifest.schemaVersion)) {
throw new Error(
- `[verify-packaged-orcad-template] manifest schemaVersion must be ${TEMPLATE_SCHEMA_VERSION}`
+ `[verify-packaged-orcad-template] manifest schemaVersion must be ${TEMPLATE_SCHEMA_VERSION} or 4`
)
}
const commonSha256 = requireRecord(manifest.commonSha256, 'commonSha256')
@@ -185,7 +191,7 @@ function verifyPackagedOrcadTemplate(resourcesDir, targets = ORCAD_TEMPLATE_TARG
'target directory inventory'
)
for (const target of expectedTargets) {
- verifyTarget(templateDir, target, manifestTargets[target])
+ verifyTarget(resourcesDir, templateDir, target, manifestTargets[target], manifest.schemaVersion)
}
console.log(
`[verify-packaged-orcad-template] OK — verified ${expectedTargets.length} Node targets`
diff --git a/config/scripts/verify-packaged-serve-payload.cjs b/config/scripts/verify-packaged-serve-payload.cjs
index a60ec33a92aa..f9f361e9715a 100644
--- a/config/scripts/verify-packaged-serve-payload.cjs
+++ b/config/scripts/verify-packaged-serve-payload.cjs
@@ -2,7 +2,7 @@ const { existsSync, readFileSync, statSync } = require('node:fs')
const { dirname, join, posix } = require('node:path')
const WEB_CLIENT_INDEX = 'web-index.html'
-const REMEDY = 'Run pnpm build:web (build:desktop and build:release already do).'
+const REMEDY = 'Run pnpm build:web-from-renderer (build:desktop and build:release already do).'
function failure(message) {
return new Error(`[verify-packaged-serve-payload] ${message}`)
@@ -27,7 +27,33 @@ function assertPackagedWebClient(resourcesDir) {
if (!existsSync(join(outDir, 'cli'))) {
throw failure(`no unpacked CLI at ${join(outDir, 'cli')}; the web client root resolves from it`)
}
- const webRoot = join(outDir, 'web')
+ const webRoot = join(outDir, 'renderer')
+ const manifestPath = join(webRoot, '.orca-web-client-files.json')
+ if (!existsSync(manifestPath)) {
+ throw failure(`missing unpacked web file manifest at ${manifestPath}. ${REMEDY}`)
+ }
+ const files = JSON.parse(readFileSync(manifestPath, 'utf8'))
+ if (
+ !Array.isArray(files) ||
+ !files.includes(WEB_CLIENT_INDEX) ||
+ !files.every(
+ (file) =>
+ typeof file === 'string' &&
+ (file === WEB_CLIENT_INDEX || /^(?:assets|cmaps|standard_fonts|wasm)\//.test(file)) &&
+ !file.includes('\\') &&
+ !file.includes('\0') &&
+ file.split('/').every((part) => part !== '' && part !== '.' && part !== '..') &&
+ !file.endsWith('.map')
+ )
+ ) {
+ throw failure(`invalid web file manifest at ${manifestPath}`)
+ }
+ const absent = files.filter(
+ (file) => !statSync(join(webRoot, file), { throwIfNoEntry: false })?.isFile()
+ )
+ if (absent.length > 0) {
+ throw failure(`missing unpacked web files: ${absent.join(', ')}`)
+ }
const indexPath = join(webRoot, WEB_CLIENT_INDEX)
if (!existsSync(indexPath)) {
throw failure(`missing unpacked ${indexPath}. ${REMEDY}`)
@@ -37,7 +63,10 @@ function assertPackagedWebClient(resourcesDir) {
throw failure(`${indexPath} references no ./ assets. ${REMEDY}`)
}
const missing = assets.filter(
- (asset) => asset.startsWith('..') || !existsSync(join(webRoot, ...asset.split('/')))
+ (asset) =>
+ asset.startsWith('..') ||
+ !files.includes(asset) ||
+ !existsSync(join(webRoot, ...asset.split('/')))
)
if (missing.length > 0) {
throw failure(`${indexPath} loads files missing from ${webRoot}: ${missing.join(', ')}`)
diff --git a/config/scripts/verify-packaged-serve-payload.test.mjs b/config/scripts/verify-packaged-serve-payload.test.mjs
index 37bfe660df51..b683174df34d 100644
--- a/config/scripts/verify-packaged-serve-payload.test.mjs
+++ b/config/scripts/verify-packaged-serve-payload.test.mjs
@@ -21,8 +21,12 @@ function packagedResources({ index, assets = [], cli = true }) {
if (cli) {
mkdirSync(join(outDir, 'cli'), { recursive: true })
}
- const webDir = join(outDir, 'web')
+ const webDir = join(outDir, 'renderer')
mkdirSync(join(webDir, 'assets'), { recursive: true })
+ writeFileSync(
+ join(webDir, '.orca-web-client-files.json'),
+ JSON.stringify(['web-index.html', ...assets])
+ )
if (index !== undefined) {
writeFileSync(join(webDir, 'web-index.html'), index)
}
@@ -66,6 +70,48 @@ describe('assertPackagedWebClient', () => {
)
})
+ it.each([
+ null,
+ {},
+ ['assets/web.js'],
+ ['web-index.html', 'index.html'],
+ ['web-index.html', 'assets/../private'],
+ ['web-index.html', 'assets//private'],
+ ['web-index.html', 'assets/./private'],
+ ['web-index.html', 'assets/a\\private'],
+ ['web-index.html', 'assets/private.js.map'],
+ ['web-index.html', '/assets/a'],
+ ['web-index.html', 'assets/a\0.js'],
+ ['web-index.html', 3]
+ ])('rejects malformed shared web manifests: %j', (files) => {
+ const resourcesDir = packagedResources({
+ index: INDEX,
+ assets: ['assets/web.js', 'assets/web.css']
+ })
+ writeFileSync(
+ join(resourcesDir, 'app.asar.unpacked/out/renderer/.orca-web-client-files.json'),
+ JSON.stringify(files)
+ )
+ expect(() => assertPackagedWebClient(resourcesDir)).toThrow(/invalid web file manifest/)
+ })
+
+ it('fails when a transitive worker is missing even though HTML assets exist', () => {
+ const resourcesDir = packagedResources({
+ index: INDEX,
+ assets: ['assets/web.js', 'assets/web.css']
+ })
+ writeFileSync(
+ join(resourcesDir, 'app.asar.unpacked/out/renderer/.orca-web-client-files.json'),
+ JSON.stringify([
+ 'web-index.html',
+ 'assets/web.js',
+ 'assets/web.css',
+ 'assets/editor.worker.js'
+ ])
+ )
+ expect(() => assertPackagedWebClient(resourcesDir)).toThrow(/editor.worker.js/)
+ })
+
it('fails when the CLI the web root resolves from is not unpacked', () => {
const resourcesDir = packagedResources({
index: INDEX,
diff --git a/config/scripts/verify-skills-cli-runtime.cjs b/config/scripts/verify-skills-cli-runtime.cjs
index 0a76078d3183..df0e5e5b8127 100644
--- a/config/scripts/verify-skills-cli-runtime.cjs
+++ b/config/scripts/verify-skills-cli-runtime.cjs
@@ -53,9 +53,8 @@ function isOptionalPackageImport(artifactRoot, importer, specifier) {
if (specifier.startsWith('.') || isAbsolute(specifier)) {
return false
}
- const segments = specifier.split('/')
- const packageName = specifier.startsWith('@') ? segments.slice(0, 2).join('/') : segments[0]
- let directory = realpathSync(dirname(importer))
+ const packageName = bareSpecifierPackage(specifier)
+ let directory = dirname(importer)
while (!isOutsideRoot(artifactRoot, directory)) {
const packageJson = join(directory, 'package.json')
@@ -78,13 +77,52 @@ function isOptionalPackageImport(artifactRoot, importer, specifier) {
return false
}
+function bareSpecifierPackage(specifier) {
+ const segments = specifier.split('/')
+ return specifier.startsWith('@') ? segments.slice(0, 2).join('/') : segments[0]
+}
+
+/**
+ * The in-root path of a package import whose realpath leaves the root.
+ *
+ * Under pnpm's global virtual store `/node_modules/` links into a store shared by every
+ * worktree, so Node's realpath lands outside the root even though the artifact does carry the
+ * dependency. Accept it only when a node_modules link inside the root really leads there.
+ */
+function linkedPathInsideRoot(artifactRoot, importer, specifier, resolved) {
+ const isBare = !specifier.startsWith('.') && !isAbsolute(specifier)
+ let directory = dirname(importer)
+ while (!isOutsideRoot(artifactRoot, directory)) {
+ // A bare import reaches the store through node_modules/; a relative one through the
+ // linked package dir it sits in.
+ const candidates = isBare
+ ? [join(directory, 'node_modules', bareSpecifierPackage(specifier)), directory]
+ : [directory]
+ for (const candidate of candidates) {
+ if (!existsSync(candidate)) {
+ continue
+ }
+ const physical = realpathSync(candidate)
+ if (physical !== candidate && !isOutsideRoot(physical, resolved)) {
+ return join(candidate, relative(physical, resolved))
+ }
+ }
+ if (directory === artifactRoot) {
+ break
+ }
+ directory = dirname(directory)
+ }
+ return null
+}
+
function resolveRuntimeImport(outDir, artifactRoot, importer, specifier) {
if (BUILTINS.has(specifier) || isBuiltin(specifier)) {
return null
}
let resolved
try {
- resolved = createRequire(importer).resolve(specifier)
+ // Why physical: Node resolves from the realpath, so a store-linked importer must not see the artifact's own node_modules.
+ resolved = createRequire(realpathSync(importer)).resolve(specifier)
} catch (error) {
if (isOptionalPackageImport(artifactRoot, importer, specifier)) {
return null
@@ -96,6 +134,10 @@ function resolveRuntimeImport(outDir, artifactRoot, importer, specifier) {
)
}
if (isOutsideRoot(artifactRoot, resolved)) {
+ const linked = linkedPathInsideRoot(artifactRoot, importer, specifier, resolved)
+ if (linked !== null) {
+ return linked
+ }
throw new Error(
`[verify-skills-cli-runtime] import "${specifier}" from ` +
`${artifactPath(outDir, importer)} resolved outside ${artifactRoot}: ${resolved}`
diff --git a/config/scripts/verify-skills-cli-runtime.test.mjs b/config/scripts/verify-skills-cli-runtime.test.mjs
index 7ef48d5fffe0..e827548434d9 100644
--- a/config/scripts/verify-skills-cli-runtime.test.mjs
+++ b/config/scripts/verify-skills-cli-runtime.test.mjs
@@ -1,4 +1,4 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'
import { realpathSync } from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
@@ -65,7 +65,7 @@ describe('skills CLI runtime closure', () => {
expect(
collectRuntimeClosure(root)
- .map((file) => relative(realpathSync(root), file))
+ .map((file) => relative(realpathSync(root), file).split('\\').join('/'))
.sort()
).toEqual(['cli/handlers/skills.js', 'cli/index.js', 'shared/first.js', 'shared/second.js'])
} finally {
@@ -139,6 +139,56 @@ describe('skills CLI runtime closure', () => {
}
})
+ it('follows an in-artifact node_modules link into a shared store', async () => {
+ const root = await mkdtemp(join(tmpdir(), 'orca-skills-cli-closure-'))
+ try {
+ const artifactRoot = join(root, 'artifact')
+ const outDir = join(artifactRoot, 'out')
+ // pnpm's global virtual store: the artifact links to a slot that lives outside it.
+ const slotDir = join(root, 'store', 'linked-package')
+ await writeSkillsCliFixture(outDir, "require('linked-package')\n")
+ await mkdir(slotDir, { recursive: true })
+ await writeFile(join(slotDir, 'package.json'), JSON.stringify({ main: 'index.js' }), 'utf8')
+ await writeFile(join(slotDir, 'index.js'), "require('./inner')\n", 'utf8')
+ await writeFile(join(slotDir, 'inner.js'), '', 'utf8')
+ await mkdir(join(artifactRoot, 'node_modules'), { recursive: true })
+ await symlink(slotDir, join(artifactRoot, 'node_modules', 'linked-package'), 'junction')
+
+ const closure = collectRuntimeClosure(outDir, artifactRoot).map((file) =>
+ relative(realpathSync(artifactRoot), file).split('\\').join('/')
+ )
+ expect(closure).toContain('node_modules/linked-package/inner.js')
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
+ it("resolves a linked package's imports from its store slot, as Node does", async () => {
+ const root = await mkdtemp(join(tmpdir(), 'orca-skills-cli-closure-'))
+ try {
+ const artifactRoot = join(root, 'artifact')
+ const outDir = join(artifactRoot, 'out')
+ const slotDir = join(root, 'store', 'linked-package')
+ await writeSkillsCliFixture(outDir, "require('linked-package')\n")
+ await mkdir(slotDir, { recursive: true })
+ await writeFile(join(slotDir, 'package.json'), JSON.stringify({ main: 'index.js' }), 'utf8')
+ await writeFile(join(slotDir, 'index.js'), "require('dep')\n", 'utf8')
+ // Node reaches the ancestor copy (like ~/node_modules) from the slot, not the artifact's own.
+ for (const dir of [
+ join(root, 'node_modules', 'dep'),
+ join(artifactRoot, 'node_modules', 'dep')
+ ]) {
+ await mkdir(dir, { recursive: true })
+ await writeFile(join(dir, 'index.js'), '', 'utf8')
+ }
+ await symlink(slotDir, join(artifactRoot, 'node_modules', 'linked-package'), 'junction')
+
+ expect(() => collectRuntimeClosure(outDir, artifactRoot)).toThrow(/resolved outside/)
+ } finally {
+ await rm(root, { recursive: true, force: true })
+ }
+ })
+
it('rejects package dependencies resolved outside the artifact', async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-skills-cli-closure-'))
try {
diff --git a/config/scripts/vitest-node-runtime-files.mjs b/config/scripts/vitest-node-runtime-files.mjs
index 04d8aaf075f6..28f6c388dd8f 100644
--- a/config/scripts/vitest-node-runtime-files.mjs
+++ b/config/scripts/vitest-node-runtime-files.mjs
@@ -3,6 +3,7 @@ import { FORK_NODE_RUNTIME_INCLUDE } from './fork-vitest-node-runtime-files.mjs'
// SQLite publication, native bindings, worker IPC, socket liveness, and V8 retention use Node.
export const NODE_RUNTIME_INCLUDE = [
+ 'src/main/runtime/structured-agent-cli-writers.test.ts',
'config/scripts/vitest-sqlite-runtime-boundary.test.ts',
'config/scripts/cloud-security-history-workflow.test.mjs',
'config/scripts/skill-recipe-shell.test.mjs',
@@ -19,6 +20,8 @@ export const NODE_RUNTIME_INCLUDE = [
'src/main/ai-vault/session-parse-cache-large-scan.test.ts',
'src/main/ai-vault/session-parse-cache-metadata-checkpoint.test.ts',
'src/main/ai-vault/session-parse-cache-persistence.test.ts',
+ 'src/main/ai-vault/session-scanner-scope-cwd-facts.test.ts',
+ 'src/main/ai-vault/session-scanner-scope-walk.test.ts',
// Electron main bridges streams with Node's implementation.
'src/main/host/electron-http-client.test.ts',
'src/main/daemon/daemon-init-provider-installation.test.ts',
@@ -63,6 +66,9 @@ export const NODE_RUNTIME_INCLUDE = [
'src/main/foreign-sqlite-readers/**/*.test.ts',
'src/main/browser/*cookie*.test.ts',
// Playwright's CDP transport needs Node's WebSocket implementation.
+ 'src/main/browser/browser-popup-adapter-compatibility.electron.test.ts',
+ 'src/main/browser/popup-origin-chrome.electron.test.ts',
+ 'src/main/browser/browser-link-materialization.electron.test.ts',
'src/main/browser/browser-popup-compatibility.electron.test.ts',
'src/main/browser/browser-popup-native-compatibility.electron.test.ts',
'src/main/browser/browser-popup-background-compatibility.electron.test.ts',
@@ -87,10 +93,9 @@ export const NODE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-conversation-name.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-status-first-input-identity.test.ts',
+ 'src/main/native-chat/structured-chat-naming.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-outgrown-child.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-permission-mode.test.ts',
- 'src/main/native-chat/structured-chat-naming-command-first.test.ts',
- 'src/main/native-chat/structured-chat-naming-status-hook.test.ts',
'src/main/runtime/agent-session-conversation-name-store.test.ts',
'src/main/runtime/structured-session-mail-redrive-wiring.test.ts',
'src/main/runtime/rpc/ws-transport.test.ts',
diff --git a/config/scripts/vitest-runtime-boundary.test.ts b/config/scripts/vitest-runtime-boundary.test.ts
index 6ff715a9255f..737bd193c37a 100644
--- a/config/scripts/vitest-runtime-boundary.test.ts
+++ b/config/scripts/vitest-runtime-boundary.test.ts
@@ -27,8 +27,7 @@ it('routes native-chat journal and record-store SQLite contracts through Node',
expect.arrayContaining([
'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-status-first-input-identity.test.ts',
- 'src/main/native-chat/structured-chat-naming-command-first.test.ts',
- 'src/main/native-chat/structured-chat-naming-status-hook.test.ts',
+ 'src/main/native-chat/structured-chat-naming.test.ts',
'src/main/runtime/agent-session-conversation-name-store.test.ts',
'src/main/runtime/structured-session-mail-redrive-wiring.test.ts'
])
diff --git a/config/scripts/vitest-sqlite-database-runtime-files.mjs b/config/scripts/vitest-sqlite-database-runtime-files.mjs
index 1cc0e2c026b7..9af339e9cf7f 100644
--- a/config/scripts/vitest-sqlite-database-runtime-files.mjs
+++ b/config/scripts/vitest-sqlite-database-runtime-files.mjs
@@ -1,5 +1,7 @@
// Database opening and lifecycle fixtures exercise Node SQLite behavior.
export const SQLITE_DATABASE_RUNTIME_INCLUDE = [
+ 'src/main/ai-vault-search/session-search-write-buffer.test.ts',
+ 'src/main/ai-vault/session-document-consumer-replacement.test.ts',
'src/main/ai-vault-search/session-search-dependent-transcripts.test.ts',
'src/main/ai-vault-search/session-search-file-write.test.ts',
'src/main/ai-vault-search/session-search-live-transcript.test.ts',
@@ -188,7 +190,6 @@ export const SQLITE_DATABASE_RUNTIME_INCLUDE = [
'src/main/runtime/structured-agent-session-codex-held-sends.test.ts',
'src/main/runtime/structured-agent-session-codex-stopped-send-order.test.ts',
'src/main/runtime/structured-agent-session-codex-turn-end-settlement.test.ts',
- 'src/main/runtime/structured-agent-session-failure-logging.test.ts',
'src/main/runtime/structured-agent-session-held-signal.test.ts',
'src/main/runtime/structured-agent-session-host-refusal.test.ts',
'src/main/runtime/structured-agent-session-install-failure.test.ts',
diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs
index 9770c6559567..940009726406 100644
--- a/config/scripts/vitest-sqlite-runtime-files.mjs
+++ b/config/scripts/vitest-sqlite-runtime-files.mjs
@@ -2,6 +2,9 @@ import { SQLITE_DATABASE_RUNTIME_INCLUDE } from './vitest-sqlite-database-runtim
// Real SQLite fixtures keep Node publication and close semantics.
export const SQLITE_RUNTIME_INCLUDE = [
+ 'src/main/pi/rpc-session-adapter.test.ts',
+ 'src/main/pi/rpc-session-startup.test.ts',
+ 'src/main/acp/acp-structured-publish-at-spawn.test.ts',
'tests/e2e/native-chat-startup-option-pickers.unit.test.ts',
...SQLITE_DATABASE_RUNTIME_INCLUDE,
'src/main/ipc/repos/project-host-setup-wsl-move-fence.test.ts',
@@ -12,7 +15,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/repo-backfill-wsl-move.test.ts',
'src/main/acp/acp-structured-host-live-listing.test.ts',
'src/main/acp/acp-structured-host-configured-default.test.ts',
- 'src/main/pi/rpc-configured-default.test.ts',
'src/main/acp/acp-structured-omp-recordings.test.ts',
'src/main/acp/acp-text-stream-identity.test.ts',
'src/main/acp/acp-text-message-ownership.test.ts',
@@ -22,6 +24,8 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/acp/acp-structured-prompt-failure.test.ts',
'src/main/acp/acp-timeline-turn-failures.test.ts',
'src/main/acp/acp-structured-session-adapter.test.ts',
+ 'src/main/acp/acp-structured-session-adapter-resume.test.ts',
+ 'src/main/acp/acp-structured-host-start-aborts.test.ts',
'src/main/acp/acp-conversation-copy.test.ts',
'src/main/pi/rpc-turn-races.test.ts',
'src/main/acp/acp-structured-child-session-end.test.ts',
@@ -30,6 +34,7 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/acp/acp-structured-child-work.test.ts',
'src/main/acp/acp-structured-host-subagent-stop.test.ts',
'src/main/acp/acp-timeline-grok-subagent-retention.test.ts',
+ 'src/main/acp/acp-scripted-child-runtime.test.ts',
'src/main/active-view-persistence-boundary.test.ts',
'src/main/automations/automation-dispatch-host-fence.test.ts',
'src/main/automations/automation-owner-fencing.test.ts',
@@ -61,8 +66,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/codex/codex-structured-journal-subagent-readers.test.ts',
'src/main/codex/codex-structured-journal-translation-turn-lifecycle.test.ts',
'src/main/codex/codex-structured-journal-turn-settles-once-journal.test.ts',
- 'src/main/codex/codex-structured-question-order.test.ts',
- 'src/main/ipc/pty/ipc/spawn-commit-ssh-lease-cardinality.test.ts',
'src/main/ipc/repos/folder-workspace-attachment-handlers.test.ts',
'src/main/ipc/ssh/ssh-host-server-on-connect-wiring.test.ts',
'src/main/native-chat/agent-session-attachments/agent-session-attachment-claims.test.ts',
@@ -75,8 +78,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/agent-session-stop-note-projection.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-acquire-isolation.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-options.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-runtime-stamp.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-adopted-import.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-append-delivery.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-at-rest-commands.test.ts',
@@ -87,7 +88,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-claude-echo-working.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-claude-hung-start-stop.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-claude-opening-send-hold.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-claude-option-queue.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-claude-queued-stop.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-ends-session.test.ts',
@@ -100,7 +100,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-note-end.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-codex-stop-row.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-command-publication.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-conversation-close.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-conversation-stop.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-crash-mid-start.test.ts',
@@ -108,17 +107,12 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-create-adoption.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-drivability.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-owner-verdict.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-sink-release.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-held-child-lease-renewal.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-host-held-child-renewal.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-host-test-abandon.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-host.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-journal-corruption.test.ts',
@@ -129,23 +123,18 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-main-agent-working-agreement.test.ts',
+ 'src/main/native-chat/agent-session-wire/structured-agent-session-message-choices.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-newer-content-host.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-option-settlement.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-send-options.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-owed-work-release.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-pre-spawn-first-answer.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-processless-reservation.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-prompt-answer-receipts.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-provider-restore.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-provider-started-catalog.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-provider-started.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-provider-open.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-ready-operation-order.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-queued-action-receipts.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts',
+ 'src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-queued-repeat-press.test.ts',
@@ -155,10 +144,7 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-recovery-exits.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-repeat-press.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-repeated-stop.test.ts',
- 'src/main/native-chat/native-chat-visuals-sweep.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-resend-answer.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts',
@@ -169,27 +155,24 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-restart-recovery-output.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts',
+ 'src/main/native-chat/agent-session-wire/structured-rewind-cut.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-clear-rewind.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-send-preparation.test.ts',
+ 'src/main/native-chat/agent-session-wire/structured-agent-session-send.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-send-restarts-failed-start.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-settled-attach-retry.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-shutdown-cut-row.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts',
+ 'src/main/native-chat/agent-session-wire/structured-agent-session-start-admission.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-writer.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-host.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-startup-attempt-registrations.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-picks.test.ts',
- 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-permission-mode.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-startup-reconcile-failure.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-child-work.test.ts',
'src/main/native-chat/agent-session-wire/structured-agent-session-status-feed-child.test.ts',
@@ -230,7 +213,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/runtime/agent-session-launch-directory.test.ts',
'src/main/runtime/agent-session-launch-env-admission.test.ts',
'src/main/runtime/agent-session-lease-renewal.test.ts',
- 'src/main/runtime/agent-session-record-options.test.ts',
'src/main/runtime/agent-session-record-store-tab-table.test.ts',
'src/main/runtime/agent-session-record-store.test.ts',
'src/main/runtime/agent-session-permission-backfill.test.ts',
@@ -242,8 +224,11 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/runtime/host-terminal-close-persistence-durability.test.ts',
'src/main/runtime/orca-runtime-terminal-close-records.test.ts',
'src/main/runtime/orca-runtime-terminal-surface-close.test.ts',
+ 'src/main/runtime/runtime-terminal-orphan-adoption-rollback.test.ts',
+ 'src/main/runtime/orchestration/mailbox-owner.test.ts',
'src/main/runtime/orchestration/orchestration-party-location.test.ts',
'src/main/runtime/orchestration/structured-worker-journal-page.test.ts',
+ 'src/main/runtime/rpc/methods/agent-launch-at-rest.test.ts',
'src/main/runtime/rpc/methods/agent-launch-caller-selection.test.ts',
'src/main/runtime/rpc/methods/orchestration/federation/federation-agent-death-liveness.test.ts',
'tests/e2e/agent-launch-cli.unit.test.ts',
@@ -260,7 +245,6 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/runtime/rpc/methods/structured-agent-session-at-rest.test.ts',
'src/main/runtime/rpc/methods/structured-agent-session-close-before-create.test.ts',
'src/main/runtime/rpc/methods/structured-agent-session-create-message.test.ts',
- 'src/main/runtime/rpc/methods/structured-agent-session-hold.test.ts',
'src/main/runtime/rpc/methods/structured-agent-session-options-workspace-scope.test.ts',
'src/main/runtime/rpc/methods/structured-agent-session-restart-unregistered-agent.test.ts',
'src/main/runtime/rpc/methods/structured-agent-session-unsubscribe.test.ts',
@@ -272,6 +256,12 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/runtime/structured-agent-session-codex-interrupt-order.test.ts',
'src/main/runtime/structured-agent-session-codex-opening-send-hold.test.ts',
'src/main/runtime/runtime-client-agent-launch-settings-durability.test.ts',
+ 'src/main/runtime/structured-agent-session-codex-scripted-child.test.ts',
+ 'src/main/runtime/structured-agent-session-codex-start-deadline.test.ts',
+ 'src/main/runtime/structured-agent-session-pi-startup.test.ts',
+ 'src/main/runtime/structured-agent-session-pi-signed-out-start.test.ts',
+ 'src/main/runtime/structured-agent-session-adapter-start-failure.test.ts',
+ 'src/main/runtime/structured-agent-session-provider-operations-while-starting.test.ts',
'src/main/runtime/structured-agent-session-codex-settlement-order.test.ts',
'src/main/runtime/structured-agent-session-integration-replay.test.ts',
'src/main/runtime/structured-agent-session-integration.test.ts',
@@ -298,7 +288,7 @@ export const SQLITE_RUNTIME_INCLUDE = [
'src/main/ssh/orcad-migration-stuck-host-profiles.test.ts',
'src/main/managed-server/lifecycle/orcad-runtime-conversion.test.ts',
'src/main/ssh/orcad-unreachable-setup-release.test.ts',
- 'src/main/ssh/ssh-target-orcad-preflight.test.ts',
+ 'src/main/ssh/targets/ssh-target-orcad-preflight.test.ts',
'src/main/worktree-identity-persistence.test.ts',
'src/main/worktree-removal-close-records.test.ts',
'src/main/worktree-removal-session-partition-fencing.test.ts',
diff --git a/config/scripts/windows-process-tree-gyp-rebuild.mjs b/config/scripts/windows-process-tree-gyp-rebuild.mjs
index 77063badeb6d..a005a2aaf41a 100644
--- a/config/scripts/windows-process-tree-gyp-rebuild.mjs
+++ b/config/scripts/windows-process-tree-gyp-rebuild.mjs
@@ -13,6 +13,7 @@ import { execFileSync } from 'node:child_process'
import { copyFileSync, existsSync, mkdirSync, readFileSync, realpathSync, rmSync } from 'node:fs'
import { createRequire } from 'node:module'
import { dirname, join, resolve } from 'node:path'
+import { loadedNativeBuildDir } from './native-build-entry.cjs'
const ROOT = resolve(import.meta.dirname, '..', '..')
@@ -75,9 +76,12 @@ export function nodeGypRebuildTimeoutMs(
: 300_000
}
-/** The binary the addon actually loads. */
+/** The binary the addon actually loads: the patched lib/index.js prefers a published entry. */
export function windowsProcessTreeAddonPath(packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR) {
- return join(packageDir, 'build', 'Release', 'windows_process_tree.node')
+ return join(
+ loadedNativeBuildDir(packageDir, '@vscode/windows-process-tree', { platform: 'win32' }),
+ 'windows_process_tree.node'
+ )
}
/** The import whose absence tells the patched binary from the published prebuilt. */
@@ -263,11 +267,13 @@ export function ensureWindowsProcessTreeCommandLinePatch(
}
// Patched binding.gyp includes deps/node-addon-api; the tarball does not ship those headers.
+// `resolveFromDir` is the installed package when `packageDir` is a scratch copy outside the tree.
export function stageWindowsProcessTreeNodeAddonApiHeaders(
- packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR
+ packageDir = WINDOWS_PROCESS_TREE_PACKAGE_DIR,
+ resolveFromDir = packageDir
) {
const nodeAddonApiDir = dirname(
- createRequire(join(packageDir, 'package.json')).resolve('node-addon-api/package.json')
+ createRequire(join(resolveFromDir, 'package.json')).resolve('node-addon-api/package.json')
)
const stagedHeaderDir = join(packageDir, 'deps', 'node-addon-api')
mkdirSync(stagedHeaderDir, { recursive: true })
diff --git a/config/scripts/workflow-ref-mirror-case-safety.test.mjs b/config/scripts/workflow-ref-mirror-case-safety.test.mjs
index 10f02d398861..ca4dba9810db 100644
--- a/config/scripts/workflow-ref-mirror-case-safety.test.mjs
+++ b/config/scripts/workflow-ref-mirror-case-safety.test.mjs
@@ -12,14 +12,14 @@ const readWorkflow = (relativePath) => parse(readFileSync(join(projectDir, relat
describe('ref-mirroring vet steps', () => {
it.each(['daily', 'hourly', 'adhoc'])('%s builds only need the current commit', (channel) => {
- const job = readWorkflow(`.github/workflows/${channel}-mac-build.yml`).jobs[
- `build-${channel}-mac`
- ]
+ const { jobs } = readWorkflow(`.github/workflows/${channel}-mac-build.yml`)
+ const job = jobs[`build-${channel}-mac`]
+ const identity = channel === 'adhoc' ? jobs['orcad-template-support'] : job
const checkout = job.steps.find((step) => step.uses === 'actions/checkout@v6')
expect(checkout.with['fetch-depth']).toBe(1)
expect(job.steps.some((step) => step.run?.includes('gh release list'))).toBe(true)
expect(
- job.steps.some((step) => step.run?.includes('ORCA_PUBLISHED_VERSIONS="$published"'))
+ identity.steps.some((step) => step.run?.includes('ORCA_PUBLISHED_VERSIONS="$published"'))
).toBe(true)
})
@@ -29,9 +29,12 @@ describe('ref-mirroring vet steps', () => {
it.each(['daily', 'hourly', 'adhoc'])(
'%s versions from git tags, not main GitHub releases',
(channel) => {
- const step = readWorkflow(`.github/workflows/${channel}-mac-build.yml`).jobs[
- `build-${channel}-mac`
- ].steps.find((candidate) => candidate.name === `Compute ${channel} version`)
+ const { jobs } = readWorkflow(`.github/workflows/${channel}-mac-build.yml`)
+ const identity =
+ channel === 'adhoc' ? jobs['orcad-template-support'] : jobs[`build-${channel}-mac`]
+ const step = identity.steps.find(
+ (candidate) => candidate.name === `Compute ${channel} version`
+ )
expect(step.run).toContain('git/matching-refs/tags/v')
expect(step.run).not.toMatch(
/gh release list[\s\S]*--repo "\$GITHUB_REPOSITORY"[\s\S]*--json tagName/
diff --git a/config/scripts/wsl-e2e-lane-contract.test.mjs b/config/scripts/wsl-e2e-lane-contract.test.mjs
index a7d66b765240..90e3e5acd99a 100644
--- a/config/scripts/wsl-e2e-lane-contract.test.mjs
+++ b/config/scripts/wsl-e2e-lane-contract.test.mjs
@@ -13,6 +13,7 @@ describe('real WSL terminal lane', () => {
'src/main/wsl/wsl-runner.ts',
'src/main/pty/wsl-orca-env.ts',
'src/shared/wsl-login-shell-command.ts',
+ 'src/shared/shell-argument-quoting.ts',
'src/shared/windows-terminal-shell.ts',
'tests/e2e/helpers/wsl-golden-stub-agent.ts',
'tests/e2e/golden-tab-bar-agent-launch.spec.ts',
diff --git a/config/scripts/wsl-git-shell-benchmark.mjs b/config/scripts/wsl-git-shell-benchmark.mjs
index 2ae1f09876df..857229d44410 100644
--- a/config/scripts/wsl-git-shell-benchmark.mjs
+++ b/config/scripts/wsl-git-shell-benchmark.mjs
@@ -128,9 +128,10 @@ async function main() {
assertRepoPath(options.mountedRepo, '/mnt/')
const distro = await resolveDistro(options.distro)
const jiti = createJiti(import.meta.url)
- const { buildWslLoginShellCommand, quotePosixShell } = await jiti.import(
+ const { buildWslLoginShellCommand } = await jiti.import(
'../../src/shared/wsl-login-shell-command.ts'
)
+ const { quotePosixShell } = await jiti.import('../../src/shared/shell-argument-quoting.ts')
const loginProbe = buildWslLoginShellCommand(
`printf '\\n__ORCA_PATH__%s\\n__ORCA_GIT__%s\\n__ORCA_HOME__%s\\n' "$PATH" "$(command -v git)" "$HOME"`
diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json
index a267b4571ea3..d819e9e3f073 100644
--- a/config/tsconfig.cli.json
+++ b/config/tsconfig.cli.json
@@ -183,6 +183,8 @@
"../src/main/provider-process/provider-process-exit-deadline.ts",
"../src/main/provider-process/managed-provider-process.ts",
"../src/main/provider-process/provider-executable-missing.ts",
+ "../src/main/provider-process/provider-group-orphan-guard.ts",
+ "../src/main/provider-process/provider-process-row-read.ts",
"../src/main/provider-process/provider-process-close.ts",
"../src/main/provider-process/provider-process-launch.ts",
"../src/main/provider-process/provider-process-supervisor.ts",
diff --git a/config/tsconfig.tc.web.json b/config/tsconfig.tc.web.json
index a36f34b159e9..1228f63c2533 100644
--- a/config/tsconfig.tc.web.json
+++ b/config/tsconfig.tc.web.json
@@ -29,9 +29,13 @@
"../src/packages/process-host/src/__fixtures__/**/*",
"../src/main/gitlab/mappers.ts",
"../src/main/runtime/sidebar-inputs/sidebar-clock.ts",
- "../src/main/runtime/sidebar-inputs/sidebar-temporal-owner.ts",
- "../src/main/runtime/sidebar-inputs/catalog-listing-change.ts",
+ "../src/main/runtime/sidebar-inputs/sidebar-retention-owner.ts",
"../src/main/runtime/sidebar-inputs/producer-revision.ts",
+ "../src/main/runtime/sidebar-inputs/sidebar-inputs-feed.ts",
+ "../src/main/runtime/sidebar-inputs/sidebar-inputs-feed-log.ts",
+ "../src/main/runtime/sidebar-inputs/sidebar-inputs-feed-pages.ts",
+ "../src/main/runtime/sidebar-inputs/sidebar-inputs-feed-pins.ts",
+ "../src/main/runtime/sidebar-inputs/sidebar-inputs-feed.test-fixture.ts",
"../src/main/jira/adf-markdown.ts",
"../src/main/jira/adf-text-marks.ts",
"../src/main/jira/adf-media-destination.ts",
@@ -52,6 +56,9 @@
"../src/main/wsl-exe-process.ts",
"../src/main/persistence/applying-settings/ui-state-read.ts",
"../src/main/persistence/tracking-repos/worktree-identity-migration.ts",
+ "../src/main/persistence/tracking-repos/worktree-identity-row-repoint.ts",
+ "../src/main/persistence/restoring-sessions/tab-terminal-pane-binding-repair.ts",
+ "../src/main/persistence/window-save-view.ts",
"../src/main/persistence/applying-settings/ui-state-update.ts",
"../src/main/persistence/applying-settings/ui-selection-normalization.ts",
"../src/main/persistence/applying-settings/ui-interaction-merge.ts",
diff --git a/config/unproven-pty-retirement-baseline.txt b/config/unproven-pty-retirement-baseline.txt
new file mode 100644
index 000000000000..77ae1a160214
--- /dev/null
+++ b/config/unproven-pty-retirement-baseline.txt
@@ -0,0 +1,13 @@
+# PTY retirements on no exit proof per file: PtyRetirementWithoutProof.because, and raw
+# ptyOwnership.delete outside finishPtyShutdown. This is a RATCHET: counts may only go DOWN.
+# Retire through finishPtyShutdown on a PtyExitProof, or record the stop durably first.
+# Prune after a fix: pnpm check:unproven-pty-retirement-ratchet --prune
+
+1 src/main/ipc/pty/ipc/renderer-kill.ts # SSH provider unregistered: R4-4 tab-close replay
+1 src/main/ipc/pty/pane/absent-pane-owner.ts # host-confirmed absence, untyped
+1 src/main/ipc/pty/provider/bind-listeners.ts # observed provider exit, untyped
+1 src/main/ipc/pty/provider/local-configure.ts # observed local exit, untyped
+1 src/main/ipc/pty/provider/ownership-state.ts # deletePtyOwnership, test-only caller
+1 src/main/ipc/pty/provider/state-cleanup.ts # clearPtyOwnershipForConnection, test-only caller
+3 src/main/ipc/pty/runtime/kill.ts # 2x SSH provider unregistered (R4-4); unconfirmed follow-up stop of a rejected spawn
+1 src/main/ssh/legacy-relay/legacy-relay-pane-stops.ts # COMPAT(legacy-relay-bridge): deleted with the folder
diff --git a/config/vitest.config.ts b/config/vitest.config.ts
index 6af92217a6e6..49ef5ffcfca1 100644
--- a/config/vitest.config.ts
+++ b/config/vitest.config.ts
@@ -1,3 +1,4 @@
+import { createRequire } from 'node:module'
import { resolve } from 'node:path'
import { defaultExclude, defineConfig } from 'vitest/config'
import { UNIT_INCLUDE, UNIT_EXCLUDE } from './scripts/ci-unit-files.mjs'
@@ -16,7 +17,9 @@ const transforms = {
define: {
ORCA_FEATURE_WALL_ENABLED: 'true',
// Why: tests only interpolate it; they must not depend on the checkout's HEAD.
- ORCA_BUILD_COMMIT: JSON.stringify('0'.repeat(40))
+ ORCA_BUILD_COMMIT: JSON.stringify('0'.repeat(40)),
+ // A dark switch set to 'main' in the env (`ORCA_LAYOUT_OWNER=main pnpm vitest`) is on in tests.
+ ...createRequire(import.meta.url)('./dark-build-switches.cjs').turnedOnDarkBuildDefines()
},
resolve: {
alias: [
diff --git a/docs/STYLEGUIDE.md b/docs/STYLEGUIDE.md
index 94c9db8b3296..08accad5fcf3 100644
--- a/docs/STYLEGUIDE.md
+++ b/docs/STYLEGUIDE.md
@@ -185,6 +185,8 @@ Sizes: `default` (36px), `sm` (32px), `xs` (24px), `lg` (40px), plus `icon`, `ic
Browse `src/renderer/src/components/ui/` for the full list. Most wrap a Radix UI primitive — exceptions are `command` (wraps `cmdk`), `sonner` (wraps `sonner`), and the visual-only wrappers (`badge`, `button-group`, `card`, `input`) which apply tokens and Tailwind utilities directly. Never reimplement headless behavior; extend the existing wrapper.
+Use `DialogContent surface="security"` for every dialog that asks the user to trust, approve, run or remove code, including plugin provenance and approval, so plugin themes cannot change the trust surface's tokens. Use its `scrollable` prop when the whole dialog scrolls. Wrap a trust decision outside a dialog, such as an agent approval card or a certificate override, in `SecuritySurface`.
+
### Picking the right primitive
When a control has multiple plausible primitives, use this fork:
diff --git a/docs/assets/wechat-qr-group13.jpg b/docs/assets/wechat-qr-group13.jpg
new file mode 100644
index 000000000000..48c67e4031bf
Binary files /dev/null and b/docs/assets/wechat-qr-group13.jpg differ
diff --git a/docs/bug-reproductions/delayed-agent-launch.md b/docs/bug-reproductions/delayed-agent-launch.md
index 0652aacd23c3..e26ce2e63e1b 100644
--- a/docs/bug-reproductions/delayed-agent-launch.md
+++ b/docs/bug-reproductions/delayed-agent-launch.md
@@ -59,14 +59,14 @@ sleep-length polling was added.
## Coverage and remaining gaps
-| Surface | Evidence |
-| --- | --- |
-| Local and daemon preparation | Plugin files, account environment and host spawn tests |
-| Renderer replies and routing | Hidden query-delivery and captured late-title tests |
-| macOS real Claude | Hidden Electron/CDP launch and completed prompt |
-| SSH/remote-runtime | Shared classification; existing fenced process reads retained; no physical delayed launch |
-| WSL | Shared Pi/OMP and OpenCode preparation; physical guest launch unproved |
-| Linux/Windows | Platform-neutral classification and existing mock coverage; physical delayed launch unproved |
+| Surface | Evidence |
+| ---------------------------- | -------------------------------------------------------------------------------------------- |
+| Local and daemon preparation | Plugin files, account environment and host spawn tests |
+| Renderer replies and routing | Hidden query-delivery and captured late-title tests |
+| macOS real Claude | Hidden Electron/CDP launch and completed prompt |
+| SSH/remote-runtime | Shared classification; existing fenced process reads retained; no physical delayed launch |
+| WSL | Shared Pi/OMP and OpenCode preparation; physical guest launch unproved |
+| Linux/Windows | Platform-neutral classification and existing mock coverage; physical delayed launch unproved |
The scanner intentionally does not parse arbitrary shell programs, variables,
pipelines or background jobs. A late agent emitting neither a recognizable title
diff --git a/docs/readme/README.es.md b/docs/readme/README.es.md
index 76b997525ab3..6d3bc84cdf3d 100644
--- a/docs/readme/README.es.md
+++ b/docs/readme/README.es.md
@@ -68,7 +68,7 @@ Terminales de nivel Ghostty con renderizado WebGL, divisiones infinitas y un scr
-
+
@@ -96,7 +96,7 @@ Explora PRs, issues y tableros de proyecto dentro de la app — abre un worktree
-
+
@@ -110,7 +110,7 @@ Ejecuta agentes en una máquina remota potente con edición completa de archivos
-
+
@@ -152,7 +152,7 @@ Los agentes también manejan Orca — automatiza cualquier flujo de trabajo con
-
+
diff --git a/docs/readme/README.fr.md b/docs/readme/README.fr.md
index 3d81041f93cd..dfab23efe009 100644
--- a/docs/readme/README.fr.md
+++ b/docs/readme/README.fr.md
@@ -72,7 +72,7 @@ Terminaux de niveau Ghostty avec rendu WebGL, splits infinis et un scrollback qu
-
+
@@ -100,7 +100,7 @@ Parcourez PRs, issues et boards de projet dans l'app — ouvrez un worktree depu
-
+
@@ -114,7 +114,7 @@ Faites tourner des agents sur une machine distante costaude, avec édition de fi
-
+
@@ -156,7 +156,7 @@ Les agents pilotent aussi Orca — scriptez n'importe quel workflow avec `orca w
@@ -96,7 +96,7 @@ PR, issue, 프로젝트 보드를 앱 안에서 탐색하세요 — 어떤 작
-
+
@@ -110,7 +110,7 @@ PR, issue, 프로젝트 보드를 앱 안에서 탐색하세요 — 어떤 작
-
+
@@ -152,7 +152,7 @@ diff의 어느 줄에든 코멘트를 남기고 에이전트에게 바로 보내
-
+
diff --git a/docs/readme/README.pt.md b/docs/readme/README.pt.md
index 44b87a271fe3..727f683b53fb 100644
--- a/docs/readme/README.pt.md
+++ b/docs/readme/README.pt.md
@@ -68,7 +68,7 @@ Terminais no nível do Ghostty com renderização WebGL, divisões infinitas e s
-
+
@@ -96,7 +96,7 @@ Navegue por PRs, issues e quadros de projeto dentro do app — abra um worktree
-
+
@@ -110,7 +110,7 @@ Execute agentes em uma máquina remota potente com edição completa de arquivos
-
+
@@ -152,7 +152,7 @@ Agentes também controlam o Orca — automatize qualquer fluxo de trabalho com `