diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 555017223ea9..d92892a425e8 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -37,6 +37,24 @@ In PowerShell, use `$env:ORCA_DEV_MAIN_SOURCEMAPS='0'` before `pnpm dev`. Remove the variable when debugging the main process with source maps. Renderer and preload settings are unaffected; production builds always keep their hidden main maps. +Local installs use pnpm's global virtual store: each package version is one slot +under `~/pnpm/store` that every worktree links to, instead of a full `node_modules` per worktree. +An existing worktree converts on its next `pnpm install`, and an older branch without the setting +converts back the same way. CI always uses the normal layout. On Windows, pnpm links with +junctions when you lack the symlink privilege, so no administrator rights or Developer Mode are +needed. Things to know: + +- Keep `~/node_modules` (and any other `node_modules` above the store) out of your home + directory. Node resolves upward from the store slot, so a stray one there supplies + dependencies the package never declared and hides missing-dependency bugs; `pnpm install` warns + when it finds one. +- Native addons are never rebuilt in place in a shared slot: each CPU gets its own build + beside it (see [`native-build-entry.cjs`](../config/scripts/native-build-entry.cjs)), so + cross-arch packaging is safe for other worktrees. +- `ORCA_PNPM_GLOBAL_VIRTUAL_STORE=0` opts out. Keep it set + in your shell: `pnpm run` re-checks the install and relinks a worktree whose layout no longer + matches. + Ordinary installs include native optional dependencies for the current OS and CPU only. Before a cross-architecture build (including `pnpm build:mac`, which produces both x64 and arm64 artifacts by default), run `pnpm install:release` to add the other CPU's variants. diff --git a/.github/actions/install-node-dependencies/action.yml b/.github/actions/install-node-dependencies/action.yml index b81dd6aed515..1332106a4db6 100644 --- a/.github/actions/install-node-dependencies/action.yml +++ b/.github/actions/install-node-dependencies/action.yml @@ -86,7 +86,7 @@ runs: case "$LOOKUP_REQUEST" in [aA][uU][tT][oO]) # Hosted runners have Node for this manifest-only check before toolchain setup. - lookup_only="$(node -p 'const p = require("./package.json"); p.engines?.node === "24" && typeof p.packageManager === "string" && p.packageManager.split("+")[0] === "pnpm@12.8.1"')" + lookup_only="$(node -p 'const p = require("./package.json"); p.engines?.node === "24" && typeof p.packageManager === "string" && p.packageManager.split("+")[0] === "pnpm@12.14.0"')" ;; esac printf 'lookup-only=%s\n' "$lookup_only" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/prepare-headless-compiler/action.yml b/.github/actions/prepare-headless-compiler/action.yml index 9e71e6df1b07..b6e60d989986 100644 --- a/.github/actions/prepare-headless-compiler/action.yml +++ b/.github/actions/prepare-headless-compiler/action.yml @@ -14,7 +14,7 @@ runs: - id: identity shell: bash env: - COMPILER_POLICY_HASH: ${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', 'config/patches/**', '.github/actions/install-node-dependencies/**', '.github/actions/restore-pnpm-verification/**', 'config/scripts/headless-detector-compiler-cache.mjs', '.github/actions/prepare-headless-compiler/action.yml') }} + COMPILER_POLICY_HASH: ${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', 'config/pnpmfile.cjs', 'config/patches/**', '.github/actions/install-node-dependencies/**', '.github/actions/restore-pnpm-verification/**', 'config/scripts/headless-detector-compiler-cache.mjs', '.github/actions/prepare-headless-compiler/action.yml') }} run: node config/scripts/headless-detector-compiler-cache.mjs identity - id: cache uses: actions/cache/restore@v5 diff --git a/.github/actions/prepare-linux-package-fixture/action.yml b/.github/actions/prepare-linux-package-fixture/action.yml index e3f297f4b104..146d4c47b25d 100644 --- a/.github/actions/prepare-linux-package-fixture/action.yml +++ b/.github/actions/prepare-linux-package-fixture/action.yml @@ -7,10 +7,16 @@ inputs: save-cache: description: Build and save a missing fixture on the default branch. default: 'false' + load: + description: Load the image here; 'false' only restores, so the caller can load it in a background run step. + default: 'true' outputs: image: description: Loaded image to use as a Docker build cache, or empty on a cache miss. value: ${{ steps.image.outputs.image }} + cache-hit: + description: Whether the fixture archive was restored under the exact key. + value: ${{ steps.cache.outputs.cache-hit }} runs: using: composite steps: @@ -20,39 +26,17 @@ runs: continue-on-error: true with: path: ${{ runner.temp }}/orca-package-fixtures/${{ inputs.fixture }}.tar - key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml') }} + key: linux-package-fixture-v1-${{ runner.os }}-${{ runner.arch }}-linux-amd64-${{ inputs.fixture }}-${{ hashFiles(format('config/docker/{0}/**', inputs.fixture), '.github/actions/prepare-linux-package-fixture/action.yml', '.github/actions/prepare-linux-package-fixture/load-fixture.sh') }} - name: Load or seed fixture build cache id: image + if: inputs.load != 'false' shell: bash env: FIXTURE: ${{ inputs.fixture }} SAVE_CACHE: ${{ inputs.save-cache }} CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} - run: | - set -euo pipefail - case "$FIXTURE" in - headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;; - *) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;; - esac - archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar" - image="orca-package-fixture-$FIXTURE:cache" - if [[ "$CACHE_HIT" == true ]] && docker load --input "$archive"; then - if docker image inspect "$image" > /dev/null; then - echo "image=$image" >> "$GITHUB_OUTPUT" - exit 0 - fi - fi - if [[ "$SAVE_CACHE" != true ]]; then - echo 'No usable fixture cache; the package runner will build normally.' - exit 0 - fi - docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \ - --tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE" - mkdir -p "$(dirname "$archive")" - docker save --output "$archive" "$image" - echo "image=$image" >> "$GITHUB_OUTPUT" - echo 'built=true' >> "$GITHUB_OUTPUT" + run: bash "$GITHUB_ACTION_PATH/load-fixture.sh" - name: Save fixture image if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && steps.image.outputs.built == 'true' diff --git a/.github/actions/prepare-linux-package-fixture/load-fixture.sh b/.github/actions/prepare-linux-package-fixture/load-fixture.sh new file mode 100755 index 000000000000..9d76ac92806f --- /dev/null +++ b/.github/actions/prepare-linux-package-fixture/load-fixture.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Loads a restored fixture image; only the default-branch warmer builds and exports a missing one. +set -euo pipefail +case "$FIXTURE" in + headless-serve-shutdown|cli-launch-contract|daemon-shutdown-descendants) ;; + *) echo "Unsupported package fixture: $FIXTURE" >&2; exit 1 ;; +esac +archive="$RUNNER_TEMP/orca-package-fixtures/$FIXTURE.tar" +image="orca-package-fixture-$FIXTURE:cache" +if [[ "${CACHE_HIT:-}" == true ]] && docker load --input "$archive"; then + if docker image inspect "$image" > /dev/null; then + echo "image=$image" >> "$GITHUB_OUTPUT" + exit 0 + fi +fi +if [[ "${SAVE_CACHE:-}" != true ]]; then + echo 'No usable fixture cache; the package runner will build normally.' + exit 0 +fi +docker build --platform linux/amd64 --build-arg BUILDKIT_INLINE_CACHE=1 \ + --tag "$image" --file "config/docker/$FIXTURE/Dockerfile" "config/docker/$FIXTURE" +mkdir -p "$(dirname "$archive")" +docker save --output "$archive" "$image" +echo "image=$image" >> "$GITHUB_OUTPUT" +echo 'built=true' >> "$GITHUB_OUTPUT" diff --git a/.github/actions/prepare-native-runtime/action.yml b/.github/actions/prepare-native-runtime/action.yml index 76f9954b7e2c..61c289c052b8 100644 --- a/.github/actions/prepare-native-runtime/action.yml +++ b/.github/actions/prepare-native-runtime/action.yml @@ -54,7 +54,7 @@ runs: env: NATIVE_RUNTIME: ${{ inputs.native-runtime }} NODE_VERSION: ${{ inputs.node-version }} - NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', '.pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/initialize-windows-native-toolchain.mjs', 'config/scripts/capture-windows-native-environment.cmd', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }} + NATIVE_SOURCE_HASH: ${{ hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', 'config/pnpmfile.cjs', '.github/actions/prepare-native-runtime/action.yml', 'config/scripts/ensure-native-runtime.mjs', 'config/scripts/initialize-windows-native-toolchain.mjs', 'config/scripts/capture-windows-native-environment.cmd', 'config/scripts/rebuild-native-deps.mjs', 'config/scripts/node-pty-job-ownership.cjs', 'config/scripts/windows-pe-machine.cjs', 'config/scripts/windows-process-tree-gyp-rebuild.mjs', 'config/scripts/windows-process-tree-creation-time.cjs', 'config/scripts/install-electron-package-binary.mjs', 'config/scripts/electron-platform-path.mjs', 'config/scripts/zip-extractor-command.mjs', 'src/shared/zip-extractor-command.ts', 'config/scripts/shared-electron-dist-cache.mjs', 'config/scripts/space-sharing-copy.mjs', 'config/patches/node-pty@1.1.0.patch', 'config/patches/@vscode__windows-process-tree@0.8.0.patch', 'native/windows-registry/src/addon.cc', 'native/windows-registry/binding.gyp', 'native/windows-registry/package.json', 'native/windows-registry/index.js', 'native/windows-security/src/addon.cc', 'native/windows-security/binding.gyp', 'native/windows-security/package.json', 'native/windows-security/index.js') }} run: | scope="${ImageOS:-$RUNNER_OS}" if [ -r /etc/os-release ]; then diff --git a/.github/actions/prepare-wayland-clipboard/action.yml b/.github/actions/prepare-wayland-clipboard/action.yml index 78c622855bd5..c35382714207 100644 --- a/.github/actions/prepare-wayland-clipboard/action.yml +++ b/.github/actions/prepare-wayland-clipboard/action.yml @@ -5,6 +5,14 @@ inputs: save-cache: description: Publish a validated helper from the default branch. default: 'false' + build: + description: Build here; 'false' only restores, so the caller can build in a background run step. + default: 'true' + +outputs: + cache-hit: + description: Whether the helper was restored under the exact key. + value: ${{ steps.cache.outputs.cache-hit }} runs: using: composite @@ -30,22 +38,14 @@ runs: key: ${{ steps.identity.outputs.key }} - name: Build or validate Wayland clipboard helper + if: inputs.build != 'false' shell: bash env: CACHE_HIT: ${{ steps.cache.outputs.cache-hit }} - run: | - if node config/scripts/build-wayland-clipboard.mjs; then - exit 0 - fi - if [ "$CACHE_HIT" != true ]; then - exit 1 - fi - echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.' - rm -rf -- native/wayland-clipboard/.build - node config/scripts/build-wayland-clipboard.mjs + run: bash "$GITHUB_ACTION_PATH/build-or-validate.sh" - name: Save validated Wayland clipboard helper from main - if: inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + if: inputs.build != 'false' && inputs.save-cache == 'true' && steps.cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' uses: actions/cache/save@v5 continue-on-error: true with: diff --git a/.github/actions/prepare-wayland-clipboard/build-or-validate.sh b/.github/actions/prepare-wayland-clipboard/build-or-validate.sh new file mode 100755 index 000000000000..d71b802edc11 --- /dev/null +++ b/.github/actions/prepare-wayland-clipboard/build-or-validate.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Validates a restored helper and rebuilds once from source when the restored copy is unusable. +set -e +if node config/scripts/build-wayland-clipboard.mjs; then + exit 0 +fi +if [ "${CACHE_HIT:-}" != true ]; then + exit 1 +fi +echo 'Discarding an unusable Wayland clipboard cache and rebuilding from source.' +rm -rf -- native/wayland-clipboard/.build +node config/scripts/build-wayland-clipboard.mjs diff --git a/.github/workflows/adhoc-mac-build.yml b/.github/workflows/adhoc-mac-build.yml index b270cb727250..3772f4e4d061 100644 --- a/.github/workflows/adhoc-mac-build.yml +++ b/.github/workflows/adhoc-mac-build.yml @@ -128,7 +128,7 @@ jobs: echo "Resolved $REQUESTED_REF -> $sha" echo "sha=$sha" >>"$GITHUB_OUTPUT" - # Why: a branch cut before the orcad template landed has none to build or ship. + # Resolve shared build identity while detecting whether this ref ships a template. orcad-template-support: needs: resolve-ref runs-on: ubuntu-latest @@ -137,13 +137,17 @@ jobs: contents: read outputs: ships: ${{ steps.detect.outputs.ships }} + version: ${{ steps.adhoc.outputs.version }} + commit: ${{ steps.adhoc.outputs.commit }} + name: ${{ steps.adhoc.outputs.name }} steps: - - name: Checkout the template packager only + - name: Checkout template detection and version scripts uses: actions/checkout@v6 with: ref: ${{ needs.resolve-ref.outputs.sha }} sparse-checkout: | - /config/scripts/packaged-orcad-template.cjs + /config/scripts/ + /package.json /config/tsconfig.relay.json sparse-checkout-cone-mode: false persist-credentials: false @@ -166,6 +170,47 @@ jobs: echo "::notice::This ref predates the orcad template; the build ships without it." fi + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: package.json + package-manager-cache: false + + - name: Compute adhoc version + id: adhoc + shell: bash + env: + REF: ${{ inputs.ref || github.ref_name }} + LABEL: ${{ inputs.label }} + MAIN_REPO_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Why this check: the version scripts are read from the branch being built, + # not from main, so a branch cut before the adhoc channel landed has no + # copy of them. Say that plainly instead of failing with a module-not-found. + for script in adhoc-build-version dev-channel-base-version; do + if [[ ! -f "config/scripts/$script.mjs" ]]; then + echo "::error::$REF has no config/scripts/$script.mjs; rebase it onto a main that has the adhoc channel." + exit 1 + fi + done + # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the + # GitHub release and leaves the tag, which still owns that number. + # Releases-only let adhoc sit on a number already taken, so the updater + # would not install it. Empty on failure — the script then falls back + # to package.json. + published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \ + "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \ + --jq '.[].ref | sub("^refs/tags/"; "")' || true)" + ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \ + node config/scripts/adhoc-build-version.mjs \ + >"$RUNNER_TEMP/adhoc-identity.txt" + if ! grep -q '^name=' "$RUNNER_TEMP/adhoc-identity.txt"; then + echo "::error::adhoc-build-version.mjs emitted no release name; $REF's copy of the script is out of sync with this workflow." + exit 1 + fi + cat "$RUNNER_TEMP/adhoc-identity.txt" >>"$GITHUB_OUTPUT" + # Design D2, as release-cut does: every desktop build ships the orcad template (server JS plus # every target's addons), merged from the node-server lanes that qualify each slot at this ref. # Without it an adhoc build cannot deploy managed orcad to an SSH host. No secrets, and the mac @@ -199,8 +244,8 @@ jobs: environment: adhoc-mac-build outputs: tag: ${{ steps.release.outputs.tag }} - version: ${{ steps.adhoc.outputs.version }} - head_sha: ${{ steps.adhoc.outputs.head_sha }} + version: ${{ needs.orcad-template-support.outputs.version }} + head_sha: ${{ needs.resolve-ref.outputs.sha }} published: ${{ steps.publish_live.outcome == 'success' && 'true' || 'false' }} ships_orcad_template: ${{ needs.orcad-template-support.outputs.ships }} runs-on: blacksmith-6vcpu-macos-15 @@ -343,42 +388,6 @@ jobs: APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} - - name: Compute adhoc version - id: adhoc - shell: bash - env: - REF: ${{ inputs.ref || github.ref_name }} - LABEL: ${{ inputs.label }} - MAIN_REPO_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - # Why this check: the version scripts are read from the branch being built, - # not from main, so a branch cut before the adhoc channel landed has no - # copy of them. Say that plainly instead of failing with a module-not-found. - for script in adhoc-build-version dev-channel-base-version; do - if [[ ! -f "config/scripts/$script.mjs" ]]; then - echo "::error::$REF has no config/scripts/$script.mjs; rebase it onto a main that has the adhoc channel." - exit 1 - fi - done - echo "head_sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT" - # Why git tags, not GitHub releases: unpublishing a buggy cut deletes the - # GitHub release and leaves the tag, which still owns that number. - # Releases-only let adhoc sit on a number already taken, so the updater - # would not install it. Empty on failure — the script then falls back - # to package.json. - published="$(GH_TOKEN="$MAIN_REPO_TOKEN" gh api \ - "repos/$GITHUB_REPOSITORY/git/matching-refs/tags/v" \ - --jq '.[].ref | sub("^refs/tags/"; "")' || true)" - ORCA_PUBLISHED_VERSIONS="$published" ORCA_ADHOC_LABEL="${LABEL:-$REF}" \ - node config/scripts/adhoc-build-version.mjs \ - >"$RUNNER_TEMP/adhoc-identity.txt" - if ! grep -q '^name=' "$RUNNER_TEMP/adhoc-identity.txt"; then - echo "::error::adhoc-build-version.mjs emitted no release name; $REF's copy of the script is out of sync with this workflow." - exit 1 - fi - cat "$RUNNER_TEMP/adhoc-identity.txt" >>"$GITHUB_OUTPUT" - - name: Build app run: pnpm build:release env: @@ -414,9 +423,9 @@ jobs: shell: bash env: GH_TOKEN: ${{ steps.app_token.outputs.token }} - TAG: v${{ steps.adhoc.outputs.version }} - NAME: ${{ steps.adhoc.outputs.name }} - SHA: ${{ steps.adhoc.outputs.head_sha }} + TAG: v${{ needs.orcad-template-support.outputs.version }} + NAME: ${{ needs.orcad-template-support.outputs.name }} + SHA: ${{ needs.resolve-ref.outputs.sha }} REF: ${{ inputs.ref || github.ref_name }} # Via env, not inline `${{ }}`: both land inside a shell string, and an # expression expanded there is substituted before bash parses the line @@ -463,13 +472,13 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_ADHOC=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_ADHOC=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Why: electron-builder's github publisher targets the repo named in the # config; the token must therefore carry write access to orca-adhoc. GH_TOKEN: ${{ steps.app_token.outputs.token }} - ORCA_ADHOC_BUILD_VERSION: ${{ steps.adhoc.outputs.version }} - ORCA_BUILD_COMMIT: ${{ steps.adhoc.outputs.commit }} + ORCA_ADHOC_BUILD_VERSION: ${{ needs.orcad-template-support.outputs.version }} + ORCA_BUILD_COMMIT: ${{ needs.orcad-template-support.outputs.commit }} # beforePack and afterPack fail the package when the template is absent. ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ needs.orcad-template-support.outputs.ships == 'true' && '1' || '' }} CSC_LINK: ${{ secrets.MAC_CERTS }} @@ -516,7 +525,7 @@ jobs: env: GH_TOKEN: ${{ steps.app_token.outputs.token }} TAG: ${{ steps.release.outputs.tag }} - NAME: ${{ steps.adhoc.outputs.name }} + NAME: ${{ needs.orcad-template-support.outputs.name }} run: | set -euo pipefail # --title again: electron-builder resolves this draft by tag and may @@ -597,27 +606,37 @@ jobs: echo "::warning::Could not prune $tag" done <<<"$stale" - # Why this runs after the mac leg rather than beside it: the tag and version - # are computed inside that job, so until it has run nothing else can name the - # release to upload into. The cost is small enough not to matter — the Windows - # leg measures ~7.5 min (install 2m45, build 35s, NSIS package 3m) against a - # ~9.5 min mac run, which keeps a adhoc run far inside its interval. - # - # Why `./` rather than a pinned `@main`: `uses:` resolves against the ref this - # file itself came from, which for an ordinary dispatch is main. Someone who - # deliberately points the Actions "Use workflow from" picker at a branch - # already gets that branch's copy of this entire file, so this follows the same - # rule instead of inventing a second one. + # Build unsigned Windows artifacts alongside Mac; publication waits below. build-adhoc-win: - needs: build-adhoc-mac - # Only once the mac release is actually live: there is no release to upload - # into otherwise, and the Windows workflow refuses to create one. - if: needs.build-adhoc-mac.outputs.published == 'true' + needs: [resolve-ref, orcad-template-support, orcad-template] + if: >- + !cancelled() && github.repository == 'stablyai/orca' && + needs.orcad-template-support.result == 'success' && + (needs.orcad-template.result == 'success' || + (needs.orcad-template.result == 'skipped' && + needs.orcad-template-support.outputs.ships == 'false')) uses: ./.github/workflows/dev-channel-win-build.yml secrets: inherit + with: + channel: adhoc + tag: v${{ needs.orcad-template-support.outputs.version }} + ref: ${{ needs.resolve-ref.outputs.sha }} + version: ${{ needs.orcad-template-support.outputs.version }} + orcad_template: ${{ needs.orcad-template-support.outputs.ships == 'true' }} + stage_only: true + + # Upload only after Mac succeeds; an unsigned build may overlap its notarization. + publish-adhoc-win: + needs: [build-adhoc-mac, build-adhoc-win] + if: >- + !cancelled() && needs.build-adhoc-mac.result == 'success' && + needs.build-adhoc-win.result == 'success' && + needs.build-adhoc-mac.outputs.published == 'true' + permissions: + contents: read + actions: read + uses: ./.github/workflows/dev-channel-win-publish.yml + secrets: inherit with: channel: adhoc tag: ${{ needs.build-adhoc-mac.outputs.tag }} - ref: ${{ needs.build-adhoc-mac.outputs.head_sha }} - version: ${{ needs.build-adhoc-mac.outputs.version }} - orcad_template: ${{ needs.build-adhoc-mac.outputs.ships_orcad_template == 'true' }} diff --git a/.github/workflows/ci-cache-warmup-platforms.yml b/.github/workflows/ci-cache-warmup-platforms.yml index 06ba3eaeb3b0..35a667ef6aae 100644 --- a/.github/workflows/ci-cache-warmup-platforms.yml +++ b/.github/workflows/ci-cache-warmup-platforms.yml @@ -20,7 +20,7 @@ on: - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' - '.npmrc' - - '.pnpmfile.cjs' + - 'config/pnpmfile.cjs' - 'config/scripts/ensure-native-runtime.mjs' - 'config/scripts/initialize-windows-native-toolchain.mjs' - 'config/scripts/capture-windows-native-environment.cmd' diff --git a/.github/workflows/ci-cache-warmup.yml b/.github/workflows/ci-cache-warmup.yml index e360f5b02e8a..f1d2f4423af1 100644 --- a/.github/workflows/ci-cache-warmup.yml +++ b/.github/workflows/ci-cache-warmup.yml @@ -29,7 +29,7 @@ on: - 'pnpm-lock.yaml' - 'pnpm-workspace.yaml' - '.npmrc' - - '.pnpmfile.cjs' + - 'config/pnpmfile.cjs' - 'config/scripts/ensure-native-runtime.mjs' - 'config/scripts/initialize-windows-native-toolchain.mjs' - 'config/scripts/capture-windows-native-environment.cmd' diff --git a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml index 728e8eabb658..cd41c5281bc8 100644 --- a/.github/workflows/cloud-deploy-relay-production-capacity-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-capacity-job.yml @@ -160,9 +160,10 @@ jobs: github-token: ${{ github.token }} run-id: ${{ inputs.monitor-run-id }} - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/cloud-deploy-relay-production-capacity.yml b/.github/workflows/cloud-deploy-relay-production-capacity.yml index 66a2f734ffb0..7f75b7e8d3e0 100644 --- a/.github/workflows/cloud-deploy-relay-production-capacity.yml +++ b/.github/workflows/cloud-deploy-relay-production-capacity.yml @@ -129,9 +129,10 @@ jobs: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/cloud-deploy-relay-production-multi-target.yml b/.github/workflows/cloud-deploy-relay-production-multi-target.yml index 9c850a31d67d..d780c402a960 100644 --- a/.github/workflows/cloud-deploy-relay-production-multi-target.yml +++ b/.github/workflows/cloud-deploy-relay-production-multi-target.yml @@ -186,9 +186,10 @@ jobs: github-token: ${{ github.token }} run-id: ${{ inputs.monitor-run-id }} - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index 8d659ce33499..c0ae3451dc72 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -91,8 +91,8 @@ jobs: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - with: { package_json_file: cloud/package.json } + - uses: pnpm/setup@v2 + with: { package-json-file: cloud/package.json, install: false } - uses: actions/setup-node@v4 with: @@ -441,9 +441,15 @@ jobs: if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end ] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \ <<< "${CURRENT_RUNTIME}" - CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \ + if ! CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \ "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")"; then + if jq -e '.error == "cell_in_reserve_mode"' \ + "${RUNNER_TEMP}/current-cell-status.json" >/dev/null 2>&1; then + echo "${TARGET_CELL_ID} is in reserve mode: flip it to admitMode=db with the cell-flags workflow and wait for it to record db before rolling it, then flip it back to reserve" >&2 + fi + exit 1 + fi SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ <<< "${CURRENT_DIRECTOR_STATUS}")" if test "${ROLLBACK_RESUME}" = true && ! jq -e \ diff --git a/.github/workflows/cloud-deploy-relay-production.yml b/.github/workflows/cloud-deploy-relay-production.yml index 257e7aeb06aa..9aa7af1f5599 100644 --- a/.github/workflows/cloud-deploy-relay-production.yml +++ b/.github/workflows/cloud-deploy-relay-production.yml @@ -83,9 +83,10 @@ jobs: github-token: ${{ github.token }} run-id: ${{ inputs.monitor-run-id }} - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/cloud-monitor-relay-production-job.yml b/.github/workflows/cloud-monitor-relay-production-job.yml index 663b40961ac7..1c10b9059345 100644 --- a/.github/workflows/cloud-monitor-relay-production-job.yml +++ b/.github/workflows/cloud-monitor-relay-production-job.yml @@ -57,9 +57,10 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/cloud-operate-relay-asia-admission.yml b/.github/workflows/cloud-operate-relay-asia-admission.yml index 0a0e764d9ffb..79752e43db66 100644 --- a/.github/workflows/cloud-operate-relay-asia-admission.yml +++ b/.github/workflows/cloud-operate-relay-asia-admission.yml @@ -194,9 +194,10 @@ jobs: with: node-version: 24 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false if: ${{ steps.inputs.outputs.canary == 'true' }} - name: Install exact canary dependencies diff --git a/.github/workflows/cloud-operate-relay-production-cell-flags.yml b/.github/workflows/cloud-operate-relay-production-cell-flags.yml index c11fd5db52ca..d3e0e2d9fd4f 100644 --- a/.github/workflows/cloud-operate-relay-production-cell-flags.yml +++ b/.github/workflows/cloud-operate-relay-production-cell-flags.yml @@ -32,7 +32,8 @@ on: Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck, admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs, - reregisterInFlight + reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip + must name phonePath) required: true type: string expected-generation: @@ -62,7 +63,7 @@ defaults: jobs: operate: if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls. timeout-minutes: 60 environment: production diff --git a/.github/workflows/cloud-operate-relay-production-rehome-job.yml b/.github/workflows/cloud-operate-relay-production-rehome-job.yml index 587cd9558696..cbc3efd38d0b 100644 --- a/.github/workflows/cloud-operate-relay-production-rehome-job.yml +++ b/.github/workflows/cloud-operate-relay-production-rehome-job.yml @@ -134,9 +134,9 @@ jobs: --drain-grace-ms "${DRAIN_GRACE_MS}" --confirmation "${CONFIRMATION}" \ | tee "${RUNNER_TEMP}/relay-rehome-control.json" - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 if: ${{ inputs.mode == 'enable' }} - with: { package_json_file: cloud/package.json } + with: { package-json-file: cloud/package.json, install: false } - run: pnpm install --frozen-lockfile if: ${{ inputs.mode == 'enable' }} diff --git a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml index e045d10928e7..2711cd062107 100644 --- a/.github/workflows/cloud-operate-relay-production-reserve-watch.yml +++ b/.github/workflows/cloud-operate-relay-production-reserve-watch.yml @@ -25,15 +25,17 @@ on: type: string flipped-at: description: >- - When the reserve write applied, ISO with a zone, e.g. 2026-10-14T12:00:00Z. The 503 - background is the median of the 10 minutes before it + When the reserve (or phonePath=memory) write applied, ISO with a zone, e.g. + 2026-10-14T12:00:00Z. The non-drain 503 background and the phone rules' background are + the 10 minutes before it required: false default: '' type: string background-503-per-minute: description: >- - Director 503s per minute before the flip; overrides the one measured before flipped-at. - One of the two is required + Director non-drain 503s per minute before the flip (all 503s less drain-return deferrals + and a host's own early or duplicate re-dials, from the director's runtime metrics); + overrides the one measured before flipped-at. One of the two is required required: false default: '' type: string @@ -71,13 +73,15 @@ env: jobs: watch: if: ${{ vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' }} - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 # Six 55-minute segments cover the longest watch; under the 360-minute job limit. timeout-minutes: 350 environment: production outputs: state: ${{ steps.outcome.outputs.state }} verdict: ${{ steps.outcome.outputs.verdict }} + # phonePath when a step-6 phone rule tripped; anything else flips the cell back whole. + flip: ${{ steps.outcome.outputs.flip }} env: DURATION_MINUTES: ${{ inputs.duration-minutes }} FLIPPED_AT: ${{ inputs.flipped-at }} @@ -88,6 +92,10 @@ jobs: set -euo pipefail [[ "${CELL_ID}" =~ ^production-gce-c[0-9]{1,3}$ ]] node -e "if (!/^[^.]+[.][^.]+[.][^.]+$/.test(process.env.ORCA_RELAY_ADMIN_ID_TOKEN ?? '')) process.exit(1)" + # A phone-rule trip writes phonePath=db mid-watch, through the flag tool. + ORCA_RELAY_CONTROL_ACCESS_TOKEN="$(gcloud auth print-access-token)" + echo "::add-mask::${ORCA_RELAY_CONTROL_ACCESS_TOKEN}" + export ORCA_RELAY_CONTROL_ACCESS_TOKEN node dev/scripts/watch-relay-reserve-cell.mjs watch \ --mode "${MODE}" --cell-id "${CELL_ID}" \ --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ @@ -247,7 +255,7 @@ jobs: # Missing data is never healthy: a watch that crashed, timed out or was cancelled (by an # operator too) flips back like a trip. In dry-run this only plans the write. if: ${{ always() && vars.ORCA_CLOUD_OPERATIONS_ENABLED == 'true' && needs.watch.result != 'skipped' && !contains(fromJSON('["completed","stopped","refused"]'), needs.watch.outputs.state) }} - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2204 # A flip back waits for the cell to lease every control: 8-50 minutes, scaled to its controls. timeout-minutes: 60 environment: production @@ -257,6 +265,7 @@ jobs: # lease wait can still kill the process before Postgres records db: the file says db, Postgres reserve. env: VERDICT: ${{ needs.watch.outputs.verdict || format('the watch ended {0} without a verdict', needs.watch.result) }} + FLIP: ${{ needs.watch.outputs.flip || 'admitMode' }} steps: - uses: actions/checkout@v4 if: ${{ always() }} @@ -296,7 +305,7 @@ jobs: --mode "${MODE}" --cell-id "${CELL_ID}" \ --cell-origin "https://${CELL_ID#production-gce-}.relay.onorca.dev" \ --project-id "${GCP_PROJECT_ID}" --director-origin "https://relay.onorca.dev" \ - --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" \ + --confirmation "${CONFIRMATION}" --verdict "${VERDICT}" --flip "${FLIP}" \ | tee "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl" # The flip-back line is logged only once the write (or its dry-run plan) finished; without @@ -307,12 +316,15 @@ jobs: if grep -qs '"event":"orca_relay_reserve_watch_flip_back"' "${RUNNER_TEMP}/relay-reserve-flip-back.jsonl"; then exit 0 fi - { - echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" - echo "Flip it to db by hand with Operate Relay Production Cell Flags." - } >> "${GITHUB_STEP_SUMMARY}" + if [[ "${FLIP}" == phonePath ]]; then + echo "## FLIP BACK FAILED: ${CELL_ID} may still have phonePath=memory" >> "${GITHUB_STEP_SUMMARY}" + echo "Set phonePath=default by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}" + else + echo "## FLIP BACK FAILED: ${CELL_ID} may still be in reserve mode" >> "${GITHUB_STEP_SUMMARY}" + echo "Flip it to db by hand with Operate Relay Production Cell Flags." >> "${GITHUB_STEP_SUMMARY}" + fi node dev/scripts/watch-relay-reserve-cell.mjs alarm --kind flip-back-failed \ - --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" + --mode "${MODE}" --cell-id "${CELL_ID}" --project-id "${GCP_PROJECT_ID}" --verdict "${VERDICT}" --flip "${FLIP}" - name: Audit summary if: ${{ always() }} diff --git a/.github/workflows/cloud-operate-relay-staging-cell-flags.yml b/.github/workflows/cloud-operate-relay-staging-cell-flags.yml index 12781fd16a99..501d00faebc1 100644 --- a/.github/workflows/cloud-operate-relay-staging-cell-flags.yml +++ b/.github/workflows/cloud-operate-relay-staging-cell-flags.yml @@ -33,7 +33,8 @@ on: Switches to change, e.g. readinessLocal=true,admitMode=reserve (key=default removes a key). Every other switch keeps its current value. Known: readinessLocal, ticketCheck, admitMode, intakePerSec, reserveDryRun, rejectionFence, readTimeoutMarginMs, - reregisterInFlight + reregisterInFlight, phonePath (memory only takes effect in reserve; a reserve flip + must name phonePath) required: true type: string expected-generation: diff --git a/.github/workflows/cloud-prove-relay-asia-staging.yml b/.github/workflows/cloud-prove-relay-asia-staging.yml index ad94ee002791..e214a8a330d5 100644 --- a/.github/workflows/cloud-prove-relay-asia-staging.yml +++ b/.github/workflows/cloud-prove-relay-asia-staging.yml @@ -124,9 +124,10 @@ jobs: with: node-version: 24 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - name: Require the exact staging director image before promotion env: @@ -433,9 +434,10 @@ jobs: with: node-version: 24 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - name: Install exact load-harness dependencies run: pnpm install --frozen-lockfile diff --git a/.github/workflows/cloud-verify.yml b/.github/workflows/cloud-verify.yml index c39edcb11a38..4d0f74949a2f 100644 --- a/.github/workflows/cloud-verify.yml +++ b/.github/workflows/cloud-verify.yml @@ -86,9 +86,10 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: @@ -125,9 +126,10 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 + - uses: pnpm/setup@v2 with: - package_json_file: cloud/package.json + package-json-file: cloud/package.json + install: false - uses: actions/setup-node@v4 with: diff --git a/.github/workflows/daily-mac-build.yml b/.github/workflows/daily-mac-build.yml index 28e95ba96d64..ce0f2fc7a045 100644 --- a/.github/workflows/daily-mac-build.yml +++ b/.github/workflows/daily-mac-build.yml @@ -388,7 +388,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_DAILY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_DAILY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Why: electron-builder's github publisher targets the repo named in the # config; the token must therefore carry write access to orca-daily. diff --git a/.github/workflows/dev-channel-win-build.yml b/.github/workflows/dev-channel-win-build.yml index 194659964863..adebe5da5e75 100644 --- a/.github/workflows/dev-channel-win-build.yml +++ b/.github/workflows/dev-channel-win-build.yml @@ -5,11 +5,8 @@ name: Dev Channel Windows Build # Windows artifact can be rebuilt for an existing tag without paying for the mac # leg's packaging and notarization again. # -# Each mac workflow calls this as a `needs:`-gated job once its release is live, -# passing the tag it created. Both legs land in that one release, so a tag -# carries every platform it managed to build. Measured cost of the Windows leg is -# ~7.5 min (install 2m45, build 35s, NSIS package 3m) against a mac run of ~9.5 -# min, so running it after the mac job keeps an hourly well inside its cron. +# Adhoc can stage Windows alongside macOS, then publish only after macOS succeeds. +# Hourly, daily, and manual dispatches keep packaging and publishing together. # # Why unsigned: Windows release installers are signed by SignPath *after* # packaging, and release-cut budgets 1h + 4h for those approval waits. That does @@ -27,9 +24,8 @@ name: Dev Channel Windows Build # installer run. Every way out — to another dev build, or back to Stable — works # through the in-app updater. The picker offers a download for exactly that jump. # -# Called as a job by each mac workflow once its release is live, and separately -# dispatchable by hand to rebuild a Windows artifact for an existing tag without -# re-running the mac leg's twenty minutes of packaging and notarization: +# Mac workflows call this directly or stage artifacts for a later publish job. +# It is also dispatchable to rebuild Windows for an existing tag: # # gh workflow run dev-channel-win-build.yml --ref main \ # -f channel=adhoc -f tag=v1.4.178-adhoc.20260819010203 \ @@ -47,7 +43,7 @@ on: required: true type: string tag: - description: Existing release tag in the channel repo + description: Release tag for staged or published artifacts required: true type: string ref: @@ -58,6 +54,11 @@ on: description: Version to package, without the leading v required: true type: string + stage_only: + description: Package unsigned artifacts for a later gated publish job + required: false + type: boolean + default: false orcad_template: description: Ship the orcad-template artifact the calling run built required: false @@ -276,6 +277,7 @@ jobs: # Why the token is minted here and not at the top: installation tokens live # one hour and nothing before this point writes anything. - name: Mint dev channel repo token + if: ${{ !inputs.stage_only }} id: app_token uses: actions/create-github-app-token@v2 with: @@ -289,6 +291,7 @@ jobs: # while this was building, that would mint a fresh, untitled, Windows-only # release. Check first and fail instead. - name: Confirm the target release still exists + if: ${{ !inputs.stage_only }} shell: bash env: GH_TOKEN: ${{ steps.app_token.outputs.token }} @@ -300,6 +303,7 @@ jobs: fi - name: Publish Windows artifacts + if: ${{ !inputs.stage_only }} uses: nick-fields/retry@v4 with: # 30 is the pack + upload budget; there is no notary queue on this leg. @@ -323,6 +327,7 @@ jobs: # release carrying the installer but not the manifest is one the picker # offers and the update 404s on, so assert both. - name: Verify Windows update manifest published + if: ${{ !inputs.stage_only }} shell: bash env: GH_TOKEN: ${{ steps.app_token.outputs.token }} @@ -338,3 +343,41 @@ jobs: fi done echo "Windows artifacts verified on $TAG." + + - name: Package Windows artifacts without publishing + if: inputs.stage_only + uses: nick-fields/retry@v4 + with: + timeout_minutes: 30 + max_attempts: 2 + retry_wait_seconds: 30 + command: node config/scripts/ensure-native-runtime.mjs --runtime=electron; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never + env: + ORCA_BUILD_COMMIT: ${{ inputs.ref }} + ORCA_REQUIRE_ORCAD_TEMPLATE: ${{ inputs.orcad_template && '1' || '' }} + + - name: Validate staged Windows artifacts + if: inputs.stage_only + shell: bash + run: | + set -euo pipefail + for required in orca-windows-setup.exe orca-windows-setup.exe.blockmap latest.yml; do + if [[ ! -s "dist/$required" ]]; then + echo "::error::Missing or empty staged Windows artifact: $required" + exit 1 + fi + done + + - name: Stage Windows artifacts for gated publishing + if: inputs.stage_only + uses: actions/upload-artifact@v7 + with: + name: dev-channel-windows-${{ inputs.tag }} + path: | + dist/orca-windows-setup.exe + dist/orca-windows-setup.exe.blockmap + dist/latest.yml + if-no-files-found: error + retention-days: 3 + compression-level: 0 + overwrite: true diff --git a/.github/workflows/dev-channel-win-publish.yml b/.github/workflows/dev-channel-win-publish.yml new file mode 100644 index 000000000000..6864d1277d00 --- /dev/null +++ b/.github/workflows/dev-channel-win-publish.yml @@ -0,0 +1,107 @@ +name: Dev Channel Windows Publish + +on: + workflow_call: + inputs: + channel: + description: Dev channel whose live release receives the staged artifacts + required: true + type: string + tag: + description: Release tag used when staging the Windows artifacts + required: true + type: string + +permissions: + contents: read + actions: read + +concurrency: + group: dev-channel-win-publish-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + publish-win: + if: github.repository == 'stablyai/orca' + environment: adhoc-mac-build + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + CHANNEL: ${{ inputs.channel }} + TAG: ${{ inputs.tag }} + steps: + - name: Vet the requested inputs + shell: bash + run: | + set -euo pipefail + case "$CHANNEL" in + hourly|daily|adhoc) ;; + *) echo "::error::Unknown dev channel '$CHANNEL'."; exit 1 ;; + esac + if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-${CHANNEL}\.[0-9]+$ ]]; then + echo "::error::Tag '$TAG' is not a $CHANNEL release tag." + exit 1 + fi + + # No run-id: artifacts must belong to this run, never another dispatch. + - name: Download the staged Windows artifacts + uses: actions/download-artifact@v8 + with: + name: dev-channel-windows-${{ inputs.tag }} + path: staged-windows + + - name: Validate staged Windows artifacts + shell: bash + run: | + set -euo pipefail + for required in orca-windows-setup.exe orca-windows-setup.exe.blockmap latest.yml; do + if [[ ! -s "staged-windows/$required" ]]; then + echo "::error::Missing or empty staged Windows artifact: $required" + exit 1 + fi + done + + - name: Mint dev channel repo token + id: app_token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.HOURLY_RELEASE_APP_ID }} + private-key: ${{ secrets.HOURLY_RELEASE_APP_PRIVATE_KEY }} + owner: stablyai + repositories: orca-${{ inputs.channel }} + + - name: Confirm the target release is live + shell: bash + env: + GH_TOKEN: ${{ steps.app_token.outputs.token }} + run: | + set -euo pipefail + draft="$(gh release view "$TAG" --repo "stablyai/orca-$CHANNEL" --json isDraft --jq '.isDraft')" + if [[ "$draft" != false ]]; then + echo "::error::Release $TAG is not live; refusing to publish Windows artifacts." + exit 1 + fi + + - name: Publish Windows artifacts + uses: nick-fields/retry@v4 + with: + timeout_minutes: 5 + max_attempts: 2 + retry_wait_seconds: 10 + command: gh release upload "$TAG" staged-windows/orca-windows-setup.exe staged-windows/orca-windows-setup.exe.blockmap staged-windows/latest.yml --clobber --repo "stablyai/orca-$CHANNEL" + env: + GH_TOKEN: ${{ steps.app_token.outputs.token }} + + - name: Verify Windows update manifest published + shell: bash + env: + GH_TOKEN: ${{ steps.app_token.outputs.token }} + run: | + set -euo pipefail + assets="$(gh release view "$TAG" --repo "stablyai/orca-$CHANNEL" --json assets --jq '.assets[].name')" + for required in latest.yml orca-windows-setup.exe orca-windows-setup.exe.blockmap; do + if ! grep -qx "$required" <<<"$assets"; then + echo "::error::$TAG is missing $required; Windows could not install this build." + exit 1 + fi + done diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a06601679553..dd78c63edd40 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -47,35 +47,32 @@ jobs: persist-credentials: false - name: Setup pnpm - # v5 avoids the v6 bootstrap/shim regression when pinning pnpm 10. - uses: pnpm/action-setup@v5 + uses: pnpm/setup@v2 with: - version: 10.34.6 - package_json_file: docs/site/package.json - run_install: false + package-json-file: docs/site/package.json + install: false - name: Setup Node.js uses: actions/setup-node@v6 with: - node-version: 22 - # The repository root pins pnpm 12 while this isolated app pins 10; - # setup-node's cache probe runs at the root and would auto-switch. + node-version-file: docs/site/package.json + # setup-node probes the store from the repository root, not this app. package-manager-cache: false - name: Install site dependencies - run: pnpm --ignore-workspace install --frozen-lockfile + run: pnpm install --frozen-lockfile - name: Run package tests - run: pnpm --ignore-workspace test + run: pnpm test - name: Lint site - run: pnpm --ignore-workspace lint + run: pnpm lint - name: Typecheck site - run: pnpm --ignore-workspace exec tsc --noEmit --incremental false + run: pnpm exec tsc --noEmit --incremental false - name: Build site - run: pnpm --ignore-workspace build + run: pnpm build release_gate: name: Authorize release @@ -222,21 +219,19 @@ jobs: persist-credentials: false - name: Setup pnpm - # v5 avoids the v6 bootstrap/shim regression when pinning pnpm 10. - uses: pnpm/action-setup@v5 + uses: pnpm/setup@v2 with: - version: 10.34.6 - package_json_file: docs/site/package.json - run_install: false + package-json-file: docs/site/package.json + install: false - name: Setup Node.js uses: actions/setup-node@v6 with: - node-version: 22 + node-version-file: docs/site/package.json package-manager-cache: false - name: Install site dependencies - run: pnpm --ignore-workspace install --frozen-lockfile + run: pnpm install --frozen-lockfile - name: Verify Vercel credentials env: @@ -254,23 +249,23 @@ jobs: VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: pnpm --ignore-workspace exec vercel pull --yes --non-interactive --environment=production + run: pnpm exec vercel pull --yes --non-interactive --environment=production - name: Run package tests - run: pnpm --ignore-workspace test + run: pnpm test - name: Lint site - run: pnpm --ignore-workspace lint + run: pnpm lint - name: Typecheck site - run: pnpm --ignore-workspace exec tsc --noEmit --incremental false + run: pnpm exec tsc --noEmit --incremental false - name: Build production site - run: pnpm --ignore-workspace exec vercel build --prod --non-interactive + run: pnpm exec vercel build --prod --non-interactive - name: Deploy production site env: VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - run: pnpm --ignore-workspace exec vercel deploy --prebuilt --prod --yes --non-interactive + run: pnpm exec vercel deploy --prebuilt --prod --yes --non-interactive diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4dfc477850c7..ef402098804a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -27,12 +27,23 @@ on: required: false type: boolean default: true + managed_ssh_shards: + description: JSON shard assignments for the selected managed SSH specs + required: false + type: string + default: '["1/2","2/2"]' # Why: artifact names are unique per workflow run, and pr.yml calls this workflow twice. artifact_suffix: description: Appended to the build and trace artifact names so two calls in one run do not collide required: false type: string default: '' + # Why: the app, orcad and its test template (--dark-test-template) build with it; releases refuse it. + layout_owner: + description: Set to main to build with ORCA_LAYOUT_OWNER=main and record save-divergence reports + required: false + type: string + default: '' workflow_dispatch: inputs: ref: @@ -43,6 +54,12 @@ on: description: JSON array of specs to run; empty runs the full suite required: false type: string + managed_ssh_shards: + description: Managed SSH shard assignments (use one for small selections) + required: false + type: choice + default: '["1/2","2/2"]' + options: ['["1/2","2/2"]', '["1/1"]'] schedule: # One complete daily reference run; targeted PR coverage remains unchanged. - cron: '0 17 * * *' @@ -107,7 +124,12 @@ jobs: env: VITE_EXPOSE_STORE: 'true' ORCA_CI_MAIN_SOURCEMAPS: '0' - run: pnpm run build:electron-vite:parallel --mode e2e + LAYOUT_OWNER: ${{ inputs.layout_owner }} + run: | + if [ -n "$LAYOUT_OWNER" ]; then + export ORCA_LAYOUT_OWNER="$LAYOUT_OWNER" + fi + pnpm run build:electron-vite:parallel --mode e2e # The CLI emits into out/main too; start only after Electron finishes clearing that tree. - name: Build shared E2E CLI @@ -125,6 +147,11 @@ jobs: - wait: e2e-cli + # The switch-on specs skip, rather than fail, when the build left no marker. + - name: Check layout-owner build marker + if: inputs.layout_owner == 'main' + run: test -f out/main/ORCA_LAYOUT_OWNER_MAIN + - name: Check shared native runtime action id: native-runtime-action run: | @@ -138,10 +165,8 @@ jobs: - uses: ./.github/actions/build-mobile-web-bundle if: hashFiles('.github/actions/build-mobile-web-bundle/action.yml') != '' - # Only the completed output upload overlaps the change from Node to Electron ABI. + # Foreground: a background JavaScript action reads the event payload while later steps rewrite it. - name: Upload E2E build output - id: e2e-build-upload - background: true uses: actions/upload-artifact@v7 with: name: e2e-build-out${{ inputs.artifact_suffix }} @@ -164,8 +189,6 @@ jobs: with: native-runtime: electron - - wait: e2e-build-upload - e2e: name: e2e ${{ matrix.shard_name }} needs: [build] @@ -325,6 +348,7 @@ jobs: - name: Run changed E2E specs env: TEST_FILES_JSON: ${{ inputs.test_files }} + LAYOUT_OWNER: ${{ inputs.layout_owner }} run: | if [ -f config/scripts/ci-e2e-job-selection.mjs ]; then printf '%s\n' "$TEST_FILES_JSON" | node config/scripts/ci-e2e-job-selection.mjs > "$RUNNER_TEMP/general-e2e-specs" @@ -347,12 +371,43 @@ jobs: fi E2E_ENV=(SKIP_BUILD=1 ORCA_E2E_FORWARD_APP_LOGS=1 ORCA_E2E_WEB_CLIENT=1) E2E_PROJECT_ARGS=() + if [ -n "$LAYOUT_OWNER" ]; then + test -f out/main/ORCA_LAYOUT_OWNER_MAIN + mkdir -p layout-divergence + E2E_ENV+=(ORCA_LAYOUT_SAVE_DIVERGENCE_LOG="$PWD/layout-divergence/window-specs.jsonl") + E2E_ENV+=(PLAYWRIGHT_JSON_OUTPUT_FILE="$PWD/layout-divergence/window-specs-results.json") + E2E_PROJECT_ARGS+=('--reporter=list,json') + fi if grep -l '@headful' "${TEST_FILES[@]}" >/dev/null; then E2E_PROJECT_ARGS+=(--project=electron-headful) fi xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh env "${E2E_ENV[@]}" \ pnpm run test:e2e "${TEST_FILES[@]}" --workers=1 "${E2E_PROJECT_ARGS[@]}" + - name: Summarize layout save divergence + if: always() && inputs.layout_owner == 'main' + continue-on-error: true + run: &summarize_layout_divergence | + # Either file names a run: a log with no report is unattributed, a report with no log is zero. + shopt -s nullglob + runs=(layout-divergence/*.jsonl layout-divergence/*-results.json) + for run in $(printf '%s\n' "${runs[@]%.jsonl}" | sed 's/-results\.json$//' | sort -u); do + node config/scripts/layout-save-divergence-summary.mjs "$run.jsonl" "$run-summary.json" "$run-results.json" + if [ -f "$run-results.json" ]; then + node config/scripts/ci-e2e-failure-summary.mjs "$run-results.json" + fi + done + + - name: Upload layout save divergence report + if: always() && inputs.layout_owner == 'main' + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: layout-divergence-changed${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }} + path: layout-divergence/ + retention-days: 14 + if-no-files-found: warn + - name: Upload Playwright traces if: failure() || cancelled() uses: actions/upload-artifact@v7 @@ -410,12 +465,24 @@ jobs: # The packaged orcad slot the T6-11 launcher runs: its own node-pty under the pinned Node. # The template is what `orca serve`'s default selection materializes that slot from. - name: Build this runner's orcad slot and template + env: + LAYOUT_OWNER: ${{ inputs.layout_owner }} run: | slot="$(node config/scripts/build-orcad-prebuilds.mjs --print-slot)" pnpm build:orcad-prebuilds pnpm build:orcad-prebuilds --require-slots "$slot" - pnpm build:orcad - node config/scripts/build-orcad-template.mjs --targets "$slot" + # The headless oracle's orcad host must hold the same layout owner as the Electron host, + # and so must the template `orca serve` materializes its server from. + if [ -n "$LAYOUT_OWNER" ]; then + export ORCA_LAYOUT_OWNER="$LAYOUT_OWNER" + pnpm build:orcad + node config/scripts/build-orcad-template.mjs --targets "$slot" --dark-test-template + test -f out/orcad/ORCA_LAYOUT_OWNER_MAIN + test -f out/orcad-template/ORCA_LAYOUT_OWNER_MAIN + else + pnpm build:orcad + node config/scripts/build-orcad-template.mjs --targets "$slot" + fi - name: Switch serve hosts on one profile if: inputs.test_files == '' || contains(inputs.test_files, 'tests/e2e/orcad-serve-mode-switch.spec.ts') env: @@ -431,7 +498,30 @@ jobs: env: SKIP_BUILD: '1' ORCA_E2E_FORWARD_APP_LOGS: '1' - run: xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/workspace-layout-oracle-headless.spec.ts --project=electron-headless --workers=1 + LAYOUT_OWNER: ${{ inputs.layout_owner }} + run: | + REPORT_ARGS=() + if [ -n "$LAYOUT_OWNER" ]; then + test -f out/main/ORCA_LAYOUT_OWNER_MAIN + mkdir -p layout-divergence + export ORCA_LAYOUT_SAVE_DIVERGENCE_LOG="$PWD/layout-divergence/headless-oracle.jsonl" + export PLAYWRIGHT_JSON_OUTPUT_FILE="$PWD/layout-divergence/headless-oracle-results.json" + REPORT_ARGS=('--reporter=list,json') + fi + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/workspace-layout-oracle-headless.spec.ts --project=electron-headless --workers=1 "${REPORT_ARGS[@]}" + - name: Summarize layout save divergence + if: always() && inputs.layout_owner == 'main' + continue-on-error: true + run: *summarize_layout_divergence + - name: Upload layout save divergence report + if: always() && inputs.layout_owner == 'main' + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: layout-divergence-headless${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }} + path: layout-divergence/ + retention-days: 14 + if-no-files-found: warn - uses: actions/upload-artifact@v7 if: failure() with: @@ -440,7 +530,7 @@ jobs: retention-days: 7 if-no-files-found: ignore - # Every managed-orcad SSH spec on a Docker host, including #24979's relay-era conversion. Needs the + # Selected managed-orcad SSH specs on a Docker host; daily runs keep the full suite. Needs the # orcad template for the fixture's target (Debian, linux-x64-glibc), which only this job builds. orcad-ssh-docker: name: ssh on managed orcad (Docker ${{ matrix.shard }}) @@ -451,7 +541,7 @@ jobs: strategy: fail-fast: false matrix: - shard: ['1/2', '2/2'] + shard: ${{ fromJSON(inputs.managed_ssh_shards || '["1/2","2/2"]') }} env: ORCA_BACKGROUND_LAUNCH: '1' steps: @@ -472,23 +562,68 @@ jobs: name: e2e-build-out${{ inputs.artifact_suffix }} path: out/ # Built to a path the app does not look at by default, so the relay-era phase has no template. + # The switch-on leg deploys a switch-on server, so both sides of the host run one layout owner. - name: Build the linux-x64-glibc orcad template + env: + LAYOUT_OWNER: ${{ inputs.layout_owner }} run: | pnpm build:orcad-prebuilds pnpm build:orcad-prebuilds --require-slots linux-x64-glibc - node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc + if [ -n "$LAYOUT_OWNER" ]; then + ORCA_LAYOUT_OWNER="$LAYOUT_OWNER" node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc --dark-test-template + test -f out/orcad-template/ORCA_LAYOUT_OWNER_MAIN + else + node config/scripts/build-orcad-template.mjs --targets linux-x64-glibc + fi mv out/orcad-template "$RUNNER_TEMP/orcad-convert-template" # The Windows CLI matrix collects here too and skips: it needs a Windows host descriptor. - name: Run managed SSH specs on a Docker host env: + TEST_FILES_JSON: ${{ inputs.test_files }} SKIP_BUILD: '1' ORCA_E2E_SSH_DOCKER: '1' ORCA_E2E_ORCAD_CONVERT_HOST: docker ORCA_E2E_FORWARD_APP_LOGS: '1' SHARD: ${{ matrix.shard }} + LAYOUT_OWNER: ${{ inputs.layout_owner }} + SHARD_INDEX: ${{ strategy.job-index }} run: | + node --input-type=module <<'NODE' > "$RUNNER_TEMP/managed-ssh-specs" + import { existsSync } from 'node:fs' + const modulePath = './config/scripts/ci-e2e-job-selection.mjs' + const selection = existsSync(modulePath) ? await import(modulePath) : {} + // Older checkout refs retain their full-suite behavior. + const specs = selection.selectManagedSshE2eSpecs + ? selection.selectManagedSshE2eSpecs(process.env.TEST_FILES_JSON) + : ['tests/e2e/ssh-orcad-'] + for (const spec of specs) { + console.log(spec.endsWith('.spec.ts') ? spec.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '$' : spec) + } + NODE + mapfile -t TEST_FILES < "$RUNNER_TEMP/managed-ssh-specs" export ORCA_E2E_ORCAD_CONVERT_TEMPLATE="$RUNNER_TEMP/orcad-convert-template" - xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts tests/e2e/ssh-orcad- --project=electron-headless --workers=1 --shard="$SHARD" + REPORT_ARGS=() + if [ -n "$LAYOUT_OWNER" ]; then + test -f out/main/ORCA_LAYOUT_OWNER_MAIN + mkdir -p layout-divergence + export ORCA_LAYOUT_SAVE_DIVERGENCE_LOG="$PWD/layout-divergence/managed-ssh-$SHARD_INDEX.jsonl" + export PLAYWRIGHT_JSON_OUTPUT_FILE="$PWD/layout-divergence/managed-ssh-$SHARD_INDEX-results.json" + REPORT_ARGS=('--reporter=list,json') + fi + xvfb-run --auto-servernum bash .github/scripts/e2e-with-window-manager.sh pnpm exec playwright test --config tests/playwright.config.ts "${TEST_FILES[@]}" --project=electron-headless --workers=1 --shard="$SHARD" "${REPORT_ARGS[@]}" + - name: Summarize layout save divergence + if: always() && inputs.layout_owner == 'main' + continue-on-error: true + run: *summarize_layout_divergence + - name: Upload layout save divergence report + if: always() && inputs.layout_owner == 'main' + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: layout-divergence-managed-ssh-${{ strategy.job-index }}${{ inputs.artifact_suffix }}-attempt-${{ github.run_attempt }} + path: layout-divergence/ + retention-days: 14 + if-no-files-found: warn - uses: actions/upload-artifact@v7 if: failure() with: diff --git a/.github/workflows/hourly-mac-build.yml b/.github/workflows/hourly-mac-build.yml index f78cc9c47e1b..4235713a9d5a 100644 --- a/.github/workflows/hourly-mac-build.yml +++ b/.github/workflows/hourly-mac-build.yml @@ -361,7 +361,7 @@ jobs: timeout_minutes: 45 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && if [ -f config/scripts/build-mac-notarization-overlap.mjs ]; then ORCA_MAC_HOURLY=1 node config/scripts/build-mac-notarization-overlap.mjs --publish always; else ORCA_MAC_HOURLY=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always; fi env: # Why: electron-builder's github publisher targets the repo named in the # config; the token must therefore carry write access to orca-hourly. diff --git a/.github/workflows/layout-owner-e2e.yml b/.github/workflows/layout-owner-e2e.yml new file mode 100644 index 000000000000..4bafb59ac827 --- /dev/null +++ b/.github/workflows/layout-owner-e2e.yml @@ -0,0 +1,32 @@ +name: Layout owner E2E (manual) +run-name: Layout owner E2E ${{ inputs.ref || github.ref }} + +on: + workflow_dispatch: + inputs: + ref: + description: Commit or ref to test (defaults to the workflow commit) + required: false + type: string + +permissions: + contents: read + +# This experiment costs about an hour of runner time; keep one suite active repository-wide. +concurrency: + group: layout-owner-e2e + cancel-in-progress: false + +jobs: + layout_owner_switch_on: + name: layout owner (switch on) + permissions: + contents: read + uses: ./.github/workflows/e2e.yml + with: + ref: ${{ inputs.ref || github.sha }} + test_files: '["tests/e2e/workspace-layout-oracle-headless.spec.ts","tests/e2e/workspace-layout-oracle.spec.ts","tests/e2e/cli-file-open-position.spec.ts","tests/e2e/relaunch-first-terminal.spec.ts","tests/e2e/relaunch-sleeping-agents-resume-on-view.spec.ts","tests/e2e/terminal-cold-activation-deferral.spec.ts","tests/e2e/cli-first-worktree-single-terminal.spec.ts","tests/e2e/unopened-workspace-agent-single-resume.spec.ts","tests/e2e/terminal-pane-close-layout-consistency.spec.ts","tests/e2e/golden-file-open-edit-save.spec.ts","tests/e2e/golden-source-control-open-diff.spec.ts","tests/e2e/paired-cli-terminal-graph-sync-tab-retention.spec.ts","tests/e2e/floating-workspace-pane-height.spec.ts","tests/e2e/editor-model-host-isolation.spec.ts","tests/e2e/editor-tab-selection-restore.spec.ts","tests/e2e/finished-agent-ghost-resume.spec.ts","tests/e2e/live-background-terminal-mount-authority.spec.ts","tests/e2e/workspace-back-forward-navigation.spec.ts","tests/e2e/worktree-lineage-toggle-shortcut.spec.ts","tests/e2e/local-cli-terminal-retention.spec.ts","tests/e2e/default-tabs-mark-crash-relaunch.spec.ts","tests/e2e/terminal-cli-split-closed-before-mount.spec.ts","tests/e2e/layout-stream-loss-reopen.spec.ts","tests/e2e/terminal-tab-close-restart-persistence.spec.ts","tests/e2e/artificial-opencode-terminal-load.spec.ts","tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts","tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts","tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts","tests/e2e/headless-paired-remote-terminal-viewer-size.spec.ts","tests/e2e/headless-serve-browser-tab-restart.spec.ts","tests/e2e/headless-serve-cli-terminal-retention-parity.spec.ts","tests/e2e/headless-serve-desktop-activation.spec.ts","tests/e2e/headless-serve-focused-terminal-create.spec.ts","tests/e2e/headless-serve-mobile-chat-tab-order.spec.ts","tests/e2e/headless-serve-mobile-native-chat-identity.spec.ts","tests/e2e/headless-serve-session-history-naming-requests.spec.ts","tests/e2e/layout-compat-old-phone-on-new-server.spec.ts","tests/e2e/layout-compat-serve-restart-keeps-at-at-terminals.spec.ts","tests/e2e/paired-mobile-serve-restart-terminal-tabs.spec.ts","tests/e2e/paired-mobile-desktop-tab-composition.spec.ts","tests/e2e/paired-remote-terminal-serve-restart-binding.spec.ts","tests/e2e/ssh-orcad-"]' + run_changed_e2e: true + needs_build: true + artifact_suffix: -layout-owner-switch-on + layout_owner: main diff --git a/.github/workflows/macos-updater-tests.yml b/.github/workflows/macos-updater-tests.yml index cf7d149ece86..fca034a15610 100644 --- a/.github/workflows/macos-updater-tests.yml +++ b/.github/workflows/macos-updater-tests.yml @@ -8,6 +8,9 @@ on: - 'src/main/macos-update-running-instances*' - 'src/main/updater*' - 'src/main/updater/**' + - 'native/macos-shipit-scheduling/**' + - 'config/scripts/macos-shipit-scheduling*' + - 'config/electron-builder.config.cjs' - 'src/main/startup/main-process-quit*' - 'src/main/window/main-window-state-lifecycle*' - 'src/main/window/dashboard-popout-window*' @@ -35,11 +38,15 @@ jobs: - uses: ./.github/actions/install-node-dependencies with: native-runtime: node + cache-electron-package: 'true' + - name: Install the pinned Electron binary for native updater probes + run: node config/scripts/install-electron-package-binary.mjs - name: Exercise native application registry and update shutdown env: ORCA_BACKGROUND_LAUNCH: '1' run: >- pnpm exec vitest run --config config/vitest.config.ts + config/scripts/macos-shipit-scheduling.test.mjs src/main/macos-update-running-instances.test.ts src/main/macos-update-running-instances.integration.test.ts src/main/updater.mac-install.test.ts diff --git a/.github/workflows/node-server-tests.yml b/.github/workflows/node-server-tests.yml index 0c0aba24d6e9..452ea7446f9a 100644 --- a/.github/workflows/node-server-tests.yml +++ b/.github/workflows/node-server-tests.yml @@ -14,7 +14,7 @@ on: - 'pnpm-workspace.yaml' - 'tsconfig.json' - '.npmrc' - - '.pnpmfile.cjs' + - 'config/pnpmfile.cjs' - '.github/actions/install-node-dependencies/**' - '.github/actions/restore-pnpm-verification/**' - '.github/actions/prepare-headless-compiler/**' @@ -128,7 +128,7 @@ jobs: fi persistence: - needs: changes + needs: [changes, mobile_web_bundle] concurrency: group: node-server-persistence-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }} @@ -136,6 +136,7 @@ jobs: # Drafts re-trigger at readiness; inherited caller events must not depend on PR fields. if: >- ${{ !cancelled() && needs.changes.outputs.should_run != 'false' && + (needs.changes.outputs.qualification == 'false' || needs.mobile_web_bundle.result == 'success') && (github.event_name != 'pull_request' || github.event.pull_request.draft != true) }} strategy: fail-fast: false @@ -154,7 +155,8 @@ jobs: id: dependencies with: native-runtime: ${{ runner.os == 'Windows' && 'node' || 'none' }} - cache-pnpm-store: ${{ runner.os != 'Windows' }} + # Template qualification must not wait for large download-cache uploads. + cache-pnpm-store: ${{ !inputs.build_template && runner.os != 'Windows' }} cache-pnpm-store-lookup-only: 'true' # Design D7 upgrade and rollback: the last Bun orcad, built from a main commit that shipped # it, using the current Bun pin beside this checkout's Node slot; hand-over skips once PROTOCOL_VERSION @@ -191,7 +193,14 @@ jobs: if: runner.os == 'Windows' with: arches: ${{ (inputs.build_template && matrix.os == 'windows-2022') && 'x64 arm64' || (runner.arch == 'ARM64' && 'arm64' || 'x64') }} + - uses: actions/download-artifact@v8 + if: needs.changes.outputs.qualification != 'false' + with: + name: orcad-mobile-web-bundle + path: out/mobile-web + # Narrow test-only runs skip the shared producer but still need a page. - uses: ./.github/actions/build-mobile-web-bundle + if: needs.changes.outputs.qualification == 'false' - run: pnpm build:orcad env: ORCA_REUSE_PREPARED_WINDOWS_SECURITY: ${{ github.event_name == 'pull_request' && runner.os == 'Windows' && steps.dependencies.outcome == 'success' }} @@ -298,8 +307,7 @@ jobs: # A rerun attempt re-uploads under the same name, which v4 otherwise refuses. overwrite: true - # The container lanes take the page from here: it is platform-independent, and building it needs - # mobile/node_modules, which those images never install. + # Every qualification lane tests the same platform-independent page. mobile_web_bundle: needs: changes if: >- @@ -324,13 +332,13 @@ jobs: overwrite: true linux_glibc_floor: - needs: [changes, persistence, mobile_web_bundle] + needs: [changes, mobile_web_bundle] concurrency: group: node-server-linux_glibc_floor-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }} - # A failed smoke already blocks qualification; missing scope still selects every platform. + # Each Linux lane owns its inputs; the template still requires every lane to pass. if: >- - ${{ !cancelled() && needs.persistence.result == 'success' && + ${{ !cancelled() && needs.mobile_web_bundle.result == 'success' && needs.changes.outputs.should_run != 'false' && needs.changes.outputs.qualification != 'false' }} @@ -397,15 +405,15 @@ jobs: overwrite: true linux_glibc217_compat: - needs: [changes, persistence] + needs: changes concurrency: group: node-server-linux_glibc217_compat-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }} - # A failed smoke already blocks qualification; missing scope still selects every platform. if: >- - ${{ !cancelled() && needs.persistence.result == 'success' && + ${{ !cancelled() && needs.changes.outputs.should_run != 'false' && needs.changes.outputs.qualification != 'false' && + (github.event_name != 'pull_request' || github.event.pull_request.draft != true) && (github.event_name != 'pull_request' || inputs.build_template || inputs.ref != '' || needs.changes.outputs.native_compatibility != 'false') }} runs-on: ubuntu-22.04 @@ -454,13 +462,12 @@ jobs: overwrite: true linux_musl: - needs: [changes, persistence, mobile_web_bundle] + needs: [changes, mobile_web_bundle] concurrency: group: node-server-linux_musl-${{ matrix.os }}-${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'push' && !inputs.build_template && inputs.ref == '' }} - # A failed smoke already blocks qualification; missing scope still selects every platform. if: >- - ${{ !cancelled() && needs.persistence.result == 'success' && + ${{ !cancelled() && needs.mobile_web_bundle.result == 'success' && needs.changes.outputs.should_run != 'false' && needs.changes.outputs.qualification != 'false' }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 61a3e218a4d1..722d88082cf0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -65,6 +65,7 @@ jobs: /config/.bun-version /config/scripts/check-readme-local-links.mjs /config/scripts/pr-code-change-scope.mjs + /.oxfmtrc.json /config/scripts/pr-package-test-paths.mjs /config/scripts/pr-layout-oracle-route.mjs /config/scripts/pr-e2e-source-routing.mjs @@ -231,6 +232,7 @@ jobs: PREFLIGHT_PHASE_SELECTED: ${{ needs.code_paths.outputs.typecheck == 'true' }} PREFLIGHT_PRIOR_SUCCESS: ${{ job.status == 'success' }} ORCA_UNIT_SELECTION_MODE: ${{ vars.ORCA_UNIT_SELECTION_MODE || 'shadow' }} + ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }} # Five shards use fewer runner minutes; the repository variable can still request ten. ORCA_UNIT_FULL_SHARD_COUNT: ${{ vars.ORCA_UNIT_FULL_SHARD_COUNT || '5' }} run: | diff --git a/.github/workflows/release-cut.yml b/.github/workflows/release-cut.yml index b2edb687dd07..45b90b59995d 100644 --- a/.github/workflows/release-cut.yml +++ b/.github/workflows/release-cut.yml @@ -1139,6 +1139,7 @@ jobs: - orcad-template - release-preflight - release-javascript + - build-mac # Fork: mac-only. Windows/Linux builds are disabled (no SignPath on this fork). # macOS is built + signed by build-mac -> release-mac-build.yml. if: false @@ -1212,7 +1213,7 @@ jobs: run: | set -euo pipefail git fetch --no-tags --depth=1 origin "$WORKFLOW_SHA" - git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs + git checkout "$WORKFLOW_SHA" -- config/scripts/assert-github-release-is-draft.mjs config/scripts/run-release-mac-build-workflow.mjs - name: Restore composite actions from the workflow ref shell: bash @@ -1582,9 +1583,26 @@ jobs: path: signing-stage/** if-no-files-found: error + # Unsigned packaging overlaps macOS; only successful macOS builds may consume signing quota. + - name: Require macOS success before Windows signing + id: mac-signing-gate + if: matrix.platform == 'win' && github.run_attempt == 1 + shell: bash + run: node config/scripts/run-release-mac-build-workflow.mjs wait + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_MAC_BUILD_REF: ${{ github.ref_name }} + RELEASE_MAC_BUILD_RELEASE_RUN_ID: ${{ github.run_id }} + RELEASE_MAC_BUILD_RUN_ID: ${{ needs.build-mac.outputs.run_id }} + RELEASE_MAC_BUILD_WORKFLOW_SHA: ${{ needs.build-mac.outputs.workflow_sha }} + RELEASE_MAC_BUILD_RUN_ATTEMPT: ${{ needs.build-mac.outputs.run_attempt }} + RELEASE_MAC_BUILD_WAIT_DEADLINE_MS: ${{ needs.build-mac.outputs.wait_deadline_ms }} + RELEASE_MAC_BUILD_TAG: ${{ needs.cut.outputs.tag }} + RELEASE_MAC_BUILD_WORKFLOW: release-mac-build.yml + - name: Submit inner binaries signing request id: submit-inner-signing - if: matrix.platform == 'win' && github.run_attempt == 1 && steps.upload-unsigned-inner.outcome == 'success' + if: matrix.platform == 'win' && github.run_attempt == 1 && steps.mac-signing-gate.outcome == 'success' && steps.upload-unsigned-inner.outcome == 'success' uses: signpath/github-action-submit-signing-request@v2 with: api-token: ${{ secrets.SIGNPATH_API_TOKEN }} @@ -1820,7 +1838,7 @@ jobs: # so the release job waits while the signing request is approved in UI. - name: Submit Windows installer signing request id: submit-signing-request - if: matrix.platform == 'win' && github.run_attempt == 1 + if: matrix.platform == 'win' && github.run_attempt == 1 && steps.mac-signing-gate.outcome == 'success' && steps.upload-unsigned-windows-installer.outcome == 'success' uses: signpath/github-action-submit-signing-request@v2 with: api-token: ${{ secrets.SIGNPATH_API_TOKEN }} diff --git a/.github/workflows/release-mac-build.yml b/.github/workflows/release-mac-build.yml index 55c03b8cf818..f82c9ac66f71 100644 --- a/.github/workflows/release-mac-build.yml +++ b/.github/workflows/release-mac-build.yml @@ -13,6 +13,11 @@ on: description: release-cut workflow run that requested this build required: true type: string + qualification_only: + description: Sign and notarize both architectures without publishing a release + required: false + default: false + type: boolean javascript_source_sha: description: Commit of the shared JavaScript artifact; empty keeps the legacy build required: false @@ -25,12 +30,23 @@ permissions: contents: write concurrency: - group: release-mac-build-${{ inputs.tag }} + # Why: a qualification run that reuses a live tag must never queue behind or replace its release build. + group: release-mac-build-${{ inputs.tag }}${{ inputs.qualification_only && '-qualification' || '' }} cancel-in-progress: false jobs: + qualification-template: + if: inputs.qualification_only + permissions: + contents: read + uses: ./.github/workflows/node-server-tests.yml + with: + ref: ${{ github.sha }} + build_template: true + build-mac: - if: github.repository == 'zpyoung/orca' + needs: qualification-template + if: ${{ !cancelled() && (!inputs.qualification_only || needs.qualification-template.result == 'success') && github.repository == 'zpyoung/orca' }} # Fork: GitHub-hosted macOS runner (no Blacksmith on this fork). Signs + # notarizes with the fork's own Apple Developer ID secrets. runs-on: macos-15 @@ -45,12 +61,13 @@ jobs: actions: read contents: write env: + ORCA_BACKGROUND_LAUNCH: '1' NODE_OPTIONS: --max-old-space-size=4096 steps: - name: Checkout uses: actions/checkout@v6 with: - ref: refs/tags/${{ inputs.tag }} + ref: ${{ inputs.qualification_only && github.sha || format('refs/tags/{0}', inputs.tag) }} # Why: a relay-era ref (before #27215) needs the Windows relay addon this build no longer compiles. - name: Refuse a ref that still ships the SSH relay @@ -198,7 +215,7 @@ jobs: with: name: orcad-template path: out/orcad-template - run-id: ${{ inputs.release_run_id }} + run-id: ${{ inputs.qualification_only && github.run_id || inputs.release_run_id }} github-token: ${{ secrets.GITHUB_TOKEN }} - name: Gate runtime file-watcher process isolation @@ -209,6 +226,7 @@ jobs: ELECTRON_RUN_AS_NODE=1 pnpm exec electron config/scripts/runtime-file-watcher-fault-harness.mjs - name: Abort if the parent release-cut run was cancelled + if: ${{ !inputs.qualification_only }} # Why: this workflow is dispatched separately, so cancelling release-cut # does not stop mac `--publish always`. A cancelled parent left v1.4.206 # public with only a partial mac upload. @@ -234,10 +252,12 @@ jobs: timeout_minutes: 70 max_attempts: 2 retry_wait_seconds: 30 - command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish always -c.publish.releaseType=draft + # Fork: no notarization-overlap builder; it signs x64 and arm64 together and this channel ships arm64 only. + command: node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON config/scripts/pinned-node-downloads.mjs darwin-arm64 darwin-x64 && node config/scripts/ensure-native-runtime.mjs --runtime=electron && ORCA_MAC_RELEASE=1 pnpm exec electron-builder --config config/electron-builder.config.cjs --mac --publish "$ORCA_MAC_PUBLISH" -c.publish.releaseType=draft env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ORCA_REQUIRE_ORCAD_TEMPLATE: '1' + ORCA_MAC_PUBLISH: ${{ inputs.qualification_only && 'never' || 'always' }} CSC_LINK: ${{ secrets.MAC_CERTS }} CSC_KEY_PASSWORD: ${{ secrets.MAC_CERTS_PASSWORD }} APPLE_ID: ${{ secrets.APPLE_ID }} @@ -245,6 +265,7 @@ jobs: APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} - name: Verify release remains draft after artifact upload + if: ${{ !inputs.qualification_only }} # Why: re-draft immediately if electron-builder flipped the GitHub # release public, then fail. Checking without restoring leaves # /releases/latest serving a missing Windows exe. @@ -259,3 +280,16 @@ jobs: # point between pack and upload without splitting those steps. - name: Verify telemetry constants present in app.asar run: node config/scripts/verify-telemetry-constants.mjs + + - name: Upload qualification artifacts + if: inputs.qualification_only + uses: actions/upload-artifact@v7 + with: + name: mac-notarization-qualification + path: | + dist/*.dmg + dist/*.zip + dist/*.blockmap + dist/*.yml + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/ssh-hostile-hosts.yml b/.github/workflows/ssh-hostile-hosts.yml index 8b1f7f7688c8..d1a3a15eee65 100644 --- a/.github/workflows/ssh-hostile-hosts.yml +++ b/.github/workflows/ssh-hostile-hosts.yml @@ -1,9 +1,9 @@ name: SSH hostile hosts # Hostile-host matrix: the real client-side managed orcad deploy against container SSH targets -# (old glibc, musl, no libstdc++, noexec home, no egress) and against each macOS runner's own +# (old glibc, musl, no libstdc++, noexec home, no egress on either side) and against each macOS runner's own # loopback sshd, asserting the runtime each host resolves and whether orcad activates there. -# Heavy (three slot builds plus seven images), so it runs only when the deploy, the runtime store +# Heavy (three slot builds plus eight images), so it runs only when the deploy, the runtime store # or the slot build changes, and on demand. on: pull_request: @@ -27,7 +27,10 @@ on: - 'config/scripts/verify-packaged-orcad-template.cjs' - 'config/patches/node-pty*' # The deploy runs over this SSH transport. - - 'src/main/ssh/ssh-connection*' + - 'src/main/ssh/connection/**' + - 'src/main/ssh/ssh-reconnect-error-classification.ts' + - 'src/main/ssh/ssh-posix-command-wrapper.ts' + - 'src/shared/shell-argument-quoting.ts' - '!src/**/*.test.ts' - 'src/main/ssh/hostile-host-test-support/ssh-hostile-hosts.docker.test.ts' - '.github/workflows/ssh-hostile-hosts.yml' diff --git a/.github/workflows/ssh-windows-hosts.yml b/.github/workflows/ssh-windows-hosts.yml index 87c9f8378031..82fcb984f553 100644 --- a/.github/workflows/ssh-windows-hosts.yml +++ b/.github/workflows/ssh-windows-hosts.yml @@ -43,7 +43,10 @@ on: - 'src/shared/windows-breakaway-launch*.ts' - 'src/main/ipc/ssh/ssh-host-server-*.ts' # The deploy runs over this SSH transport. - - 'src/main/ssh/ssh-connection*' + - 'src/main/ssh/connection/**' + - 'src/main/ssh/ssh-reconnect-error-classification.ts' + - 'src/main/ssh/ssh-posix-command-wrapper.ts' + - 'src/shared/shell-argument-quoting.ts' - '!src/**/*.test.ts' - 'src/main/managed-server/host-state/orcad-windows-host-lane.test.ts' - 'tests/e2e/ssh-orcad-auto-convert.spec.ts' diff --git a/.github/workflows/terminal-perf.yml b/.github/workflows/terminal-perf.yml index 58d93671d31a..c95854d49ee8 100644 --- a/.github/workflows/terminal-perf.yml +++ b/.github/workflows/terminal-perf.yml @@ -88,7 +88,7 @@ jobs: const shared = process.env.RUNNER_KIND === 'github-hosted' && !process.env.JOB_CONTAINER && process.env.RUNNER_OS === 'Linux' && process.env.RUNNER_ARCH === 'X64' && manifest.engines?.node === '24' && typeof manifest.packageManager === 'string' && - manifest.packageManager.split('+')[0] === 'pnpm@12.8.1' && + manifest.packageManager.split('+')[0] === 'pnpm@12.14.0' && supportedPostinstalls.includes(manifest.scripts?.postinstall) && /^ native-runtime:/m.test(inputs) && /^ cache-pnpm-store-lookup-only:/m.test(inputs) && fs.existsSync('.github/actions/prepare-native-runtime/action.yml') && diff --git a/.github/workflows/unit-plan.yml b/.github/workflows/unit-plan.yml index b427908b26d1..acc6a8fc500a 100644 --- a/.github/workflows/unit-plan.yml +++ b/.github/workflows/unit-plan.yml @@ -39,6 +39,7 @@ jobs: id: plan env: ORCA_UNIT_SELECTION_MODE: ${{ inputs.selection_mode }} + ORCA_PULL_REQUEST_DRAFT: ${{ github.event.pull_request.draft }} run: node config/scripts/ci-unit-plan.mjs - uses: actions/upload-artifact@v7 continue-on-error: true diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 180bd6c6e156..6f705b57f8dd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -260,8 +260,8 @@ jobs: if: ${{ !cancelled() && steps.root_dependencies.outcome == 'success' }} working-directory: cloud run: | - npx --yes pnpm@10.34.6 --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts - npx --yes pnpm@10.34.6 --filter '@orca-cloud/relay^...' build + pnpm --filter '@orca-cloud/relay...' install --frozen-lockfile --ignore-scripts + pnpm --filter '@orca-cloud/relay^...' build - name: Test relay integration contracts if: ${{ !cancelled() && steps.relay_dependencies.outcome == 'success' }} diff --git a/.github/workflows/wsl-orcad-e2e.yml b/.github/workflows/wsl-orcad-e2e.yml index e33250d06e4c..5d06bc260252 100644 --- a/.github/workflows/wsl-orcad-e2e.yml +++ b/.github/workflows/wsl-orcad-e2e.yml @@ -3,24 +3,38 @@ name: WSL orcad E2E # WSL via orcad (step C10) on real WSL2 distros: a GitHub-hosted windows-2025 runner imports a # pinned Ubuntu 24.04 rootfs twice (systemd off and on) and runs # src/main/wsl/wsl-orcad-real-distro-lane.test.ts against both: deploy and launch, the wsl.exe -# stdio bridge, runtime RPC, liveness recovery, holder survival and teardown. Not a required check. +# stdio bridge, runtime RPC, liveness recovery, holder survival, structured chat and teardown. Not a required check. # Hosted WSL2 was proven in #27783: WSL 2.7.14, import 13-18 s, cold boot 8-14 s. on: pull_request: types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] paths: - 'src/main/wsl/**' - - '!src/main/wsl/**/*.test.ts' - - 'src/main/wsl/wsl-orcad-real-distro-lane.test.ts' - # This lane executes only the real distro test above. + - 'src/main/runtime/**' + - 'src/main/native-chat/**' + - 'src/main/claude/**' + - 'src/main/claude-accounts/**' + - 'src/main/codex/**' + - 'src/main/acp/**' + - 'src/main/pi/**' + - 'src/main/orcad/**' + - 'src/main/server/**' + - 'src/main/persistence/**' + - 'mobile/src/**' + - 'src/main/ipc/*wsl*' + - 'src/shared/**' + - 'src/renderer/src/components/native-chat/**' + - 'src/renderer/src/lib/*agent-session*' + - 'config/scripts/*orcad*' + - 'package.json' + - 'pnpm-lock.yaml' - 'src/main/ssh/orcad-*' - 'src/main/managed-server/**' - - '!src/main/ssh/orcad-*.test.ts' - - '!src/main/managed-server/**/*.test.ts' - - 'src/main/managed-server/install/remote-install-transport.ts' - # orcad-entry runs this on every boot of the orcad the lane launches in the distro. - - 'src/main/orcad/orcad-wsl-*' - - '!src/main/orcad/orcad-wsl-*.test.ts' + - 'src/packages/process-host/**' + # Unit tests never run in this lane; only the real distro test below does. + - '!**/*.test.ts' + - '!**/*.test.tsx' + - 'src/main/wsl/wsl-orcad-real-distro-lane.test.ts' - '.github/workflows/wsl-orcad-e2e.yml' workflow_dispatch: @@ -175,6 +189,7 @@ jobs: # A non-root default user, as on a real install; git for the lane's repo. $setup = "set -e; id orca >/dev/null 2>&1 || useradd -m -s /bin/bash orca; " + "printf '[boot]\nsystemd=$($d.Systemd)\n[user]\ndefault=orca\n' > /etc/wsl.conf; " + + "printf 'export ORCA_BACKGROUND_LAUNCH=1\nexport USERPROFILE=`$HOME\nexport APPDATA=`$HOME/.config\nexport LOCALAPPDATA=`$HOME/.local/share\nexport XDG_CONFIG_HOME=`$HOME/.config\nexport XDG_DATA_HOME=`$HOME/.local/share\nexport XDG_CACHE_HOME=`$HOME/.cache\nexport XDG_STATE_HOME=`$HOME/.local/state\nexport CLAUDE_CONFIG_DIR=`$HOME/.claude\nexport CODEX_HOME=`$HOME/.codex\n' >> /home/orca/.profile; " + "command -v git >/dev/null || (apt-get update -qq && apt-get install -y -qq git)" wsl.exe -d $d.Name -u root --exec sh -c $setup if ($LASTEXITCODE -ne 0) { throw "provisioning $($d.Name) failed: $LASTEXITCODE" } @@ -201,6 +216,20 @@ jobs: with: path: ${{ runner.temp }}/wsl-rootfs key: wsl-orcad-rootfs-${{ env.ROOTFS_SHA256 }} + - name: Isolate the desktop test home + run: | + New-Item -ItemType Directory -Force $env:ORCA_WSL_ORCAD_LANE_RECEIPTS | Out-Null + $files = @('.claude/settings.json', '.claude.json', '.codex/hooks.json', '.codex/config.toml') | ForEach-Object { Join-Path $env:USERPROFILE $_ } + $hashes = @($files | ForEach-Object { + @{ Path = $_; Hash = $(if (Test-Path $_) { (Get-FileHash $_ -Algorithm SHA256).Hash } else { 'absent' }) } + }) + $hashes | ConvertTo-Json | Set-Content (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'real-home-hashes.json') + $homeDir = Join-Path $env:RUNNER_TEMP 'wsl-orcad-desktop-home' + foreach ($name in @('HOME', 'USERPROFILE', 'APPDATA', 'LOCALAPPDATA', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_CACHE_HOME', 'XDG_STATE_HOME', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME')) { + $path = Join-Path $homeDir $name + New-Item -ItemType Directory -Force $path | Out-Null + "$name=$path" >> $env:GITHUB_ENV + } - name: 'Phase 1: deploy, RPC, kill orcad, terminate distro' timeout-minutes: 20 env: @@ -212,8 +241,8 @@ jobs: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/wsl/wsl-orcad-real-distro-lane.test.ts --reporter=verbose 2>&1 | Tee-Object (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'launch.log') if ($global:LASTEXITCODE -ne 0) { throw "launch phase failed: $global:LASTEXITCODE" } # A new process: the holders must have kept both distros Running since phase 1 exited. - - name: 'Phase 2: holders survive, reconnect, teardown' - timeout-minutes: 10 + - name: 'Phase 2: holders survive, chat, reconnect, teardown' + timeout-minutes: 15 env: ORCA_RUN_WSL_ORCAD_LANE: '1' ORCA_WSL_ORCAD_LANE_PHASE: survive @@ -221,6 +250,16 @@ jobs: run: | node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/wsl/wsl-orcad-real-distro-lane.test.ts --reporter=verbose 2>&1 | Tee-Object (Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'survive.log') if ($global:LASTEXITCODE -ne 0) { throw "survive phase failed: $global:LASTEXITCODE" } + - name: Real desktop agent configuration is unchanged + if: always() + run: | + $baseline = Join-Path $env:ORCA_WSL_ORCAD_LANE_RECEIPTS 'real-home-hashes.json' + if (Test-Path $baseline) { + foreach ($file in (Get-Content $baseline -Raw | ConvertFrom-Json)) { + $hash = if (Test-Path $file.Path) { (Get-FileHash $file.Path -Algorithm SHA256).Hash } else { 'absent' } + if ($hash -ne $file.Hash) { throw "Real agent configuration changed: $($file.Path)" } + } + } # Belt and braces for the runner: a missing VM-wide interop entry is restored, then fails. - name: WSLInterop is intact if: ${{ !cancelled() && steps.import.outcome == 'success' }} diff --git a/README.md b/README.md index f0623fff44d4..dc6b13791676 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Ghostty-class terminals with WebGL rendering, infinite splits, and scrollback th - Terminal splits + Terminal splits @@ -96,7 +96,7 @@ Browse PRs, issues, and project boards in-app — open a worktree from any task - GitHub and Linear task workflows in Orca + GitHub and Linear task workflows in Orca @@ -110,7 +110,7 @@ Run agents on a beefy remote box with full file editing, git, and terminals — - Remote worktrees over SSH + Remote worktrees over SSH @@ -152,7 +152,7 @@ Agents drive Orca too — script every workflow with `orca worktree create`, `sn - Script Orca from the CLI + Script Orca from the CLI @@ -242,9 +242,9 @@ Pair with your desktop app to monitor and steer your agents from your phone. - **Discord:** Join the community on **[Discord](https://discord.gg/fzjDKHxv8Q)**. - **Twitter / X:** Follow **[@orca_build](https://x.com/orca_build)** for updates and announcements. -- **WeChat:** Group 11 is full. Scan to join the Orca community WeChat group 12. +- **WeChat:** Group 12 is full. Scan to join the Orca community WeChat group 13. - WeChat group 12 QR code for the Orca community + WeChat group 13 QR code for the Orca community - **Feedback & Ideas:** We ship fast. Missing something? [Request a new feature](https://github.com/stablyai/orca/issues). - **Privacy:** See the [privacy & telemetry docs](https://www.onorca.dev/docs/telemetry) for what anonymous usage data Orca collects and how to opt out. diff --git a/cloud/.npmrc b/cloud/.npmrc deleted file mode 100644 index e05d6820ed56..000000000000 --- a/cloud/.npmrc +++ /dev/null @@ -1,3 +0,0 @@ -engine-strict=false -package-manager-strict=true - diff --git a/cloud/apps/push/src/durable-push-worker.test.ts b/cloud/apps/push/src/durable-push-worker.test.ts index 7cbd6074b360..615689a320b3 100644 --- a/cloud/apps/push/src/durable-push-worker.test.ts +++ b/cloud/apps/push/src/durable-push-worker.test.ts @@ -1,3 +1,4 @@ +import { setImmediate } from 'node:timers/promises' import { afterEach, expect, it, vi } from 'vitest' import type { PushNotification } from '@orca-cloud/push-contract' import { DurablePushStore } from './durable-push-store.js' @@ -7,6 +8,7 @@ import { PushDeviceRegistryStore } from './device-registry-store.js' import { openInMemoryPushDatabase } from './push-database.js' import type { PushDelivery } from './push-delivery-message.js' import type { PushProviderOutcome } from './push-provider-outcome.js' +import { WORKER_DRAINS } from './push-worker-concurrency.js' const cleanups: (() => Promise)[] = [] afterEach(async () => { @@ -119,8 +121,9 @@ it('keeps untrackable bells and per-phone deliveries individually replaceable', ).toMatchObject({ orca: { notificationId: 'note-2', notificationSeq: 2 } }) }) -it('persists provider retry delay and resumes it through a new worker', async () => { +it('preserves provider retry delay when waking a restarted worker', async () => { const h = await fixture() + vi.useFakeTimers() h.send.mockResolvedValueOnce({ status: 'error', reason: 'busy', @@ -132,10 +135,13 @@ it('persists provider retry delay and resumes it through a new worker', async () expect(h.send).toHaveBeenCalledOnce() await h.worker.stop() const restarted = new DurablePushWorker(h.store, h.dispatcher, { now: h.now, onRetry: h.onRetry }) + restarted.start() h.advance(9999) + restarted.wake() await restarted.runDue() expect(h.send).toHaveBeenCalledOnce() h.advance(1) + restarted.wake() await restarted.runDue() expect(h.send).toHaveBeenCalledTimes(2) expect(h.send.mock.calls.map(([delivery]) => delivery.expiresAt)).toEqual([1_300_000, 1_300_000]) @@ -175,6 +181,7 @@ it('rechecks the device before a persisted retry and does not send after unregis it('joins active work on shutdown and leaves unclaimed work for the next instance', async () => { const h = await fixture() + vi.useFakeTimers() let finish!: (outcome: PushProviderOutcome) => void let started!: () => void const entered = new Promise((resolve) => { @@ -187,9 +194,12 @@ it('joins active work on shutdown and leaves unclaimed work for the next instanc }) }) await h.accept(note(1)) + h.worker.start() + h.worker.wake() const pending = h.worker.runDue() await entered await h.accept(note(2)) + h.worker.wake() let stopped = false const stopping = h.worker.stop().then(() => { stopped = true @@ -200,7 +210,10 @@ it('joins active work on shutdown and leaves unclaimed work for the next instanc await Promise.all([pending, stopping]) expect(stopped).toBe(true) expect(h.send).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) const resumed = new DurablePushWorker(h.store, h.dispatcher, { now: h.now }) + resumed.start() + await vi.advanceTimersByTimeAsync(1000) await resumed.runDue() expect(h.send).toHaveBeenCalledTimes(2) await resumed.stop() @@ -219,3 +232,129 @@ it('runs due work on its timer and releases the timer on stop', async () => { await h.worker.stop() expect(vi.getTimerCount()).toBe(0) }) + +it('wakes only while started and keeps the durability poll', async () => { + const h = await fixture() + vi.useFakeTimers() + const claim = vi.spyOn(h.store, 'claim') + await h.accept(note(1)) + h.worker.wake() + expect(claim).not.toHaveBeenCalled() + h.worker.start() + h.worker.wake() + expect(claim).toHaveBeenCalled() + await h.worker.runDue() + expect(h.send).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(1) + await h.worker.stop() + const calls = claim.mock.calls.length + await h.accept(note(2)) + h.worker.wake() + await vi.advanceTimersByTimeAsync(2000) + expect(claim).toHaveBeenCalledTimes(calls) + expect(await h.store.pendingCount(h.registrationId)).toBe(1) + expect(vi.getTimerCount()).toBe(0) +}) + +it('coalesces wakes arriving after every active drain found an empty queue', async () => { + const h = await fixture() + vi.useFakeTimers() + const entered = Promise.withResolvers() + const release = Promise.withResolvers() + const claim = h.store.claim.bind(h.store) + let empty = 0 + let holding = true + vi.spyOn(h.store, 'claim').mockImplementation(async () => { + const delivery = await claim() + if (!delivery && holding) { + if (++empty === WORKER_DRAINS) entered.resolve() + await release.promise + } + return delivery + }) + h.worker.start() + const startedAt = Date.now() + const pending = h.worker.runDue() + try { + await entered.promise + await h.accept(note(1)) + for (let count = 0; count < 10; count++) h.worker.wake() + expect(h.store.claim).toHaveBeenCalledTimes(WORKER_DRAINS) + holding = false + release.resolve() + await pending + await vi.waitFor(() => expect(h.send).toHaveBeenCalledOnce()) + await vi.waitFor(() => expect(h.store.claim).toHaveBeenCalledTimes(WORKER_DRAINS + 2)) + expect(Date.now() - startedAt).toBeLessThan(1000) + expect(h.send.mock.calls[0]?.[0].orca.notificationSeq).toBe(1) + } finally { + holding = false + release.resolve() + } +}) + +it('bounds claims by sends plus drains for a burst and a trickle of wakes', async () => { + const h = await fixture() + const registrations: string[] = [] + for (let index = 0; index < 50; index++) { + const device = await h.devices.upsert({ + hostFingerprint: `host-${index}`, + deviceId: `phone-${index}`, + platform: 'android', + token: `token-${index}` + }) + if (!device.ok) throw new Error('registration failed') + registrations.push(device.registrationId) + } + const claim = vi.spyOn(h.store, 'claim') + const sendOne = async (index: number) => { + const status = await h.store.accept( + `host-${index % 50}`, + registrations[index % 50]!, + note(index + 1) + ) + expect(status).toBe('queued') + h.worker.wake() + } + // Fake timers hold the poll, so only wakes claim. + vi.useFakeTimers() + h.worker.start() + const settle = async () => { + for (let ticks = 0; ticks < 1000; ticks++) await setImmediate() + } + await Promise.all(Array.from({ length: 1000 }, (_, index) => sendOne(index))) + await settle() + // Each drain stops after 25 sends, so the poll finishes a large burst. + let polls = 0 + while (h.send.mock.calls.length < 1000 && polls < 10) { + await vi.advanceTimersByTimeAsync(1000) + await settle() + polls++ + } + expect(h.send).toHaveBeenCalledTimes(1000) + expect(claim.mock.calls.length).toBeLessThanOrEqual(1000 + WORKER_DRAINS * (polls + 2)) + claim.mockClear() + // Before one-drain wakes, each idle send started every drain: ~13 claims per send. + for (let index = 1000; index < 1020; index++) { + await sendOne(index) + await settle() + } + expect(h.send).toHaveBeenCalledTimes(1020) + expect(claim).toHaveBeenCalledTimes(40) +}, 60_000) + +it('recovers from a failed automatic wake without leaving another timer', async () => { + const h = await fixture() + vi.useFakeTimers() + vi.spyOn(h.store, 'claim').mockRejectedValueOnce(new Error('database unavailable')) + h.worker.start() + h.worker.wake() + await vi.waitFor(() => + expect(console.warn).toHaveBeenCalledWith(JSON.stringify({ event: 'orca_push_worker_failed' })) + ) + await h.accept(note(1)) + h.worker.wake() + await vi.waitFor(() => expect(h.send).toHaveBeenCalledOnce()) + await h.worker.stop() + expect(vi.getTimerCount()).toBe(0) +}) diff --git a/cloud/apps/push/src/durable-push-worker.ts b/cloud/apps/push/src/durable-push-worker.ts index f5bcc20fc2c4..77eacb248843 100644 --- a/cloud/apps/push/src/durable-push-worker.ts +++ b/cloud/apps/push/src/durable-push-worker.ts @@ -5,8 +5,9 @@ import { WORKER_DRAINS } from './push-worker-concurrency.js' export class DurablePushWorker { private timer?: NodeJS.Timeout - private running: Promise | null = null + private readonly drains = new Set>() private stopped = false + private wakePending = false constructor( private readonly store: DurablePushStore, private readonly dispatcher: PushDispatcher, @@ -17,32 +18,45 @@ export class DurablePushWorker { if (this.timer) return this.stopped = false this.timer = setInterval(() => { - void this.runDue().catch(() => { - console.warn(JSON.stringify({ event: 'orca_push_worker_failed' })) - }) + void this.runDue().catch(warnWorkerFailed) }, 1000) this.timer.unref() } - async runDue(): Promise { - if (this.running) { - await this.running + // One drain per wake keeps claims O(sends + drains); the poll restores full concurrency. + wake(): void { + if (!this.timer || this.stopped) return + if (this.drains.size >= WORKER_DRAINS) { + this.wakePending = true return } + this.startDrain().catch(warnWorkerFailed) + } + + async runDue(): Promise { if (this.stopped) return - const drains = Array.from({ length: WORKER_DRAINS }, () => this.drain()) - const pending = Promise.allSettled(drains).then( - (results) => { - const failure = results.find((result) => result.status === 'rejected') - if (failure?.status === 'rejected') throw failure.reason - } + const started = Array.from({ length: WORKER_DRAINS - this.drains.size }, () => + this.startDrain() ) - this.running = pending - try { - await pending - } finally { - this.running = null - } + const active = Promise.allSettled(this.drains) + const failure = (await Promise.allSettled(started)).find( + (result) => result.status === 'rejected' + ) + await active + if (failure?.status === 'rejected') throw failure.reason + } + + private startDrain(): Promise { + const drain: Promise = this.drain().finally(() => { + this.drains.delete(drain) + // A committed enqueue can arrive after every drain has found an empty queue. + if (this.wakePending) { + this.wakePending = false + this.wake() + } + }) + this.drains.add(drain) + return drain } private async drain(): Promise { @@ -84,8 +98,13 @@ export class DurablePushWorker { async stop(): Promise { this.stopped = true + this.wakePending = false if (this.timer) clearInterval(this.timer) this.timer = undefined - await this.running + await Promise.allSettled(this.drains) } } + +function warnWorkerFailed(): void { + console.warn(JSON.stringify({ event: 'orca_push_worker_failed' })) +} diff --git a/cloud/apps/push/src/push-server-wake.test.ts b/cloud/apps/push/src/push-server-wake.test.ts new file mode 100644 index 000000000000..156e8936ce31 --- /dev/null +++ b/cloud/apps/push/src/push-server-wake.test.ts @@ -0,0 +1,122 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { startPushBackground } from './push-background.js' +import { createPushHostKeypair } from './host-challenge-answering.test-fixture.js' +import { createPushServerHarness, notification } from './push-server-harness.test-fixture.js' + +const cleanups: (() => Promise)[] = [] +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup() + vi.useRealTimers() + vi.restoreAllMocks() +}) + +async function fixture() { + const h = await createPushServerHarness() + const token = await h.signIn(createPushHostKeypair(31)) + const registrationId = await h.registerAndroid(token) + let stop = async () => {} + cleanups.push(async () => { + await stop() + await h.close() + }) + vi.useFakeTimers() + return { + ...h, + registrationId, + start: () => { + stop = startPushBackground({ mode: 'active' }, h.server) + }, + send: (event = notification(), ids = [registrationId]) => + h.post('/v1/send', { v: 1, registrationIds: ids, notification: event }, token), + invalid: () => h.post('/v1/send', { v: 1 }, token) + } +} + +it('delivers a committed send before the next poll without waiting in the HTTP handler', async () => { + const h = await fixture() + h.start() + const startedAt = Date.now() + const claim = h.server.deliveryStore.claim.bind(h.server.deliveryStore) + const release = Promise.withResolvers() + vi.spyOn(h.server.deliveryStore, 'claim').mockImplementation(async () => { + await release.promise + return claim() + }) + try { + const response = await h.send() + expect(await response.json()).toEqual({ + results: [{ registrationId: h.registrationId, status: 'queued' }] + }) + expect(h.fcmRequests).toHaveLength(0) + release.resolve() + await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1)) + expect(Date.now() - startedAt).toBeLessThan(1000) + expect(JSON.parse(h.fcmRequests[0]!.body).message.data.notificationSeq).toBe('1') + } finally { + release.resolve() + } +}) + +it('keeps concurrent accepted retries idempotent while automatic delivery runs', async () => { + const h = await fixture() + h.start() + await Promise.all(Array.from({ length: 10 }, () => h.send())) + await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1)) + await h.send() + await h.server.worker.runDue() + expect(h.fcmRequests).toHaveLength(1) + expect(await h.server.deliveryStore.pendingCount(h.registrationId)).toBe(0) +}) + +it('does not wake for invalid, unowned or dead registrations', async () => { + const h = await fixture() + h.start() + const wake = vi.spyOn(h.server.worker, 'wake') + expect((await h.invalid()).status).toBe(400) + expect(await (await h.send(notification(), ['unknown'])).json()).toEqual({ + results: [{ registrationId: 'unknown', status: 'error' }] + }) + const device = await h.server.devices.findById(h.registrationId) + if (!device) throw new Error('registration missing') + await h.server.devices.markDead(device) + expect(await (await h.send()).json()).toEqual({ + results: [{ registrationId: h.registrationId, status: 'dead' }] + }) + expect(wake).not.toHaveBeenCalled() + expect(h.fcmRequests).toHaveLength(0) +}) + +it.each(['error', 'rate_limited'] as const)('does not wake a refused %s send', async (status) => { + const h = await fixture() + h.start() + const wake = vi.spyOn(h.server.worker, 'wake') + vi.spyOn(h.server.deliveryStore, 'accept').mockResolvedValue(status) + expect(await (await h.send()).json()).toEqual({ + results: [{ registrationId: h.registrationId, status }] + }) + expect(wake).not.toHaveBeenCalled() +}) + +it.each(['queued', 'sent'] as const)( + 'preserves dismissal of an already %s alert', + async (phase) => { + const h = await fixture() + if (phase === 'sent') h.start() + await h.send() + if (phase === 'sent') await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(1)) + await h.send(notification({ kind: 'dismiss', notificationSeq: 2, sound: false })) + if (phase === 'queued') { + h.start() + await vi.advanceTimersByTimeAsync(1000) + } + await vi.waitFor(() => expect(h.fcmRequests).toHaveLength(phase === 'sent' ? 2 : 1)) + const last = h.fcmRequests.at(-1) + if (!last) throw new Error('dismissal missing') + const message = JSON.parse(last.body).message + expect(message.data.kind).toBe('dismiss') + expect(message.data.notificationId).toBe('note-1') + expect(message.data.notificationSeq).toBe('2') + expect(message.data.title).toBeUndefined() + expect(message.data.sound).toBeUndefined() + } +) diff --git a/cloud/apps/push/src/push-server.ts b/cloud/apps/push/src/push-server.ts index 0aee3765570a..17c72b14b160 100644 --- a/cloud/apps/push/src/push-server.ts +++ b/cloud/apps/push/src/push-server.ts @@ -279,6 +279,7 @@ export function createPushServer( observability.record('send_queued') results.push({ registrationId, status: 'queued' }) } + if (results.some((result) => result.status === 'queued')) worker.wake() return context.json({ results }) }) diff --git a/cloud/apps/relay-ops/src/environment-config.test.ts b/cloud/apps/relay-ops/src/environment-config.test.ts index 321167d7cc8a..f3522a300cd3 100644 --- a/cloud/apps/relay-ops/src/environment-config.test.ts +++ b/cloud/apps/relay-ops/src/environment-config.test.ts @@ -64,6 +64,7 @@ function terraformCells(environment: 'production' | 'staging'): Array<{ machineType: string capacityRequests: number databasePoolMax: number + connectionHardCap: number | null }> { const tfvars = readFileSync( fileURLToPath(new URL(`../../../infra/terraform/environments/${environment}.tfvars`, import.meta.url)), @@ -77,7 +78,10 @@ function terraformCells(environment: 'production' | 'staging'): Array<{ zone: /zone\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', machineType: /machine_type\s*=\s*"([^"]+)"/.exec(match[2] ?? '')?.[1] ?? '', capacityRequests: Number(/capacity_requests\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1]), - databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10) + databasePoolMax: Number(/database_pool_max\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] ?? 10), + connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))( + /connection_hard_cap\s*=\s*(\d+)/.exec(match[2] ?? '')?.[1] + ) })) .sort((left, right) => cellOrdinal(left.cellId) - cellOrdinal(right.cellId)) } @@ -97,7 +101,8 @@ describe('relay operations environment config', () => { zone: cell.zone, machineType: cell.machineType, capacityRequests: cell.capacityRequests, - databasePoolMax: cell.databasePoolMax + databasePoolMax: cell.databasePoolMax, + connectionHardCap: cell.connectionHardCap })) ).toEqual(expected) } @@ -122,17 +127,20 @@ describe('relay operations environment config', () => { { cellId: 'production-gce-c27', hostname: 'c27', origin: 'https://c27.relay.onorca.dev', region: 'asia-east2', zone: 'asia-east2-a', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, + configuredAdmission: false }, { cellId: 'production-gce-c28', hostname: 'c28', origin: 'https://c28.relay.onorca.dev', region: 'asia-east2', zone: 'asia-east2-b', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, + configuredAdmission: false }, { cellId: 'production-gce-c29', hostname: 'c29', origin: 'https://c29.relay.onorca.dev', region: 'asia-east2', zone: 'asia-east2-c', machineType: 'e2-standard-4', - capacityRequests: 6000, databasePoolMax: 10, configuredAdmission: false + capacityRequests: 6000, databasePoolMax: 10, connectionHardCap: null, + configuredAdmission: false } ]) diff --git a/cloud/apps/relay-ops/src/environment-config.ts b/cloud/apps/relay-ops/src/environment-config.ts index da3de7765edf..efd890367c8a 100644 --- a/cloud/apps/relay-ops/src/environment-config.ts +++ b/cloud/apps/relay-ops/src/environment-config.ts @@ -14,6 +14,8 @@ export type RelayOpsCellConfig = { machineType: RelayOpsMachineType capacityRequests: number databasePoolMax: number + // The cell's rendered connectionHardCap; null when Terraform leaves it unset. + connectionHardCap: number | null configuredAdmission: boolean } @@ -74,6 +76,9 @@ export function relayOpsCellsFromTerraform(input: { required(body, /\bcapacity_requests\s*=\s*(\d+)/, `${cellId} capacity`) ), databasePoolMax: Number(/\bdatabase_pool_max\s*=\s*(\d+)/.exec(body)?.[1] ?? 10), + connectionHardCap: ((raw) => (raw === undefined ? null : Number(raw)))( + /\bconnection_hard_cap\s*=\s*(\d+)/.exec(body)?.[1] + ), configuredAdmission: configuredAdmission === undefined || configuredAdmission === 'true' } }) diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts index 32319df7f759..704f55717fa2 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.test.ts @@ -543,6 +543,92 @@ describe('incident monitor sources', () => { expect(waits).toEqual([]) }) + // Why: on 2026-10-19 cell-status refused reserve production-gce-c7 with a 404 on every sample. + it('marks a reserve cell unprobed and skips its refused migration reads', async () => { + const gcloud: GcloudClient = { + accessToken: async () => 'unused', + identityToken: async () => 'unused' + } + const reserveCellId = 'production-gce-c7' + const selector = { + generation: 1, + membership: { + existingOnly: [], + migrationOnly: ['production-gce-c2'], + general: productionCells.filter((cellId) => cellId !== 'production-gce-c2') + } + } + const waits: number[] = [] + const fetchImpl: typeof fetch = async (input, init) => { + const path = new URL(String(input)).pathname + const body = JSON.parse(String(init?.body)) as Record + if (path === '/v1/admin/admission-selector/status') return Response.json({ selector }) + if (path === '/v1/admin/cell-status') { + if (body.cellId === reserveCellId) { + return Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 }) + } + return Response.json({ + status: { + enabled: true, + connectionCapacity: { hardCap: 600 }, + runtime: { lastHeartbeatAt: now - 1_000, heartbeatFresh: true } + } + }) + } + expect(path).toBe('/v1/admin/evacuation-status') + expect(body.sourceCellId).not.toBe(reserveCellId) + return Response.json({ + blocked: 0, + blockedExpiredUnregistered: 0, + registeredTargetInactive: 0 + }) + } + const result = await directorSignals('production', selector, gcloud, now, fetchImpl, async (ms) => { + waits.push(ms) + }) + expect(waits).toEqual([]) + expect(result.cells.find((cell) => cell.cellId === reserveCellId)).toMatchObject({ + reserve: true + }) + expect(result.cells.find((cell) => cell.cellId === 'production-gce-c1')).not.toHaveProperty( + 'reserve' + ) + const signals = result.source.signals + expect(signals[`cell.${reserveCellId}.admission_state`]).toMatchObject({ value: 2 }) + // Why: 10-19 run 39225932592 froze reserve c7 at 731 connections against the 500 default. + const configuredCap = RELAY_OPS_ENVIRONMENTS.production.cells.find( + (cell) => cell.cellId === reserveCellId + )?.connectionHardCap + expect(configuredCap).toBe(1_000) + expect(signals[`cell.${reserveCellId}.connection_hard_cap`]).toMatchObject({ + value: configuredCap + }) + for (const suffix of ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked']) { + expect(signals[`cell.${reserveCellId}.${suffix}`]).toBeUndefined() + } + expect(signals['cell.production-gce-c1.heartbeat_age_ms']).toMatchObject({ value: 1_000 }) + }) + + it('still fails a cell-status 404 that is not the reserve refusal', async () => { + const gcloud: GcloudClient = { + accessToken: async () => 'unused', + identityToken: async () => 'unused' + } + const selector = { + generation: 1, + membership: { existingOnly: [], migrationOnly: [], general: productionCells } + } + const collect = (error: string) => + directorSignals('production', selector, gcloud, now, async (input) => { + const path = new URL(String(input)).pathname + if (path === '/v1/admin/admission-selector/status') return Response.json({ selector }) + return Response.json({ error }, { status: 404 }) + }, async () => {}) + for (const error of ['cell_not_found', 'cell_in_reserve_mode_soon', '']) { + await expect(collect(error)).rejects.toThrow('Relay admin telemetry returned 404') + } + }) + // A 404 the director means (wrong role) must not be retried either. it('does not retry a director-only 404', async () => { const gcloud: GcloudClient = { diff --git a/cloud/apps/relay-ops/src/incident-monitor-sources.ts b/cloud/apps/relay-ops/src/incident-monitor-sources.ts index 9e041940af7f..f69d57b11bdc 100644 --- a/cloud/apps/relay-ops/src/incident-monitor-sources.ts +++ b/cloud/apps/relay-ops/src/incident-monitor-sources.ts @@ -452,10 +452,7 @@ function transientFetchFailure(error: unknown): boolean { return name === 'AbortError' || name === 'TimeoutError' } -function retryableAdminStatus(status: number, body: string): boolean { - if (ADMIN_RETRYABLE_STATUSES.has(status)) return true - // 401/403/400/413 and a 404 for 'director_only' are decisions, not weather. - if (status !== 404) return false +function adminErrorMessage(body: string): string | null { const parsed = AdminErrorBodySchema.safeParse( ((): unknown => { try { @@ -465,12 +462,27 @@ function retryableAdminStatus(status: number, body: string): boolean { } })() ) + return parsed.success ? parsed.data.error : null +} + +function retryableAdminStatus(status: number, body: string): boolean { + if (ADMIN_RETRYABLE_STATUSES.has(status)) return true + // 401/403/400/413 and a 404 for 'director_only' are decisions, not weather. + if (status !== 404) return false + const message = adminErrorMessage(body) return ( - parsed.success && - ADMIN_TRANSIENT_ERROR_MESSAGES.some((message) => parsed.data.error.includes(message)) + message !== null && + ADMIN_TRANSIENT_ERROR_MESSAGES.some((transient) => message.includes(transient)) ) } +// Step 5: cell-status refuses a reserve cell with exactly this 404 body; any other 404 stays a failure. +class AdminReserveCellRefusal extends Error { + constructor(status: number) { + super(`Relay admin telemetry returned ${status}`) + } +} + async function adminPost( fetchImpl: typeof fetch, origin: string, @@ -502,6 +514,9 @@ async function adminPost( // Status only: the body carries a director error message, which must not // reach the log the way an identity would. const errorBody = await response.text().catch(() => '') + if (response.status === 404 && adminErrorMessage(errorBody) === 'cell_in_reserve_mode') { + throw new AdminReserveCellRefusal(response.status) + } if (lastAttempt || !retryableAdminStatus(response.status, errorBody)) { throw new Error(`Relay admin telemetry returned ${response.status}`) } @@ -545,23 +560,32 @@ export async function directorSignals( } const statuses: Array<{ cell: RelayOpsEnvironment['cells'][number] - status: z.infer['status'] + status: ({ reserve: false } & z.infer['status']) | { reserve: true } }> = [] for (const cell of environment.cells) { + let response: unknown + try { + response = await adminPost( + fetchImpl, + environment.directorOrigin, + token, + '/v1/admin/cell-status', + { v: 1, cellId: cell.cellId }, + wait + ) + } catch (error) { + if (!(error instanceof AdminReserveCellRefusal)) throw error + statuses.push({ cell, status: { reserve: true } }) + continue + } statuses.push({ cell, - status: CellStatusSchema.parse( - await adminPost( - fetchImpl, - environment.directorOrigin, - token, - '/v1/admin/cell-status', - { v: 1, cellId: cell.cellId }, - wait - ) - ).status + status: { reserve: false, ...CellStatusSchema.parse(response).status } }) } + const reserveCellIds = new Set( + statuses.filter(({ status }) => status.reserve).map(({ cell }) => cell.cellId) + ) const migrationEntries: Array<{ sourceCellId: string migration: z.infer @@ -570,6 +594,8 @@ export async function directorSignals( for (const source of environment.cells) { for (const target of environment.cells) { if (source.cellId === target.cellId || !migrationTargets.has(target.cellId)) continue + // The director refuses migration reads, and every migration, on a reserve cell. + if (reserveCellIds.has(source.cellId)) continue migrationEntries.push({ sourceCellId: source.cellId, migration: MigrationStatusSchema.parse( @@ -601,6 +627,24 @@ export async function directorSignals( const signals: Record = {} for (const { cell, status } of statuses) { const prefix = `cell.${cell.cellId}` + if (status.reserve) { + // cell-status carries the live cap; unset in Terraform keeps the evaluator's default, as there. + if (cell.connectionHardCap !== null) { + addSignal(signals, `${prefix}.connection_hard_cap`, cell.connectionHardCap, nowAt) + } + // Legacy admission needs relay_cells.enabled, which only cell-status reads: left missing. + if (selector.generation !== 0) { + addSignal( + signals, + `${prefix}.admission_state`, + ['existing-only', 'migration-only', 'general'].indexOf( + selectorCellState(selector, cell.cellId) + ), + nowAt + ) + } + continue + } const admissionState = effectiveAdmissionState( selector, status.enabled, @@ -645,12 +689,13 @@ export async function directorSignals( return { source: { observedAt: nowAt, signals }, selector, - cells: statuses.map(({ cell }) => ({ + cells: statuses.map(({ cell, status }) => ({ cellId: cell.cellId, region: cell.region, runtimeKnown: true, powered: true, - expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId) + expectedAdmissionState: selectorCellState(expectedSelector, cell.cellId), + ...(status.reserve ? { reserve: true } : {}) })) } } diff --git a/cloud/apps/relay-ops/src/incident-monitor.test.ts b/cloud/apps/relay-ops/src/incident-monitor.test.ts index db1d8797d1c3..a0eadd1b1636 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.test.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.test.ts @@ -643,6 +643,71 @@ describe('incident monitor evaluator', () => { expect.objectContaining({ code: 'selector_mismatch' }) ) }) + + // Why: the director refuses cell-status and migration reads on a reserve cell (2026-10-19, c7). + it('judges a reserve cell on probes, logs and selector, without its refused reads', () => { + const reserveSample = (): IncidentSample => { + const sample = healthySample() + sample.cells[0] = { ...sample.cells[0]!, reserve: true } + const admin = sample.sources['director-admin']!.signals + delete admin['cell.production-gce-c1.heartbeat_fresh'] + delete admin['cell.production-gce-c1.heartbeat_age_ms'] + delete admin['cell.production-gce-c1.migration_blocked'] + delete admin['cell.production-gce-c1.migration_target_inactive'] + return sample + } + expect(evaluateIncidentSample(reserveSample(), startedAt)).toMatchObject({ + status: 'green', + failures: [] + }) + + const crowded = reserveSample() + crowded.sources['relay-logs']!.signals['cell.production-gce-c1.queued_bytes'] = signal( + INCIDENT_MONITOR_THRESHOLDS.cellQueuedBytes + 1 + ) + expect(evaluateIncidentSample(crowded, startedAt).failures).toContainEqual( + expect.objectContaining({ signal: 'cell.production-gce-c1.queued_bytes' }) + ) + + // The connections ceiling is never skipped: the collector's configured cap, else the default. + const busy = reserveSample() + busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(731) + expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual( + expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 500 }) + ) + busy.sources['director-admin']!.signals['cell.production-gce-c1.connection_hard_cap'] = + signal(1_000) + expect(evaluateIncidentSample(busy, startedAt).status).toBe('green') + busy.sources['relay-logs']!.signals['cell.production-gce-c1.connections'] = signal(1_000) + expect(evaluateIncidentSample(busy, startedAt).failures).toContainEqual( + expect.objectContaining({ signal: 'cell.production-gce-c1.connections', threshold: 999 }) + ) + + const unready = reserveSample() + unready.sources['active-probe']!.signals['cell.production-gce-c1.ready'] = signal(0) + expect(evaluateIncidentSample(unready, startedAt).failures).toContainEqual( + expect.objectContaining({ signal: 'cell.production-gce-c1.ready' }) + ) + + const unknownAdmission = reserveSample() + delete unknownAdmission.sources['director-admin']!.signals[ + 'cell.production-gce-c1.admission_state' + ] + expect(evaluateIncidentSample(unknownAdmission, startedAt).failures).toContainEqual({ + code: 'signal_missing', + source: 'director-admin', + signal: 'cell.production-gce-c1.admission_state' + }) + + const notReserve = reserveSample() + notReserve.cells[0] = { ...notReserve.cells[0]!, reserve: false } + expect(evaluateIncidentSample(notReserve, startedAt).failures).toContainEqual( + expect.objectContaining({ + code: 'signal_missing', + signal: 'cell.production-gce-c1.heartbeat_fresh' + }) + ) + }) }) describe('incident monitor lifecycle', () => { diff --git a/cloud/apps/relay-ops/src/incident-monitor.ts b/cloud/apps/relay-ops/src/incident-monitor.ts index 94c7a4791498..bcb3c30bf092 100644 --- a/cloud/apps/relay-ops/src/incident-monitor.ts +++ b/cloud/apps/relay-ops/src/incident-monitor.ts @@ -182,6 +182,8 @@ export type IncidentCellExpectation = { runtimeKnown: boolean powered: boolean expectedAdmissionState: AdmissionState + // Step 5 reserve mode: the director's cell-status and migration reads refuse the cell. + reserve?: boolean } export type IncidentSample = { @@ -558,6 +560,15 @@ function checkCell( ] ] as const for (const [source, signals, signalName, threshold, comparison] of checks) { + // Probes, logs and selector still judge a reserve cell; the reserve watch reads its runtime. + if ( + cell.reserve && + ['heartbeat_fresh', 'heartbeat_age_ms', 'migration_blocked', 'migration_target_inactive'].some( + (suffix) => signalName.endsWith(suffix) + ) + ) { + continue + } if ( migrationPolicy === 'recover-forward' && cell.cellId === recoverySourceCellId && diff --git a/cloud/apps/relay/src/activity-lease-reader-census.test.ts b/cloud/apps/relay/src/activity-lease-reader-census.test.ts new file mode 100644 index 000000000000..b8bc4aadc202 --- /dev/null +++ b/cloud/apps/relay/src/activity-lease-reader-census.test.ts @@ -0,0 +1,182 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import ts from 'typescript' +import { afterEach, describe, expect, it } from 'vitest' +import { RelayAssignmentStore } from './assignment-store.js' +import type { RelayCellConfig } from './config.js' +import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' + +// Step 6 (design §3.5): a phone on a `phonePath=memory` cell writes no `confirmation:` or +// `splice:` lease. Every reader of the lease table is classified by why that is safe; a new +// reader fails this test until someone decides. While the cell is recorded as reserve its +// control holds no lease either, so step 5's guards already cover it; after a flip back a +// memory phone looks like today's phone once its 90 s lease has expired. +const ACTIVITY_LEASE_READERS: Record = { + // Reads control leases only, so phone leases never change the answer. + 'assignment-store.ts:activateControlOnce': 'control-only', + 'assignment-store.ts:reconcileIdleRegionalRehome': 'control-only', + 'assignment-store.ts:renewTransactionalControlActivity': 'control-only', + 'assignment-store.ts:startRegionalRehomeCandidate': 'control-only', + 'control-renewal-statement.ts:CONTROL_RENEWAL_BATCH_SQL': 'control-only', + 'idle-regional-rehome-selection.ts:selectIdleRegionalRehomes': 'control-only', + // Refuses or skips a cell recorded as reserve (or an unreadable set). + 'assignment-store.ts:cellDeploymentStatus': 'refuses-or-skips-reserve-cells', + 'assignment-store.ts:cellEvacuationCapacity': 'refuses-or-skips-reserve-cells', + 'assignment-store.ts:releaseExpiredActivity': 'refuses-or-skips-reserve-cells', + 'assignment-store.ts:releaseExpiredActivityLeases': 'refuses-or-skips-reserve-cells', + 'assignment-store.ts:startEvacuation': 'refuses-or-skips-reserve-cells', + // Its only caller drops candidates whose source or target is a reserve cell. + 'idle-regional-rehome-selection.ts:sourceRequestUnits': 'refuses-or-skips-reserve-cells', + // Only hosts in an open drain, migration or rehome, which no reserve cell can have: + // setCellAdmitMode refuses reserve while one is open, and every start refuses a reserve cell. + 'assignment-store.ts:activeCellMigrations': 'open-flow-only', + 'assignment-store.ts:additionalActivityCellIds': 'open-flow-only', + 'assignment-store.ts:assertCellReservationAccounting': 'open-flow-only', + 'assignment-store.ts:beginCellDrainSend': 'open-flow-only', + 'assignment-store.ts:beginCellDrainSendOnce': 'open-flow-only', + 'assignment-store.ts:completeReadyEvacuations': 'open-flow-only', + 'assignment-store.ts:completeReadyRegionalRehomes': 'open-flow-only', + 'assignment-store.ts:reconcileDrainMigrationAccounting': 'open-flow-only', + 'assignment-store.ts:reconcileReservationAccounting': 'open-flow-only', + 'assignment-store.ts:refreshDrainMigrationLeasesOnce': 'open-flow-only', + 'registered-migration-abandonment.ts:ABANDONED_REGISTERED_MIGRATION': 'open-flow-only', + // One host's rows under lock; each caller is classified in the reserve-mode census. + 'assignment-store.ts:lockAssignmentActivities': 'host-lock', + // Metrics and previews: they report, and act on nothing. + 'assignment-inventory-snapshot.ts:readAssignmentInventorySnapshot': 'report-only', + 'region-correction-preview.ts:previewRegionalRehomeEligibility': 'report-only', + 'registered-migration-inventory.ts:readRegisteredMigrationInventory': 'report-only', + // Step 5's existing-only stranded release, fixed separately in #30609 before the first flip. + 'assignment-store.ts:assignmentStrandedOnUnservedCell': 'step-5-stranded-release' +} + +// A read is FROM, JOIN or USING on the table; `DELETE FROM` alone only writes. Not caught: a +// second read inside an already listed function, and counters derived from leases elsewhere. +const LEASE_TABLE = 'relay_assignment_activity_leases' +// Bare table-name values that never reach SQL. +const BARE_TABLE_NAME_USES: Record = { + 'database.ts:POSTGRES_TRANSACTION_PHASES': 'labels a failed statement by the table it names' +} +const LEASE_READ = /(? { + let database: RelayDatabase | undefined + + afterEach(async () => { + await database?.close() + database = undefined + }) + + it('classifies every reader of the activity-lease table', () => { + const { readers, bareTableNames } = activityLeaseReaders() + // A bare name could reach `FROM ${table}`, which the scan cannot see: write the table into the SQL. + expect(bareTableNames.filter((use) => !(use in BARE_TABLE_NAME_USES))).toEqual([]) + expect(readers.filter((name) => !(name in ACTIVITY_LEASE_READERS))).toEqual([]) + expect(Object.keys(ACTIVITY_LEASE_READERS).filter((name) => !readers.includes(name))).toEqual([]) + }) + + async function setup(now: () => number) { + database = await openInMemoryRelayDatabase() + const store = new RelayAssignmentStore(database, now) + await store.reconcileCells(CELLS) + return store + } + + // A host whose source cell kept no lease for it, as a memory cell's control and phones do. + async function registeredEvacuationWithoutSourceLeases(store: RelayAssignmentStore) { + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + expect(first.cellId).toBe('cell-a') + const migration = await store.startEvacuation(identity, 'cell-b') + await store.activateControl(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch, + generation: 1 + }) + await store.markMigrationTargetRegistered(identity, { + cellId: 'cell-b', + assignmentEpoch: migration.assignmentEpoch + }) + await store.releaseActivity(identity, `control-pending:${first.assignmentEpoch}`) + return { identity, migration } + } + + it('completes no evacuation from a reserve source, though it holds no source lease', async () => { + const now = () => 1_000 + const store = await setup(now) + const { identity, migration } = await registeredEvacuationWithoutSourceLeases(store) + // setCellAdmitMode refuses this with the migration open; written directly to reach the second guard. + await database!.query( + `INSERT INTO relay_cell_admit_modes (cell_id, admit_mode, updated_at) VALUES ('cell-a', 'reserve', 1000)` + ) + const fresh = new RelayAssignmentStore(database!, now) + await expect(fresh.completeEvacuation(identity, migration.assignmentEpoch)).rejects.toThrow( + 'cell_in_reserve_mode' + ) + await expect(fresh.completeReadyEvacuations()).rejects.toThrow('cell_in_reserve_mode') + expect( + await database!.query(`SELECT completed_at FROM relay_assignment_migrations`) + ).toEqual([{ completed_at: null }]) + // The same host on a db source completes: the guard, not the fixture, held it. + await store.setCellAdmitMode('cell-a', 'db') + expect(await store.completeReadyEvacuations()).toBe(1) + }) + + it('refuses evacuation status for a reserve cell, where activeCellMigrations would count no units', async () => { + const store = await setup(() => 1_000) + await store.setCellAdmitMode('cell-a', 'reserve') + await expect(store.cellEvacuationStatus('cell-a', 'cell-b', true)).rejects.toThrow( + 'cell_in_reserve_mode' + ) + await expect(store.cellEvacuationStatus('cell-b', 'cell-a', false)).rejects.toThrow( + 'cell_in_reserve_mode' + ) + }) +}) + +function activityLeaseReaders(): { readers: string[]; bareTableNames: string[] } { + const directory = fileURLToPath(new URL('.', import.meta.url)) + const readers = new Set() + const bareTableNames: string[] = [] + const files = readdirSync(directory, { recursive: true, encoding: 'utf8' }).filter( + (file) => file.endsWith('.ts') && !file.includes('.test.') && !file.split(/[\\/]/).includes('test-fixtures') + ) + for (const file of files) { + const text = readFileSync(join(directory, file), 'utf8') + if (!text.includes(LEASE_TABLE)) continue + const source = ts.createSourceFile(file, text, ts.ScriptTarget.ES2022, true) + const visit = (node: ts.Node): void => { + if (ts.isNoSubstitutionTemplateLiteral(node) || ts.isTemplateExpression(node) || ts.isStringLiteral(node)) { + if ((ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) && node.text.trim() === LEASE_TABLE) { + bareTableNames.push(`${file}:${enclosingName(node)}`) + } + if (LEASE_READ.test(node.getText())) { + readers.add(`${file}:${enclosingName(node)}`) + return + } + } + ts.forEachChild(node, visit) + } + visit(source) + } + return { readers: [...readers].sort(), bareTableNames } +} + +function enclosingName(node: ts.Node): string { + for (let current = node.parent; current; current = current.parent) { + if ((ts.isMethodDeclaration(current) || ts.isFunctionDeclaration(current)) && current.name) { + return current.name.getText() + } + // A module-level SQL constant, never a local `const rows = ...` inside a method. + if (ts.isVariableStatement(current) && ts.isSourceFile(current.parent)) { + return current.declarationList.declarations.map((declaration) => declaration.name.getText()).join(',') + } + } + return '' +} diff --git a/cloud/apps/relay/src/app.ts b/cloud/apps/relay/src/app.ts index a9d6c370c168..694bf72f113a 100644 --- a/cloud/apps/relay/src/app.ts +++ b/cloud/apps/relay/src/app.ts @@ -31,6 +31,7 @@ import { RelayAssignmentRowBusyError, RelayCellInReserveModeError, RelayHomeCellUnavailableError, + type AssignOptions, type CellFenceAttemptEvidence, type RelayAssignment, type RelayAssignmentStore, @@ -143,6 +144,7 @@ export function createRelayApp( cellReserverPoll?: () => void cellAdmitModeEffective?: () => 'db' | 'reserve' cellAdmitModeRaw?: () => 'db' | 'reserve' + cellPhonePathEffective?: () => 'db' | 'memory' cellReserveCounts?: () => { bookings: number intake: { perSec: number; burst: number; tokens: number } @@ -459,13 +461,14 @@ export function createRelayApp( .setExpirationTime('5m') .sign(config.assignmentSigningKey) // Step 5: switched-on cells answer from the map; anything else keeps today's path below. - let step5: { epochFloor?: number; placeFresh?: () => Promise } = {} - if (operations.reservePlacement && !body.data.regionCorrection) { + let step5: Pick = {} + if (operations.reservePlacement && body.data.regionCorrection?.action !== 'report') { let plan: ReservePlan try { plan = await operations.reservePlacement.plan(identity, { reconnect: Boolean(body.data.reconnect), - region: targetRegion + region: targetRegion, + regionCorrection: body.data.regionCorrection !== undefined }) } catch (error) { if (!(error instanceof ReservePaceError)) throw error @@ -491,7 +494,7 @@ export function createRelayApp( context.header('Retry-After', String(plan.retryAfterSeconds)) return context.json({ error: 'assignment_temporarily_unavailable' }, 503) } - step5 = { epochFloor: plan.epochFloor, placeFresh: plan.placeFresh } + step5 = { epochFloor: plan.epochFloor, placeFresh: plan.placeFresh, regionReserve: plan.regionReserve } } let admission: { release(): void } | null = null let lane: AssignmentAdmissionLane = 'placement' @@ -891,6 +894,9 @@ export function createRelayApp( ...(operations.cellAdmitModeEffective ? { admitModeEffective: operations.cellAdmitModeEffective() } : {}), + ...(operations.cellPhonePathEffective + ? { phonePathEffective: operations.cellPhonePathEffective() } + : {}), // Placement as built, not as configured: `on` only when this director books reserve cells. ...(config.role === 'director' ? { reservePlacement: operations.reservePlacement?.placementMode ?? 'off' } diff --git a/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts b/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts index 460a0afcf887..306c25043d93 100644 --- a/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts +++ b/cloud/apps/relay/src/assignment-isolated-cell-replacement.test.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto' import { afterEach, describe, expect, it, vi } from 'vitest' import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js' +import { ReservePaceError } from './reserve-assignment.js' import { encodeMembership, type CellAdmissionMembership, @@ -564,6 +565,100 @@ describe('re-placing a host off a cell isolated for a roll', () => { expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1) }) + describe('with a reserve cell in its own region', () => { + const onReserve = { + ...IDENTITY, + cellId: 'asia-reserve', + cellUrl: 'https://asia-reserve.example.com', + assignmentEpoch: 9, + leaseExpiresAt: START_MS + 1_000_000, + region: 'asia-east2' as const + } + + it('books the reserve cell before leaving the region, and the canary still counts the move', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const { store, database, isolateForRoll } = await setup() + const first = await store.assign(IDENTITY, 'asia-east2') + await isolateForRoll(first.cellId) + warn.mockClear() + const regionReserve = vi.fn(async () => onReserve) + await expect( + store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve }) + ).resolves.toEqual(onReserve) + expect(regionReserve).toHaveBeenCalledOnce() + expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([ + { + event: 'orca_relay_sticky_replaced_off_isolated_cell', + fromCellId: first.cellId, + fromRegion: 'asia-east2', + admissionState: 'migration-only', + toCellId: 'asia-reserve', + region: 'asia-east2', + via: 'reserve' + } + ]) + // The refused in-region attempt wrote nothing. + expect( + await database.query(`SELECT cell_id, assignment_epoch FROM relay_assignments`) + ).toEqual([{ cell_id: first.cellId, assignment_epoch: first.assignmentEpoch }]) + }) + + it('leaves the region as before when the reserve cell has no room either', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const { store, isolateForRoll } = await setup() + const first = await store.assign(IDENTITY, 'asia-east2') + await isolateForRoll(first.cellId) + warn.mockClear() + const regionReserve = vi.fn(async () => null) + const moved = await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve }) + expect(moved).toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 }) + expect(regionReserve).toHaveBeenCalledOnce() + expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([ + { fromCellId: first.cellId, region: 'us-central1' } + ]) + }) + + it('treats a throttled reserve cell as no room, so the drain does not wait on its intake', async () => { + const { store, isolateForRoll } = await setup() + const first = await store.assign(IDENTITY, 'asia-east2') + await isolateForRoll(first.cellId) + const regionReserve = vi.fn(async () => { + throw new ReservePaceError(4) + }) + await expect( + store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve }) + ).resolves.toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 }) + expect(regionReserve).toHaveBeenCalledOnce() + }) + + it('counts a share-draw booking off the isolated cell for the canary', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const { store, isolateForRoll } = await setup() + const first = await store.assign(IDENTITY, 'asia-east2') + await isolateForRoll(first.cellId) + warn.mockClear() + await expect( + store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { placeFresh: async () => onReserve }) + ).resolves.toEqual(onReserve) + expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([ + { fromCellId: first.cellId, toCellId: 'asia-reserve', via: 'reserve' } + ]) + }) + + it('does not hold a host whose preferred region is elsewhere', async () => { + const { store, isolateForRoll, park } = await setup() + const first = await store.assign(IDENTITY, 'us-central1') + const other = CELLS.find((cell) => cell.region === 'us-central1' && cell.id !== first.cellId)! + await park(other.id, 'migration-only') + await isolateForRoll(first.cellId) + const regionReserve = vi.fn(async () => onReserve) + await expect( + store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve }) + ).resolves.toMatchObject({ cellId: 'asia-c1' }) + expect(regionReserve).not.toHaveBeenCalled() + }) + }) + it('takes the dead-cell path when a stamped cell stops heartbeating', async () => { // Why: the fence is the only proof that a cell we cannot reach has stopped // serving the sockets it still holds. A stamp does not make it reachable. diff --git a/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts b/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts index a10b49940132..077552c8f03c 100644 --- a/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts +++ b/cloud/apps/relay/src/assignment-reserve-mode-census.test.ts @@ -228,6 +228,27 @@ describe('reserve-mode census', () => { await restore() }) + it('keeps a connected host pinned when its reserve cell goes existing-only after a ledger write', async () => { + let now = 1_000 + const store = await setup(() => now) + const identity = { userId: 'user-a', relayHostId: 'host000000000001' } + const first = await store.assign(identity) + await setReserve(store, [first.cellId]) + // The reserve ledger stamps last_activity_at and the cell keeps no activity rows. + now += 5 * 60_000 + await database!.query('DELETE FROM relay_assignment_activity_leases WHERE user_id = ?', [identity.userId]) + await database!.query('UPDATE relay_assignments SET last_activity_at = ? WHERE user_id = ?', [now, identity.userId]) + await store.setCellAdmissionState(first.cellId, 'existing-only') + now += 2 * 60_000 + expect(await store.assign(identity)).toMatchObject({ + cellId: first.cellId, + assignmentEpoch: first.assignmentEpoch + }) + const { unknown, restore } = await unknownSet(() => now) + expect((await unknown.assign(identity)).assignmentEpoch).toBe(first.assignmentEpoch) + await restore() + }) + // Only the first reserve-set read is held; it is released (or failed) by the test. function holdFirstRead(options: { fail?: boolean } = {}) { const query = database!.query.bind(database!) diff --git a/cloud/apps/relay/src/assignment-store.ts b/cloud/apps/relay/src/assignment-store.ts index 66d90177bdac..38ce3e919a25 100644 --- a/cloud/apps/relay/src/assignment-store.ts +++ b/cloud/apps/relay/src/assignment-store.ts @@ -12,6 +12,7 @@ import { type RegionalRehomePollGate } from './regional-rehome-poll-telemetry.js' import { readRegionCorrectionOutcomes } from './region-correction-outcomes.js' +import { ReservePaceError } from './reserve-assignment.js' import { previewRegionalRehomeEligibility, type RegionCorrectionPreview @@ -145,6 +146,8 @@ type RelayAssignmentStoreOptions = { export type AssignOptions = { epochFloor?: number placeFresh?: () => Promise + // A reserve cell in the placement region with room: the database lane leaves the region only after it. + regionReserve?: () => Promise hostRequested?: boolean } @@ -523,7 +526,7 @@ class AssignmentInventoryLockUnavailable extends Error { // Postgres holds row locks to COMMIT, so the next tier needs a new transaction. class AssignmentIsolatedTierExhausted extends Error { - constructor() { + constructor(readonly source: { cellId: string; region: RelayRegion }) { super('assignment_isolated_tier_exhausted') } } @@ -534,6 +537,17 @@ class AssignmentInventoryScopeChanged extends Error { } } +// Sticky's "re-place me": the host's pin is on a cell isolated for a roll. +type IsolatedPin = { isolatedFrom: { cellId: string; region: RelayRegion } } + +// The placement region has no database cell with room; nothing was written. +class AssignmentRegionFull extends Error { + // Set when the host is leaving a roll-isolated cell in that region. + constructor(readonly isolatedSource?: { cellId: string; region: RelayRegion }) { + super('assignment_region_full') + } +} + // Why a reason of its own: a host whose home cell is fenced-but-unattested is // refused regardless of fleet headroom, so reporting it as capacity sends // operators after capacity that was never short. Every cell boot and every @@ -1003,30 +1017,67 @@ export class RelayAssignmentStore { step5: AssignOptions = {} ): Promise { const sticky = await this.assignStickyWithLockRetry(identity, lockMode, preferredRegion) - if (sticky) return sticky - const booked = await step5.placeFresh?.() + if (sticky && !('isolatedFrom' in sticky)) return sticky + let isolatedFrom = sticky?.isolatedFrom + // Off a roll-isolated cell: the canary counts the move whichever pool takes it, and a + // throttled reserve cell is no room, so the drain does not wait on reserve intake. + const bookReserve = async (book: () => Promise) => { + let reserved: RelayAssignment | null + try { + reserved = await book() + } catch (error) { + if (!(isolatedFrom && error instanceof ReservePaceError)) throw error + reserved = null + } + if (reserved && isolatedFrom) { + console.warn( + JSON.stringify({ + event: 'orca_relay_sticky_replaced_off_isolated_cell', + fromCellId: isolatedFrom.cellId, + fromRegion: isolatedFrom.region, + admissionState: ROLL_ISOLATED_ADMISSION, + toCellId: reserved.cellId, + region: reserved.region, + via: 'reserve' + }) + ) + } + return reserved + } + const booked = step5.placeFresh ? await bookReserve(step5.placeFresh) : null if (booked) return booked // Only placement needs the global inventory critical section; queueing those // attempts locally avoids turning true placement bursts into NOWAIT storms. - return await this.serializeAssignment( - async () => - await this.assignWithLockRetry( - identity, - lockMode, - preferredRegion, - placementRegion, - step5.epochFloor ?? 0, - // Only through the step-5 plan: its floor holds every epoch the map knows for the host. - step5.hostRequested === true && step5.epochFloor !== undefined - ) - ) + const placeOnDatabase = async (keepRegion: boolean) => + await this.serializeAssignment( + async () => + await this.assignWithLockRetry( + identity, + lockMode, + preferredRegion, + placementRegion, + step5.epochFloor ?? 0, + // Only through the step-5 plan: its floor holds every epoch the map knows for the host. + step5.hostRequested === true && step5.epochFloor !== undefined, + keepRegion + ) + ) + const regionReserve = step5.regionReserve + if (!regionReserve) return await placeOnDatabase(false) + try { + return await placeOnDatabase(true) + } catch (error) { + if (!(error instanceof AssignmentRegionFull)) throw error + isolatedFrom ??= error.isolatedSource + } + return (await bookReserve(regionReserve)) ?? (await placeOnDatabase(false)) } private async assignStickyWithLockRetry( identity: AssignmentIdentity, lockMode: CellInventoryLockMode, preferredRegion?: RelayRegion - ): Promise { + ): Promise { return await this.withAssignmentLockRetry( async (inventoryFirst) => await this.assignStickyOnce(identity, inventoryFirst, lockMode, preferredRegion) @@ -1039,7 +1090,8 @@ export class RelayAssignmentStore { preferredRegion?: RelayRegion, placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION, epochFloor = 0, - hostRequested = false + hostRequested = false, + keepRegion = false ): Promise { const deadline = Date.now() + ASSIGNMENT_LOCK_RETRY_DEADLINE_MS let inventoryScope: AssignmentInventoryScope = 'none' @@ -1052,7 +1104,8 @@ export class RelayAssignmentStore { preferredRegion, placementRegion, epochFloor, - hostRequested + hostRequested, + keepRegion ) } catch (error) { if (error instanceof AssignmentInventoryScopeChanged) { @@ -1060,6 +1113,10 @@ export class RelayAssignmentStore { continue } if (error instanceof AssignmentIsolatedTierExhausted) { + // An in-region reserve cell comes before the other regions. + if (keepRegion && error.source.region === placementRegion) { + throw new AssignmentRegionFull(error.source) + } inventoryScope = 'isolated-other-regions' continue } @@ -1097,7 +1154,7 @@ export class RelayAssignmentStore { inventoryFirst: boolean, lockMode: CellInventoryLockMode, preferredRegion?: RelayRegion - ): Promise { + ): Promise { const now = this.now() await this.reserveModeCells() return await this.database.transaction(async (transaction) => { @@ -1177,7 +1234,8 @@ export class RelayAssignmentStore { pinnedAdmission ) ) { - return null + const cellId = text(existing, 'cell_id') + return { isolatedFrom: { cellId, region: await this.cellRegion(transaction, cellId) } } } const currentCellId = text(existing, 'cell_id') @@ -1290,6 +1348,11 @@ export class RelayAssignmentStore { if (state !== 'existing-only') { return false } + // A reserve cell keeps no activity rows and its ledger stamps last_activity_at, so zero + // leases proves nothing; a dead one is left to the dead-reserve path and its fence. + if (await this.mayBeReserveCell(text(existing, 'cell_id'), transaction)) { + return false + } const liveLeases = ( await transaction.query( `SELECT COUNT(*) AS live FROM relay_assignment_activity_leases @@ -1320,7 +1383,9 @@ export class RelayAssignmentStore { preferredRegion?: RelayRegion, placementRegion: RelayRegion = preferredRegion ?? RELAY_DEFAULT_REGION, epochFloor = 0, - hostRequested = false + hostRequested = false, + // Refuse to spill out of placementRegion: AssignmentRegionFull instead. + keepRegion = false ): Promise { const now = this.now() // An isolated retry takes its tier's rows before anything reassigns this, @@ -1414,7 +1479,7 @@ export class RelayAssignmentStore { // Same region full: try the other regions. All full: decide under the // all-rows lock, which keeps the pin as before. if (!isolatedTarget) { - if (isolatedTier === 'same-region') throw new AssignmentIsolatedTierExhausted() + if (isolatedTier === 'same-region') throw new AssignmentIsolatedTierExhausted(source) throw new AssignmentInventoryScopeChanged() } isolatedIncumbent = source @@ -1582,8 +1647,15 @@ export class RelayAssignmentStore { // cross-region spill it exists to refuse. const target = isolatedTarget ?? - (await this.leastLoadedCell(transaction, lockedCells, placementRegion)) - if (!target) throw new Error('relay_capacity_exhausted') + (await this.leastLoadedCell( + transaction, + lockedCells, + placementRegion, + keepRegion ? 'require' : 'prefer' + )) + if (!target) { + throw keepRegion ? new AssignmentRegionFull() : new Error('relay_capacity_exhausted') + } if (deadReserveSource) { events.push( JSON.stringify({ @@ -8132,15 +8204,18 @@ export class RelayAssignmentStore { mode: CellInventoryLockMode ): Promise { if (sourceCellId === undefined) throw new AssignmentInventoryScopeChanged() + const sourceRegion = await this.cellRegion(database, sourceCellId) const candidates = await database.query( `SELECT admission.cell_id FROM relay_cell_admission admission LEFT JOIN relay_cell_regions region ON region.cell_id = admission.cell_id WHERE admission.admission_state = 'general' AND admission.cell_id <> ? AND COALESCE(region.region, ?) ${tier === 'same-region' ? '=' : '<>'} ?`, - [sourceCellId, RELAY_DEFAULT_REGION, await this.cellRegion(database, sourceCellId)] + [sourceCellId, RELAY_DEFAULT_REGION, sourceRegion] ) if (candidates.length === 0) { - if (tier === 'same-region') throw new AssignmentIsolatedTierExhausted() + if (tier === 'same-region') { + throw new AssignmentIsolatedTierExhausted({ cellId: sourceCellId, region: sourceRegion }) + } throw new AssignmentInventoryScopeChanged() } return await this.lockCellRows( diff --git a/cloud/apps/relay/src/cell-flags.ts b/cloud/apps/relay/src/cell-flags.ts index ccf22204d591..a3c3e08bcc8f 100644 --- a/cloud/apps/relay/src/cell-flags.ts +++ b/cloud/apps/relay/src/cell-flags.ts @@ -19,6 +19,8 @@ export type CellFlags = { reserveDryRun: boolean // Flip-back leases in flight at once; absent is a third of the pool. Speeds up one cell. reregisterInFlight?: number + // Step 6: `memory` resumes paired phones from one device read; it acts only in reserve mode. + phonePath: 'db' | 'memory' } export const CELL_FLAG_DEFAULTS: CellFlags = { @@ -26,7 +28,8 @@ export const CELL_FLAG_DEFAULTS: CellFlags = { ticketCheck: 'off', rejectionFence: true, admitMode: 'db', - reserveDryRun: false + reserveDryRun: false, + phonePath: 'db' } export const READ_TIMEOUT_MARGIN_MIN_MS = 1_000 @@ -52,7 +55,8 @@ const CellFlagsSchema = z.object({ admitMode: z.enum(['db', 'reserve']).default(CELL_FLAG_DEFAULTS.admitMode), intakePerSec: z.number().positive().max(1_000).optional(), reserveDryRun: z.boolean().default(CELL_FLAG_DEFAULTS.reserveDryRun), - reregisterInFlight: z.number().int().min(1).max(16).optional() + reregisterInFlight: z.number().int().min(1).max(16).optional(), + phonePath: z.enum(['db', 'memory']).default(CELL_FLAG_DEFAULTS.phonePath) }) const KNOWN_FLAG_KEYS = new Set(Object.keys(CellFlagsSchema.shape)) diff --git a/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts new file mode 100644 index 000000000000..fb6fb7abe140 --- /dev/null +++ b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts @@ -0,0 +1,530 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { reserveDeadManWindowMs } from './cell-reserve-dead-man.js' +import { openRelayDatabase } from './database.js' +import { nextFrame, RESERVE_MODE_CELL_ID, startReserveModeCell } from './test-fixtures/reserve-mode-cell.js' +import { StatementRecordingDatabase } from './test-fixtures/statement-recording-database.js' + +// Step 6: a reserve-mode cell with phonePath=memory resumes paired phones without the database. + +// Phone credentials are 32 random bytes. +const token = (fill: number) => Buffer.alloc(32, fill).toString('base64url') +const TOKEN = token(1) +const OTHER_CELL = { id: 'production-gce-c4', url: 'https://c4.relay.onorca.dev', capacityRequests: 900 } + +// A paired phone's accept and attach with the switch off, as main issued them at 9ea54343710. +const DATABASE_PATH_ACCEPT = [ + 'SELECT * FROM relay_devices WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)', + 'SELECT assignment.*, cell.cell_url, region.region FROM relay_assignments assignment JOIN relay_cells cell ON cell.cell_id = assignment.cell_id LEFT JOIN relay_cell_regions region ON region.cell_id = cell.cell_id WHERE assignment.user_id = ? AND assignment.relay_host_id = ?', + 'BEGIN', + 'SELECT * FROM relay_invites WHERE relay_host_id = ? AND token_hash = ?', + 'SELECT * FROM relay_devices WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)', + 'COMMIT', + 'BEGIN', + 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ? [locked]', + 'SELECT * FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC [locked]', + 'UPDATE relay_assignments SET pending_confirmations = CASE WHEN pending_confirmations + ? < 0 THEN 0 ELSE pending_confirmations + ? END, lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?', + 'INSERT INTO relay_assignment_activity_leases (user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units, expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + 'UPDATE relay_cells SET reserved_requests = CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, updated_at = ? WHERE cell_id = ? AND (? <= 0 OR reserved_requests + ? <= capacity_requests) RETURNING cell_id', + 'COMMIT', + 'BEGIN', + 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ? [locked]', + 'SELECT * FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC [locked]', + 'UPDATE relay_assignments SET reserved_splices = CASE WHEN reserved_splices + ? < 0 THEN 0 ELSE reserved_splices + ? END, lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?', + 'INSERT INTO relay_assignment_activity_leases (user_id, relay_host_id, activity_id, activity_kind, cell_id, request_units, expires_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + 'UPDATE relay_cells SET reserved_requests = CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, updated_at = ? WHERE cell_id = ? AND (? <= 0 OR reserved_requests + ? <= capacity_requests) RETURNING cell_id', + 'COMMIT', + 'INSERT INTO relay_connection_bases (basis_conn_id, user_id, relay_host_id, relay_device_id, owning_control_generation, credential_kind, invite_token_hash, accepted_credential_version, accepted_as, deadline, active, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)' +] + +const CONFIRM_DEVICE_READ = + 'SELECT * FROM relay_devices WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?' + +// The flag-off confirm, as main issued it at 9ea54343710. +const DATABASE_PATH_CONFIRM = [ + 'BEGIN', + 'SELECT basis_conn_id, result_json FROM relay_confirm_results WHERE user_id = ? AND relay_host_id = ? AND req_id = ?', + 'SELECT * FROM relay_connection_bases WHERE basis_conn_id = ?', + 'UPDATE relay_devices SET updated_at = updated_at WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?', + 'SELECT * FROM relay_devices WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?', + 'UPDATE relay_devices SET current_expires_at = ?, updated_at = ? WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?', + 'INSERT INTO relay_confirm_results (user_id, relay_host_id, req_id, basis_conn_id, tuple_json, result_json, committed_at) VALUES (?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO relay_audit_events (id, at, type, user_id, relay_host_id, relay_device_id, detail_json) VALUES (?, ?, ?, ?, ?, ?, ?)', + 'COMMIT', + 'BEGIN', + 'SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ? [locked]', + 'SELECT * FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? ORDER BY activity_id ASC [locked]', + 'DELETE FROM relay_assignment_activity_leases WHERE user_id = ? AND relay_host_id = ? AND activity_id = ?', + 'UPDATE relay_assignments SET pending_confirmations = CASE WHEN pending_confirmations + ? < 0 THEN 0 ELSE pending_confirmations + ? END, lease_expires_at = CASE WHEN lease_expires_at > ? THEN lease_expires_at ELSE ? END, last_activity_at = ? WHERE user_id = ? AND relay_host_id = ?', + 'UPDATE relay_cells SET reserved_requests = CASE WHEN reserved_requests + ? < 0 THEN 0 ELSE reserved_requests + ? END, updated_at = ? WHERE cell_id = ? AND (? <= 0 OR reserved_requests + ? <= capacity_requests) RETURNING cell_id', + 'COMMIT' +] + +async function until(check: () => Promise, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs + while (!(await check())) { + if (Date.now() > deadline) throw new Error('condition not reached') + await new Promise((resolve) => setTimeout(resolve, 50)) + } +} + +describe('the memory phone path', () => { + const cleanup: Array<() => Promise | void> = [] + + afterEach(async () => { + for (const close of cleanup.splice(0).reverse()) await close() + vi.restoreAllMocks() + }) + + async function cell(now?: () => number) { + vi.spyOn(console, 'log').mockImplementation(() => undefined) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(console, 'error').mockImplementation(() => undefined) + const dataDir = mkdtempSync(join(tmpdir(), 'orca-relay-phone-path-')) + cleanup.push(() => rmSync(dataDir, { recursive: true, force: true })) + const inner = await openRelayDatabase({ dataDir }) + cleanup.push(() => inner.close()) + const database = new StatementRecordingDatabase(inner) + const reserve = await startReserveModeCell({ database, dataDir, cleanup, ...(now ? { now } : {}) }) + // The admit-effective writer ticks on its own; every other statement is the phone's. + const phoneStatements = () => + database.statements.filter((sql) => !sql.includes('relay_cell_admit_effective')) + // The test's own reads and writes go around the recorder. + return { ...reserve, database, raw: inner, phoneStatements } + } + + async function seatedPhoneHost(phonePath: 'db' | 'memory') { + const reserve = await cell() + reserve.setAdmitMode('reserve', { phonePath }) + const seated = await reserve.bookedHost(3) + await reserve.installDevice(seated.identity.hostId, TOKEN) + reserve.database.statements.length = 0 + return { reserve, seated } + } + + // A phone resumed and spliced to its desktop, waiting to confirm. + async function splicedPhone( + phonePath: 'db' | 'memory', + device: { expiresAt?: number } = {}, + clock: { now?: () => number; beforeDial?: () => void } = {} + ) { + const reserve = await cell(clock.now) + reserve.setAdmitMode('reserve', { phonePath }) + const seated = await reserve.bookedHost(3) + await reserve.installDevice(seated.identity.hostId, TOKEN, device) + clock.beforeDial?.() + const connOpen = nextFrame(seated.socket, 'conn-open') + const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN) + const basisConnId = String((await connOpen).connId) + await reserve.attachData(await connOpen, seated.ack.generation) + expect(await phone.hello).toMatchObject({ ok: true }) + reserve.database.statements.length = 0 + // The desktop's confirm, sent in reply to the phone over the end-to-end channel. + const confirm = async (reqId: string, basis = basisConnId) => { + const answer = Promise.race([ + nextFrame(seated.socket, 'device-resume-confirmed'), + nextFrame(seated.socket, 'control-error') + ]) + seated.socket.send(JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId: basis })) + return await answer + } + return { reserve, seated, phone, basisConnId, confirm } + } + + const deviceRow = async (reserve: Awaited>) => + (await reserve.raw.query(`SELECT current_expires_at FROM relay_devices`))[0] + + const warnings = (event: string) => + vi + .mocked(console.warn) + .mock.calls.map(([line]) => String(line)) + .filter((line) => line.includes(event)) + .map((line) => JSON.parse(line)) + + it('acts only while the cell is in reserve mode, so admitMode=db or a dead-man trip turns it off', async () => { + let now = Date.now() + const reserve = await cell(() => now) + expect(reserve.relay.sessions.phonePathEffective()).toBe('db') + reserve.setAdmitMode('reserve', { phonePath: 'memory' }) + expect(reserve.relay.sessions.phonePathEffective()).toBe('memory') + reserve.setAdmitMode('db', { phonePath: 'memory' }) + expect(reserve.relay.sessions.phonePathEffective()).toBe('db') + reserve.setAdmitMode('reserve', { phonePath: 'memory' }) + expect(reserve.relay.sessions.phonePathEffective()).toBe('memory') + now += reserveDeadManWindowMs(reserve.config.cellId) + 1_000 + expect(reserve.relay.sessions.inReserveMode()).toBe(false) + expect(reserve.relay.sessions.phonePathEffective()).toBe('db') + }) + + it('admits a seated phone resume from one device read and writes no activity lease', async () => { + const { reserve, seated } = await seatedPhoneHost('memory') + const connOpen = nextFrame(seated.socket, 'conn-open') + const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN) + expect(await connOpen).toMatchObject({ kind: 'resume', relayDeviceId: 'device-1' }) + expect(reserve.phoneStatements()).toEqual([DATABASE_PATH_ACCEPT[0]]) + await reserve.attachData(await connOpen, seated.ack.generation) + expect(await phone.hello).toMatchObject({ type: 'relay-hello', ok: true, credentialKind: 'resume' }) + expect(reserve.phoneStatements()).toEqual([DATABASE_PATH_ACCEPT[0]]) + expect(await reserve.activityLeases()).toEqual([]) + }) + + it('keeps the database path statement for statement while phonePath is db', async () => { + const { reserve, seated } = await seatedPhoneHost('db') + const connOpen = nextFrame(seated.socket, 'conn-open') + const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN) + await reserve.attachData(await connOpen, seated.ack.generation) + expect(await phone.hello).toMatchObject({ type: 'relay-hello', ok: true }) + expect(reserve.phoneStatements()).toEqual(DATABASE_PATH_ACCEPT) + }) + + it('closes a revoked or expired device 4401 and never tries it as an invite', async () => { + const reserve = await cell() + reserve.setAdmitMode('reserve', { phonePath: 'memory' }) + const seated = await reserve.bookedHost(3) + await reserve.installDevice(seated.identity.hostId, token(2), { revokedAt: Date.now() - 1_000 }) + await reserve.installDevice(seated.identity.hostId, token(3), { + relayDeviceId: 'device-2', + expiresAt: Date.now() - 1_000 + }) + for (const refused of [token(2), token(3)]) { + reserve.database.statements.length = 0 + const phone = await reserve.dialPhone(seated.identity.hostId, refused) + expect(await phone.hello).toEqual({ type: 'relay-hello', ok: false, code: 4401 }) + expect((await phone.closed).code).toBe(4401) + expect(reserve.phoneStatements()).toEqual([DATABASE_PATH_ACCEPT[0]]) + } + }) + + it('closes 4503 when the device read fails, without trying the invite path', async () => { + const { reserve, seated } = await seatedPhoneHost('memory') + vi.spyOn(reserve.relay.store, 'lookupResume').mockRejectedValue(new Error('connection terminated')) + const invite = vi.spyOn(reserve.relay.store, 'resolveInviteForMove') + for (let dial = 0; dial < 2; dial += 1) { + const phone = await reserve.dialPhone(seated.identity.hostId, TOKEN) + expect((await phone.closed).code).toBe(4503) + } + expect(invite).not.toHaveBeenCalled() + expect(reserve.phoneStatements()).toEqual([]) + // One line opens the window; the repeat waits for its closing line. + expect(warnings('orca_relay_client_accept_db_error')).toEqual([ + { event: 'orca_relay_client_accept_db_error', cause: 'error', cellId: reserve.config.cellId } + ]) + }) + + it('falls through to the invite path only when no device row matches', async () => { + const { reserve, seated } = await seatedPhoneHost('memory') + const created = nextFrame(seated.socket, 'invite-created') + seated.socket.send(JSON.stringify({ type: 'invite-create', reqId: 'invite-1', relayDeviceId: 'device-9' })) + const invite = await created + const connOpen = nextFrame(seated.socket, 'conn-open') + await reserve.dialPhone(seated.identity.hostId, String(invite.inviteToken)) + expect(await connOpen).toMatchObject({ kind: 'invite', relayDeviceId: 'device-9' }) + // Invites keep today's leases: install validates their basis in Postgres. + expect((await reserve.activityLeases()).map((lease) => lease.activity_kind)).toEqual(['invite', 'invite']) + }) + + it('answers a phone whose host has no seat here from one assignment read', async () => { + const reserve = await cell() + reserve.setAdmitMode('reserve', { phonePath: 'memory' }) + await reserve.relay.assignments.reconcileCells([...reserve.config.cells, OTHER_CELL]) + const dial = async (row: 'elsewhere' | 'here' | 'none') => { + const host = await reserve.host() + if (row !== 'none') await reserve.insertRow(host.hostId, 3, row === 'here' ? reserve.config.cellId : OTHER_CELL.id) + await reserve.installDevice(host.hostId, TOKEN) + reserve.database.statements.length = 0 + const phone = await reserve.dialPhone(host.hostId, TOKEN) + const closed = await phone.closed + expect(reserve.phoneStatements()).toEqual(DATABASE_PATH_ACCEPT.slice(0, 2)) + return closed.code + } + expect(await dial('elsewhere')).toBe(4409) + expect(await dial('none')).toBe(4404) + expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([]) + expect(await dial('here')).toBe(4404) + expect(warnings('orca_relay_client_miss_row_names_here')).toEqual([ + expect.objectContaining({ cellId: reserve.config.cellId, relayHostIdDigest: expect.any(String) }) + ]) + vi.spyOn(reserve.relay.assignments, 'resolve').mockRejectedValueOnce(new Error('connection terminated')) + const host = await reserve.host() + await reserve.installDevice(host.hostId, TOKEN) + expect((await (await reserve.dialPhone(host.hostId, TOKEN)).closed).code).toBe(4409) + expect(warnings('orca_relay_client_miss_read_error')).toEqual([ + { event: 'orca_relay_client_miss_read_error', cause: 'error', cellId: reserve.config.cellId } + ]) + }) + + it('takes the database path for a phone whose switch turned off during its device read', async () => { + const { reserve, seated } = await seatedPhoneHost('memory') + const lookup = reserve.relay.store.lookupResume.bind(reserve.relay.store) + vi.spyOn(reserve.relay.store, 'lookupResume').mockImplementation(async (...args) => { + const found = await lookup(...args) + reserve.setAdmitMode('reserve', { phonePath: 'db' }) + return found + }) + const connOpen = nextFrame(seated.socket, 'conn-open') + await reserve.dialPhone(seated.identity.hostId, TOKEN) + expect(await connOpen).toMatchObject({ kind: 'resume' }) + expect((await reserve.activityLeases()).map((lease) => lease.activity_kind)).toEqual(['confirmation']) + }) + + it('confirms from the memory basis with one fresh read, renewing a device due for it', async () => { + const { reserve, confirm } = await splicedPhone('memory') + const before = Number((await deviceRow(reserve))?.current_expires_at) + const answer = await confirm('confirm-1') + expect(answer).toMatchObject({ type: 'device-resume-confirmed', reqId: 'confirm-1', renewed: true, acceptedAs: 'current' }) + const after = Number((await deviceRow(reserve))?.current_expires_at) + expect(after).toBeGreaterThan(before) + expect(answer.resumeExpiresAt).toBe(after) + expect(reserve.phoneStatements()).toEqual([ + CONFIRM_DEVICE_READ, + expect.stringMatching(/^UPDATE relay_devices SET current_expires_at = \?.*RETURNING current_expires_at$/) + ]) + expect(await reserve.activityLeases()).toEqual([]) + // Its audit row leaves with the next batch. + await reserve.relay.store.resumeAudit.flush() + const audits = await reserve.raw.query( + `SELECT detail_json FROM relay_audit_events WHERE type = 'resume-confirmed'` + ) + expect(audits.map((row) => JSON.parse(String(row.detail_json)))).toEqual([ + { reqId: 'confirm-1', basisConnId: expect.any(String), renewed: true } + ]) + // A retry of the same request answers from memory; another basis cannot reuse its reqId. + reserve.database.statements.length = 0 + expect(await confirm('confirm-1')).toEqual(answer) + expect(reserve.phoneStatements()).toEqual([]) + expect(await confirm('confirm-1', 'another-basis')).toMatchObject({ + type: 'control-error', + code: 'confirmation_tuple_mismatch' + }) + }) + + it('answers renewed:true with the stored expiry when renewal is not due, or its write fails', async () => { + const fresh = Date.now() + 29.5 * 24 * 60 * 60 * 1000 + const notDue = await splicedPhone('memory', { expiresAt: fresh }) + expect(await notDue.confirm('confirm-1')).toMatchObject({ renewed: true, resumeExpiresAt: fresh }) + expect(notDue.reserve.phoneStatements()).toEqual([CONFIRM_DEVICE_READ]) + + const due = Date.now() + 20 * 24 * 60 * 60 * 1000 + const failing = await splicedPhone('memory', { expiresAt: due }) + failing.reserve.database.failing = (sql) => sql.startsWith('UPDATE relay_devices') + expect(await failing.confirm('confirm-1')).toMatchObject({ renewed: true, resumeExpiresAt: due }) + expect(Number((await deviceRow(failing.reserve))?.current_expires_at)).toBe(due) + // Never silent: a renewal left failing ages phones toward re-pairing. + expect(warnings('orca_relay_resume_renewal_failed')).toEqual([ + { event: 'orca_relay_resume_renewal_failed', cause: 'error', cellId: failing.reserve.config.cellId } + ]) + }) + + it('confirms from memory after the switch turned off between accept and confirm, taking no lease', async () => { + const { reserve, confirm } = await splicedPhone('memory') + reserve.setAdmitMode('reserve', { phonePath: 'db' }) + expect(await confirm('confirm-1')).toMatchObject({ type: 'device-resume-confirmed', renewed: true }) + expect(reserve.phoneStatements()[0]).toBe(CONFIRM_DEVICE_READ) + expect(await reserve.activityLeases()).toEqual([]) + }) + + it('replays a stored answer after its splice closed, and drops the basis itself', async () => { + const { reserve, phone, confirm } = await splicedPhone('memory') + const answer = await confirm('confirm-1') + phone.socket.close() + await phone.closed + await new Promise((resolve) => setTimeout(resolve, 50)) + reserve.database.statements.length = 0 + expect(await confirm('confirm-1')).toEqual(answer) + expect(await confirm('confirm-1', 'another-basis')).toMatchObject({ code: 'confirmation_tuple_mismatch' }) + expect(reserve.phoneStatements()).toEqual([]) + // A new request on the closed basis finds none in memory, so it goes to the database. + expect(await confirm('confirm-2')).toMatchObject({ code: 'confirmation_not_active' }) + expect(reserve.phoneStatements()[0]).toBe('BEGIN') + }) + + it('answers a grace-version confirm renewed:false with its expiry unchanged, as today', async () => { + const { reserve, confirm } = await splicedPhone('memory') + const stored = Number((await deviceRow(reserve))?.current_expires_at) + // An install rotated the credential after the accept read, so the accepted version is grace. + await reserve.raw.query( + `UPDATE relay_devices SET grace_hash = current_hash, grace_version = 1, grace_expires_at = ?, + current_hash = 'rotated', current_version = 2`, + [Date.now() + 60_000] + ) + expect(await confirm('confirm-1')).toMatchObject({ + type: 'device-resume-confirmed', + currentVersion: 2, + acceptedAs: 'current', + renewed: false, + resumeExpiresAt: stored + }) + expect(reserve.phoneStatements()).toEqual([CONFIRM_DEVICE_READ]) + }) + + it('answers every device change since the accept read as the database confirm does', async () => { + const changes: Record = { + revoked: ['UPDATE relay_devices SET revoked_at = ?', [Date.now() - 1]], + expired: ['UPDATE relay_devices SET current_expires_at = ?', [Date.now() - 1]], + retired: ["UPDATE relay_devices SET current_hash = 'rotated-twice', current_version = 3, grace_version = 2", []], + unchanged: ['UPDATE relay_devices SET updated_at = updated_at', []] + } + const outcome = async (phonePath: 'db' | 'memory', [sql, params]: [string, unknown[]]) => { + const { reserve, confirm } = await splicedPhone(phonePath) + await reserve.raw.query(sql, params) + const answer = await confirm('confirm-1') + return answer.type === 'control-error' ? answer.code : `${answer.type}:${answer.acceptedAs}:${answer.renewed}` + } + for (const [name, change] of Object.entries(changes)) { + expect(await outcome('memory', change), name).toBe(await outcome('db', change)) + } + expect(await outcome('memory', changes.revoked!)).toBe('reject-revoked') + }) + + it('sends a confirm with no memory basis to the database, which holds none either', async () => { + const { reserve, phone, confirm } = await splicedPhone('memory') + phone.socket.close() + await phone.closed + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(await confirm('confirm-1')).toMatchObject({ type: 'control-error', code: 'confirmation_not_active' }) + expect(reserve.phoneStatements()[0]).toBe('BEGIN') + expect(warnings('orca_relay_resume_confirm_failed')).toEqual([ + { + event: 'orca_relay_resume_confirm_failed', + code: 'confirmation_not_active', + phonePath: 'db', + cellId: reserve.config.cellId + } + ]) + }) + + it('does not count a lease release that failed after a committed confirm as a failed confirm', async () => { + const { reserve, confirm } = await splicedPhone('db') + vi.spyOn(reserve.relay.assignments, 'releaseActivity').mockRejectedValueOnce(new Error('connection terminated')) + // As today, the desktop still hears the release failure. + expect(await confirm('confirm-1')).toMatchObject({ type: 'control-error' }) + expect(warnings('orca_relay_resume_confirm_failed')).toEqual([]) + }) + + it('keeps the database confirm statement for statement while phonePath is db', async () => { + const { reserve, confirm } = await splicedPhone('db') + expect(await confirm('confirm-1')).toMatchObject({ type: 'device-resume-confirmed', renewed: true }) + expect(reserve.phoneStatements()).toEqual(DATABASE_PATH_CONFIRM) + }) + + describe('rolling a memory cell back', () => { + // Flip back by the switch, or by the dead-man; either way the confirm finishes from memory. + type Cell = Awaited> + type Advance = (ms: number) => void + type Rollback = { beforeDial?: (advance: Advance) => void; roll: (reserve: Cell, advance: Advance) => void } + // The dead-man trips just after the phone's accept: its window ran from the host's booking. + const window = reserveDeadManWindowMs(RESERVE_MODE_CELL_ID) + const rollbacks: Record = { + 'admitMode=db': { roll: (reserve) => reserve.setAdmitMode('db', { phonePath: 'memory' }) }, + 'phonePath=db': { roll: (reserve) => reserve.setAdmitMode('reserve', { phonePath: 'db' }) }, + 'a dead-man trip': { beforeDial: (advance) => advance(window - 5_000), roll: (_reserve, advance) => advance(6_000) } + } + + for (const [name, rollback] of Object.entries(rollbacks)) { + it(`finishes an in-flight confirm on its memory basis after ${name}, writing and leaking no lease`, async () => { + let now = Date.now() + const advance: Advance = (ms) => { + now += ms + } + const spliced = await splicedPhone('memory', {}, { + now: () => now, + beforeDial: () => rollback.beforeDial?.(advance) + }) + const { reserve, seated } = spliced + // The flag tool records reserve in Postgres before the cell admits from memory. + await reserve.relay.assignments.reconcileCells([...reserve.config.cells, OTHER_CELL]) + await reserve.relay.assignments.setCellAdmitMode(reserve.config.cellId, 'reserve') + rollback.roll(reserve, advance) + expect(reserve.relay.sessions.phonePathEffective()).toBe('db') + reserve.database.statements.length = 0 + expect(await spliced.confirm('confirm-1')).toMatchObject({ type: 'device-resume-confirmed', renewed: true }) + expect(reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))).toEqual([ + CONFIRM_DEVICE_READ, + expect.stringMatching(/^UPDATE relay_devices SET current_expires_at/) + ]) + // Only the control lease a flip back re-registers; nothing for the phone. + const leases = async () => (await reserve.activityLeases()).map((lease) => lease.activity_kind) + expect((await leases()).filter((kind) => kind !== 'control')).toEqual([]) + // Postgres still records reserve, so evacuation leaves the cell alone. + await expect( + reserve.relay.assignments.startEvacuation( + { userId: 'user-1', relayHostId: seated.identity.hostId }, + OTHER_CELL.id + ) + ).rejects.toThrow('cell_in_reserve_mode') + // The splice still runs, and closing it releases nothing it never took. + expect(spliced.phone.socket.readyState).toBe(spliced.phone.socket.OPEN) + // Let a flip back finish leasing its control (and fold its relay_cells write) first. + await until(async () => reserve.relay.sessions.reregistrationPending() === 0) + await new Promise((resolve) => setTimeout(resolve, 250)) + reserve.database.statements.length = 0 + spliced.phone.socket.close() + await spliced.phone.closed + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(reserve.phoneStatements()).toEqual([]) + expect((await leases()).filter((kind) => kind !== 'control')).toEqual([]) + }) + } + }) + + // E-dep-6 in miniature: the staging drill blocks one memory cell's database for 10 minutes. + it('rides out a database outage: splices keep forwarding, new work fails cleanly and recovers', async () => { + let now = Date.now() + const reserve = await cell(() => now) + reserve.setAdmitMode('reserve', { phonePath: 'memory' }) + const seated = await reserve.bookedHost(3) + const confirm = async (reqId: string, basisConnId: string) => { + const answer = Promise.race([ + nextFrame(seated.socket, 'device-resume-confirmed'), + nextFrame(seated.socket, 'control-error') + ]) + seated.socket.send(JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId })) + return await answer + } + const splice = async (fill: number) => { + await reserve.installDevice(seated.identity.hostId, token(fill), { relayDeviceId: `device-${fill}` }) + const connOpen = nextFrame(seated.socket, 'conn-open') + const phone = await reserve.dialPhone(seated.identity.hostId, token(fill)) + const basisConnId = String((await connOpen).connId) + const data = await reserve.attachData(await connOpen, seated.ack.generation) + expect(await phone.hello).toMatchObject({ ok: true }) + return { phone, data, basisConnId } + } + const confirmed = await splice(1) + expect(await confirm('confirm-a', confirmed.basisConnId)).toMatchObject({ type: 'device-resume-confirmed' }) + const inFlight = await splice(2) + + reserve.database.failing = () => true + // A seated phone keeps working. + const relayed = new Promise((resolve) => + confirmed.data.once('message', (frame) => resolve(String(frame))) + ) + confirmed.phone.socket.send('still here') + expect(await relayed).toBe('still here') + // A new resume and a confirm fail cleanly; nothing crashes. + expect((await (await reserve.dialPhone(seated.identity.hostId, token(1))).closed).code).toBe(4503) + expect(await confirm('confirm-b', inFlight.basisConnId)).toMatchObject({ type: 'control-error' }) + // Flipped back during the block: the in-flight confirm still finishes on its memory basis. + reserve.setAdmitMode('db', { phonePath: 'memory' }) + reserve.database.failing = null + reserve.database.statements.length = 0 + expect(await confirm('confirm-c', inFlight.basisConnId)).toMatchObject({ type: 'device-resume-confirmed' }) + expect(reserve.phoneStatements().filter((sql) => !sql.includes('relay_assignment'))[0]).toBe(CONFIRM_DEVICE_READ) + // And new resumes take the database path again. + const connOpen = nextFrame(seated.socket, 'conn-open') + await reserve.dialPhone(seated.identity.hostId, token(1)) + expect(await connOpen).toMatchObject({ kind: 'resume' }) + + // Audit rows held past 10 minutes of outage are dropped and counted, never written late. + reserve.database.failing = (sql) => sql.startsWith('INSERT INTO relay_audit_events') + now += 11 * 60 * 1000 + await reserve.relay.store.resumeAudit.flush() + reserve.database.failing = null + const written = await reserve.raw.query(`SELECT id FROM relay_audit_events WHERE type = 'resume-confirmed'`) + expect(written).toEqual([]) + expect(reserve.relay.runtimeCounts().resumeAuditDropped).toBe(2) + }) +}) + diff --git a/cloud/apps/relay/src/cell-seat-feed.test.ts b/cloud/apps/relay/src/cell-seat-feed.test.ts index 487f6356709e..1945af12b8cc 100644 --- a/cloud/apps/relay/src/cell-seat-feed.test.ts +++ b/cloud/apps/relay/src/cell-seat-feed.test.ts @@ -72,7 +72,8 @@ const SeatFeedReplySchema = z.object({ ticketCheck: z.enum(['off', 'shadow', 'enforce']), rejectionFence: z.boolean(), admitMode: z.enum(['db', 'reserve']), - reserveDryRun: z.boolean() + reserveDryRun: z.boolean(), + phonePath: z.enum(['db', 'memory']) }), ignoredKeys: z.array(z.string()).optional() }) @@ -200,6 +201,7 @@ function createCell(relayConfig = config(), cellFlags?: () => AppliedControlFlag cellIncarnation: incarnation, cellSeatFeed: (sinceSeq) => registry.seatFeed(sinceSeq), cellFlags, + cellPhonePathEffective: () => registry.phonePathEffective(), isDraining: () => registry.isDraining(), runtimeCounts: () => ({ totalConnections: 0, @@ -349,6 +351,8 @@ describe('cell seat feed', () => { }) expect(await runtime.json()).toMatchObject({ flagsApplied: applied, + // Not reserve mode, so db whatever the switch says. + phonePathEffective: 'db', databasePoolMax: expect.any(Number), // What the flag tool checks a write against before it writes. supportedFlags: { @@ -359,7 +363,8 @@ describe('cell seat feed', () => { admitMode: { type: 'enum', values: ['db', 'reserve'] }, intakePerSec: { type: 'number', max: 1_000 }, reserveDryRun: { type: 'boolean' }, - reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true } + reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }, + phonePath: { type: 'enum', values: ['db', 'memory'] } } }) }) diff --git a/cloud/apps/relay/src/control-close-reason-text.ts b/cloud/apps/relay/src/control-close-reason-text.ts new file mode 100644 index 000000000000..19ac79ea648f --- /dev/null +++ b/cloud/apps/relay/src/control-close-reason-text.ts @@ -0,0 +1,7 @@ +// Peer-supplied close reasons are logged; keep them printable and short. +export function printableCloseReason(reason: Buffer | string): string { + return reason + .toString() + .replace(/[^\x20-\x7e]/g, '') + .slice(0, 80) +} diff --git a/cloud/apps/relay/src/credential-store.ts b/cloud/apps/relay/src/credential-store.ts index 8d17ba623f39..58e9ae3c47aa 100644 --- a/cloud/apps/relay/src/credential-store.ts +++ b/cloud/apps/relay/src/credential-store.ts @@ -7,6 +7,7 @@ import { } from '@orca-cloud/relay-contract' import type { RelayDatabase, SqlRow } from './database.js' import { HeapWindowReaper } from './heap-window-reaper.js' +import { ResumeConfirmedAuditBuffer } from './resume-confirmed-audit-buffer.js' const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000 // Released desktops validate invite expiry against their own clock with zero @@ -95,6 +96,64 @@ function string(row: SqlRow, field: string): string { return value } +// Memory confirms renew a device at most this often (design decision 3: at most once a day of use). +export const RESUME_RENEWAL_INTERVAL_MS = 24 * 60 * 60 * 1000 + +function tokenFamily(device: SqlRow) { + return { + currentVersion: number(device, 'current_version'), + currentHash: string(device, 'current_hash'), + currentExpiresAt: number(device, 'current_expires_at'), + graceVersion: optionalNumber(device, 'grace_version'), + graceHash: typeof device.grace_hash === 'string' ? device.grace_hash : undefined, + graceExpiresAt: optionalNumber(device, 'grace_expires_at'), + revokedAt: optionalNumber(device, 'revoked_at') + } +} + +// A device row's answer to a resume token: null when revoked, or neither hash is still valid. +function resumeReservation( + device: SqlRow, + relayHostId: string, + tokenHash: string, + reservationId: string, + now: number +): CredentialReservation | null { + if (optionalNumber(device, 'revoked_at') !== undefined) return null + const currentHash = string(device, 'current_hash') + const currentExpiresAt = number(device, 'current_expires_at') + const graceHash = device.grace_hash + const graceExpiresAt = optionalNumber(device, 'grace_expires_at') + let acceptedAs: 'current' | 'grace' + let acceptedCredentialVersion: number + if (equalHash(currentHash, tokenHash) && currentExpiresAt > now) { + acceptedAs = 'current' + acceptedCredentialVersion = number(device, 'current_version') + } else if ( + typeof graceHash === 'string' && + equalHash(graceHash, tokenHash) && + (graceExpiresAt ?? 0) > now + ) { + acceptedAs = 'grace' + acceptedCredentialVersion = number(device, 'grace_version') + } else { + return null + } + return { + userId: string(device, 'user_id'), + relayHostId, + relayDeviceId: string(device, 'relay_device_id'), + credentialKind: 'resume', + tokenHash, + reservationId, + leaseExpiresAt: now + RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs, + acceptedCredentialVersion, + acceptedAs, + resumeExpiresAt: currentExpiresAt, + graceExpiresAt + } +} + export class RelayCredentialStore { // committed_at has no index, so a LIMIT delete would plan as a sequential scan of the whole table. private readonly confirmResultReaper = new HeapWindowReaper( @@ -103,10 +162,15 @@ export class RelayCredentialStore { CONFIRM_RESULT_REAP_BUDGET ) + // Only memory confirms add to it, so its flush timer never starts on a director. + readonly resumeAudit: ResumeConfirmedAuditBuffer + constructor( private readonly database: RelayDatabase, private readonly now: () => number = Date.now - ) {} + ) { + this.resumeAudit = new ResumeConfirmedAuditBuffer(database, now) + } async createInvite(identity: RelayIdentity, relayDeviceId: string): Promise<{ inviteToken: string @@ -216,42 +280,33 @@ export class RelayCredentialStore { [relayHostId, tokenHash, tokenHash] ) const device = deviceRows[0] - if (!device || optionalNumber(device, 'revoked_at') !== undefined) return null - const currentHash = string(device, 'current_hash') - const currentExpiresAt = number(device, 'current_expires_at') - const graceHash = device.grace_hash - const graceExpiresAt = optionalNumber(device, 'grace_expires_at') - let acceptedAs: 'current' | 'grace' - let acceptedCredentialVersion: number - if (equalHash(currentHash, tokenHash) && currentExpiresAt > now) { - acceptedAs = 'current' - acceptedCredentialVersion = number(device, 'current_version') - } else if ( - typeof graceHash === 'string' && - equalHash(graceHash, tokenHash) && - (graceExpiresAt ?? 0) > now - ) { - acceptedAs = 'grace' - acceptedCredentialVersion = number(device, 'grace_version') - } else { - return null - } - return { - userId: string(device, 'user_id'), - relayHostId, - relayDeviceId: string(device, 'relay_device_id'), - credentialKind: 'resume', - tokenHash, - reservationId, - leaseExpiresAt: now + RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs, - acceptedCredentialVersion, - acceptedAs, - resumeExpiresAt: currentExpiresAt, - graceExpiresAt - } + return device ? resumeReservation(device, relayHostId, tokenHash, reservationId, now) : null }) } + // Step 6's one-read resume. Only `none` may fall through to the invite path: a device row + // that refuses this token closes the phone (4401), as reserveCredential would. + async lookupResume( + relayHostId: string, + token: string, + reservationId: string = randomUUID() + ): Promise< + | { outcome: 'none' } + | { outcome: 'refused' } + | { outcome: 'resume'; reservation: CredentialReservation } + > { + const tokenHash = hashCredential(token) + const rows = await this.database.query( + `SELECT * FROM relay_devices + WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)`, + [relayHostId, tokenHash, tokenHash] + ) + const device = rows[0] + if (!device) return { outcome: 'none' } + const reservation = resumeReservation(device, relayHostId, tokenHash, reservationId, this.now()) + return reservation ? { outcome: 'resume', reservation } : { outcome: 'refused' } + } + async recordConnectionBasis(input: CredentialReservation & { basisConnId: string owningControlGeneration: number @@ -483,19 +538,7 @@ export class RelayCredentialStore { const device = devices[0] if (!device) throw new RelayStoreError('credential_not_found') const acceptedVersion = number(basis, 'accepted_credential_version') - const decision = decideResumeCommit( - { - currentVersion: number(device, 'current_version'), - currentHash: string(device, 'current_hash'), - currentExpiresAt: number(device, 'current_expires_at'), - graceVersion: optionalNumber(device, 'grace_version'), - graceHash: typeof device.grace_hash === 'string' ? device.grace_hash : undefined, - graceExpiresAt: optionalNumber(device, 'grace_expires_at'), - revokedAt: optionalNumber(device, 'revoked_at') - }, - acceptedVersion, - now - ) + const decision = decideResumeCommit(tokenFamily(device), acceptedVersion, now) if (decision.startsWith('reject-')) throw new RelayStoreError(decision) const renewed = decision === 'renew-current' const resumeExpiresAt = renewed @@ -547,6 +590,82 @@ export class RelayCredentialStore { }) } + // Step 6's confirm for a basis the cell holds in memory: one fresh device read, and a renewal + // write at most once a day per device. A current-version confirm always answers renewed:true + // with the expiry the database now holds, so a phone keeps scheduling its renewals. + async confirmMemoryResume( + input: RelayIdentity & { + reqId: string + basisConnId: string + relayDeviceId: string + acceptedCredentialVersion: number + acceptedAs: 'current' | 'grace' + }, + options: { onRenewalFailure?: (error: unknown) => void } = {} + ): Promise { + const now = this.now() + const devices = await this.database.query( + `SELECT * FROM relay_devices + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ?`, + [input.userId, input.relayHostId, input.relayDeviceId] + ) + const device = devices[0] + if (!device) throw new RelayStoreError('credential_not_found') + const decision = decideResumeCommit(tokenFamily(device), input.acceptedCredentialVersion, now) + if (decision.startsWith('reject-')) throw new RelayStoreError(decision) + const renewed = decision === 'renew-current' + let resumeExpiresAt = number(device, 'current_expires_at') + const renewalDue = resumeExpiresAt < now + RELAY_PROTOCOL_LIMITS.resumeTtlMs - RESUME_RENEWAL_INTERVAL_MS + if (renewed && renewalDue) { + // A failing renewal still answers the expiry the database holds, but must stay visible: + // left failing, phones age toward re-pairing. + resumeExpiresAt = await this.renewResume(input, device, now).catch((error: unknown) => { + options.onRenewalFailure?.(error) + return resumeExpiresAt + }) + } + this.resumeAudit.add({ + userId: input.userId, + relayHostId: input.relayHostId, + relayDeviceId: input.relayDeviceId, + detail: { reqId: input.reqId, basisConnId: input.basisConnId, renewed } + }) + return { + v: 1, + reqId: input.reqId, + currentVersion: number(device, 'current_version'), + acceptedAs: input.acceptedAs, + renewed, + resumeExpiresAt, + ...(optionalNumber(device, 'grace_expires_at') === undefined + ? {} + : { graceExpiresAt: optionalNumber(device, 'grace_expires_at') }) + } + } + + // Guarded by the row just read, so a revoke or install that committed since leaves it alone; + // the answer then keeps the expiry that was read, which the database still holds. + private async renewResume( + identity: RelayIdentity & { relayDeviceId: string }, + device: SqlRow, + now: number + ): Promise { + const renewedTo = now + RELAY_PROTOCOL_LIMITS.resumeTtlMs + const rows = await this.database.query( + `UPDATE relay_devices SET current_expires_at = ?, updated_at = ? + WHERE user_id = ? AND relay_host_id = ? AND relay_device_id = ? + AND current_version = ? AND current_hash = ? AND revoked_at IS NULL + AND current_expires_at < ? + RETURNING current_expires_at`, + [ + renewedTo, now, identity.userId, identity.relayHostId, identity.relayDeviceId, + number(device, 'current_version'), string(device, 'current_hash'), + now + RELAY_PROTOCOL_LIMITS.resumeTtlMs - RESUME_RENEWAL_INTERVAL_MS + ] + ) + return rows[0] ? number(rows[0], 'current_expires_at') : number(device, 'current_expires_at') + } + async deactivateBasis(basisConnId: string): Promise { await this.database.query(`UPDATE relay_connection_bases SET active = ? WHERE basis_conn_id = ?`, [ 0, @@ -580,25 +699,9 @@ export class RelayCredentialStore { relayHostId: string, token: string ): Promise<{ userId: string; relayDeviceId: string } | null> { - const tokenHash = hashCredential(token) - const rows = await this.database.query( - `SELECT * FROM relay_devices - WHERE relay_host_id = ? AND (current_hash = ? OR grace_hash = ?)`, - [relayHostId, tokenHash, tokenHash] - ) - const device = rows[0] - if (!device || optionalNumber(device, 'revoked_at') !== undefined) return null - const now = this.now() - const currentValid = - equalHash(string(device, 'current_hash'), tokenHash) && - number(device, 'current_expires_at') > now - const graceHash = device.grace_hash - const graceValid = - typeof graceHash === 'string' && - equalHash(graceHash, tokenHash) && - (optionalNumber(device, 'grace_expires_at') ?? 0) > now - return currentValid || graceValid - ? { userId: string(device, 'user_id'), relayDeviceId: string(device, 'relay_device_id') } + const found = await this.lookupResume(relayHostId, token) + return found.outcome === 'resume' + ? { userId: found.reservation.userId, relayDeviceId: found.reservation.relayDeviceId } : null } diff --git a/cloud/apps/relay/src/floating-promise-census.test.ts b/cloud/apps/relay/src/floating-promise-census.test.ts index fcc056a54462..10574467484d 100644 --- a/cloud/apps/relay/src/floating-promise-census.test.ts +++ b/cloud/apps/relay/src/floating-promise-census.test.ts @@ -15,6 +15,7 @@ const NEVER_REJECTS = new Map([ ['cell-reserve-client.ts#send', 'one try/catch around the request; resolves every item'], ['cell-reserve-client.ts#demote', 'one try/catch around the request'], ['reserve-ledger-writer.ts#flush', 'one try/catch around the write; retries with backoff'], + ['resume-confirmed-audit-buffer.ts#flush', 'catches the insert; a failed batch goes back to the buffer'], ['control-renewal-batch.ts#flush', 'rejects the waiters, never itself'], ['regional-rehome-worker.ts#run', 'one try/catch around the whole poll'], ['relay-control-flag-channel.ts#poll', 'one catch on the whole read'] diff --git a/cloud/apps/relay/src/host-session-client-accept.test.ts b/cloud/apps/relay/src/host-session-client-accept.test.ts index 7806e8d25b37..7448631383a2 100644 --- a/cloud/apps/relay/src/host-session-client-accept.test.ts +++ b/cloud/apps/relay/src/host-session-client-accept.test.ts @@ -114,7 +114,9 @@ function harness(options: { random?: () => number; now?: () => number } = {}) { recordSql: vi.fn(), recordClientAcceptAbandoned: vi.fn(), recordClientAcceptCompleted: vi.fn(), - recordControlRtt: vi.fn() + recordControlRtt: vi.fn(), + recordPhoneRejection: vi.fn(), + recordPhoneAttachDeadline: vi.fn() } satisfies RelayRuntimeObserver const registry = new HostSessionRegistry( config, @@ -369,6 +371,42 @@ describe('client accept abandoned mid-DB-phase', () => { } }) + it('logs and counts an attach deadline with what the cell knew of the control', async () => { + let now = 1_700_000_000_000 + const h = harness({ now: () => now }) + await activeHost(h) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const client = new FakeSocket() + await h.registry.acceptClient(client as unknown as WebSocket, identity.relayHostId, 'credential') + now += RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs + vi.advanceTimersByTime(RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs) + // The phone-only reason; relay-hello itself is unchanged for strict old parsers. + expect(client.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE, 'attach-timeout') + expect(client.send).toHaveBeenCalledWith(JSON.stringify({ type: 'relay-hello', ok: false, code: 4404 })) + expect(h.observer.recordPhoneAttachDeadline).toHaveBeenCalledOnce() + expect(h.observer.recordPhoneRejection).toHaveBeenCalledWith(RELAY_CLOSE_CODE.HOST_OFFLINE) + const lines = warn.mock.calls + .map(([line]) => String(line)) + .filter((line) => line.includes('orca_relay_attach_deadline_offline')) + expect(lines.map((line) => JSON.parse(line))).toEqual([ + { + event: 'orca_relay_attach_deadline_offline', + relayHostIdDigest: expect.any(String), + credentialKind: 'resume', + msSinceLastPong: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs, + pingOutstanding: false, + controlAgeMs: RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs, + controlRttMs: null, + pong: 'fresh', + cellId: config.cellId + } + ]) + expect(lines[0]).not.toContain(identity.relayHostId) + warn.mockRestore() + h.registry.drain(0) + vi.advanceTimersByTime(0) + }) + it('still opens the connection when the phone is holding on', async () => { const h = harness() const control = await activeHost(h) @@ -437,7 +475,8 @@ describe('successful client accept timing', () => { expect(ownerProbe).toHaveBeenCalledOnce() expect(h.observer.recordClientAcceptCompleted).toHaveBeenCalledWith({ totalMs: 49, - stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 } + stageMs: { assignment: 5, credential: 7, activity: 11, attach: 23, basis: 3 }, + phonePath: 'db' }) const line = log.mock.calls .map((call) => String(call[0])) diff --git a/cloud/apps/relay/src/host-session-registry.test.ts b/cloud/apps/relay/src/host-session-registry.test.ts index d58899e84b61..0db469ad457e 100644 --- a/cloud/apps/relay/src/host-session-registry.test.ts +++ b/cloud/apps/relay/src/host-session-registry.test.ts @@ -7,11 +7,11 @@ import { RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME, RELAY_PROTOCOL_LIMITS } from '@orca-cloud/relay-contract' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest' import type WebSocket from 'ws' import type { RelayAssignmentStore } from './assignment-store.js' import type { RelayConfig } from './config.js' -import type { RelayCredentialStore } from './credential-store.js' +import { RelayStoreError, type RelayCredentialStore } from './credential-store.js' import { CONTROL_RENEWAL_BATCH_INTERVAL_MS } from './control-renewal-batch.js' import { CONTROL_RENEWAL_STATEMENT_OUTCOMES, @@ -130,6 +130,7 @@ function createRegistry( observer: { recordAuth: ReturnType recordControlClose: ReturnType + recordPostProofClose: ReturnType recordSpliceClose: ReturnType } } { @@ -177,6 +178,7 @@ function createRegistry( recordReconnect: vi.fn(), recordSql: vi.fn(), recordControlClose: vi.fn(), + recordPostProofClose: vi.fn(), recordSpliceClose: vi.fn() } satisfies RelayRuntimeObserver const registry = new HostSessionRegistry( @@ -2038,3 +2040,234 @@ describe('paced drain', () => { expect(sockets.map((socket) => socket.send.mock.calls.length)).toEqual(sendsAfterEmergency) }) }) + +describe('controls closed after host proof but before wiring', () => { + let warn: MockInstance + beforeEach(() => { + vi.useFakeTimers() + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + }) + afterEach(() => { + warn.mockRestore() + vi.clearAllTimers() + vi.useRealTimers() + }) + const refusedLines = (): string[] => + warn.mock.calls + .map((call) => String(call[0])) + .filter((line) => line.includes('control refused after proof')) + + function expectRefusal( + observer: { recordPostProofClose: ReturnType }, + code: number | null, + reason: string, + cause?: string + ): void { + expect(observer.recordPostProofClose).toHaveBeenCalledWith(reason) + const [line] = refusedLines() + expect(line).toContain(`host=${relayHostLogDigest(identity.relayHostId)} gen=1 app="1.4.173"`) + expect(line).toContain(`code=${code ?? 'none'} reason=${JSON.stringify(reason)}`) + if (cause) expect(line).toContain(`cause=${cause}`) + expect(line).not.toContain(identity.relayHostId) + expect(line).not.toContain(identity.sub) + } + + it('logs an idle cutover refusal', async () => { + const h = createRegistry(vi.fn()) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: beginIdleWork only checks the host key's presence. + const idle = h.registry as unknown as { idleAttempts: Map } + idle.idleAttempts.set(identity.relayHostId, {}) + const socket = new FakeSocket() + await h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + expect(socket.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.WRONG_CELL, + 'idle cutover in progress' + ) + expectRefusal(h.observer, RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress') + }) + + it('logs a stalled activation queue once', async () => { + const stalled = deferred() + const h = createRegistry( + vi.fn().mockReturnValueOnce(stalled.promise) + ) + const first = h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1) + const waiting = new FakeSocket() + const second = h.activate(waiting as unknown as WebSocket, identity, null, 1, false, 1) + await Promise.resolve() + await Promise.resolve() + vi.advanceTimersByTime(30_000) + expectRefusal(h.observer, RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled') + stalled.resolve('control:production-gce-c3:1') + await Promise.all([first, second]) + expect(h.observer.recordPostProofClose).toHaveBeenCalledOnce() + }) + + it('logs a superseded activation', async () => { + const h = createRegistry(vi.fn()) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: only compared by identity against the empty session map. + const stale = {} as HostSession + await h.activate(new FakeSocket() as unknown as WebSocket, identity, stale, 1, false, 1) + expectRefusal(h.observer, RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded') + }) + + it.each([ + { error: new Error('wrong_assignment'), cause: 'wrong_assignment' }, + { error: Object.assign(new Error('timeout'), { code: '57014' }), cause: 'database_transient' }, + // Assembled at runtime so the secret scanner does not read the fixture as a real URL. + { + error: new Error(`connect ${['postgres', '//user:secret@db/relay'].join(':')}`), + cause: 'error' + } + ])('logs a failed host-proof assignment check as $cause', async ({ error, cause }) => { + const h = createRegistry( + vi.fn().mockRejectedValueOnce(error) + ) + await h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1) + expectRefusal( + h.observer, + RELAY_CLOSE_CODE.WRONG_CELL, + 'assignment changed during host proof', + cause + ) + expect(refusedLines()[0]).not.toContain('secret') + }) + + it('logs a drain that lands during activation', async () => { + const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1')) + h.registry.drain(100) + await h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1) + expectRefusal(h.observer, RELAY_CLOSE_CODE.DRAINING, 'relay draining') + }) + + it('logs a host that hangs up during activation, with no cell close code', async () => { + const blocked = deferred() + const h = createRegistry( + vi.fn().mockReturnValueOnce(blocked.promise) + ) + const socket = new FakeSocket() + const activation = h.activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + await vi.advanceTimersByTimeAsync(0) + socket.close() + blocked.resolve('control:production-gce-c3:1') + await activation + expectRefusal(h.observer, null, 'host closed during activation') + }) + + it('logs a host that hangs up while queued, with no cell close code', async () => { + const stalled = deferred() + const h = createRegistry( + vi + .fn() + .mockReturnValueOnce(stalled.promise) + .mockResolvedValue('control:production-gce-c3:1') + ) + const first = h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1) + const queued = new FakeSocket() + const second = h.activate(queued as unknown as WebSocket, identity, null, 1, false, 1) + queued.close() + stalled.resolve('control:production-gce-c3:1') + await Promise.all([first, second]) + expectRefusal(h.observer, null, 'host closed while queued') + }) + + it('logs the hang-up, not a stall, when the queued host left before the deadline', async () => { + const stalled = deferred() + const h = createRegistry( + vi.fn().mockReturnValueOnce(stalled.promise) + ) + const first = h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1) + const queued = new FakeSocket() + const second = h.activate(queued as unknown as WebSocket, identity, null, 1, false, 1) + await Promise.resolve() + await Promise.resolve() + queued.readyState = queued.CLOSED + vi.advanceTimersByTime(30_000) + expect(queued.close).toHaveBeenCalledWith( + RELAY_CLOSE_CODE.LIMIT_EXCEEDED, + 'control activation queue stalled' + ) + expectRefusal(h.observer, null, 'host closed while queued') + stalled.resolve('control:production-gce-c3:1') + await Promise.all([first, second]) + }) + + it('does not log a control that was wired', async () => { + const h = createRegistry(vi.fn().mockResolvedValue('control:production-gce-c3:1')) + await h.activate(new FakeSocket() as unknown as WebSocket, identity, null, 1, false, 1) + expect(h.observer.recordPostProofClose).not.toHaveBeenCalled() + expect(refusedLines()).toEqual([]) + }) +}) + +describe('control error codes', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => { + vi.clearAllTimers() + vi.useRealTimers() + }) + + async function revokeFailure(error: Error): Promise<{ sent: string[]; warned: string[] }> { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const activateControl = vi + .fn() + .mockResolvedValue('control:production-gce-c3:1') + const { registry, activate } = createRegistry(activateControl, { + revoke: vi.fn().mockRejectedValue(error) + }) + const socket = new FakeSocket() + await activate(socket as unknown as WebSocket, identity, null, 1, false, 1) + socket.emit( + 'message', + Buffer.from(JSON.stringify({ type: 'device-revoke', reqId: 'revoke-1', relayDeviceId: 'device-1' })), + false + ) + await vi.advanceTimersByTimeAsync(0) + const sent = socket.send.mock.calls.map((call) => String(call[0])) + const warned = warn.mock.calls.map((call) => String(call[0])) + warn.mockRestore() + registry.drain(0) + vi.advanceTimersByTime(0) + return { sent, warned } + } + + function controlErrors(sent: string[]): unknown[] { + return sent + .map((line) => JSON.parse(line) as Record) + .filter((message) => message.type === 'control-error') + } + + it('never sends a database error message, which can carry the connection URL', async () => { + // Assembled at runtime so secret scanners do not flag a fixture URL. + const url = ['postgres', '://relay:', 'fake-pw', '@db.internal:5432/relay'].join('') + const error = Object.assign(new Error(`connect to ${url} failed: Connection terminated`), { + code: '08006' + }) + const { sent, warned } = await revokeFailure(error) + + expect(controlErrors(sent)).toEqual([ + { type: 'control-error', reqId: 'revoke-1', code: 'database_transient' } + ]) + expect(sent.join('\n')).not.toContain('fake-pw') + expect(warned).toContain('[orca-relay] control device-revoke failed: Error: redacted code=08006') + expect(warned.join('\n')).not.toContain('fake-pw') + }) + + it('maps any other unexpected message to a fixed code', async () => { + const { sent } = await revokeFailure(new Error('Unexpected token < in JSON')) + + expect(controlErrors(sent)).toEqual([ + { type: 'control-error', reqId: 'revoke-1', code: 'control_operation_failed' } + ]) + }) + + it.each([ + ['a store code', new RelayStoreError('reject-revoked'), 'reject-revoked'], + ['an invariant slug', new Error('authorization_expired'), 'authorization_expired'] + ])('keeps %s on the wire', async (_kind, error, code) => { + const { sent, warned } = await revokeFailure(error) + + expect(controlErrors(sent)).toEqual([{ type: 'control-error', reqId: 'revoke-1', code }]) + expect(warned.join('\n')).not.toContain('control device-revoke failed') + }) +}) diff --git a/cloud/apps/relay/src/host-session-registry.ts b/cloud/apps/relay/src/host-session-registry.ts index 4bbbf896830f..1ccbabb98e78 100644 --- a/cloud/apps/relay/src/host-session-registry.ts +++ b/cloud/apps/relay/src/host-session-registry.ts @@ -18,10 +18,12 @@ import { RELAY_HOST_CAPABILITY_IDLE_REGIONAL_REHOME, RELAY_PROTOCOL_LIMITS, RELAY_CLOSE_CODE, + RELAY_PHONE_CLOSE_REASON, type IdleRegionalRehomeCommit, type IdleRegionalRehomeDeferReason, type IdleRegionalRehomeResult, type RelayHostCloseReason, + type RelayPhoneCloseReason, type RelayRegion } from '@orca-cloud/relay-contract' import nacl from 'tweetnacl' @@ -35,9 +37,18 @@ import type { CellAdmitMode, CellReserveBook } from './cell-reserve-book.js' import type { DemoteRequest, ReserveOutcome, ReserveRequest } from './cell-reserve-contract.js' import { CellSeatLog, type CellSeatChange, type CellSeatFeedPage } from './cell-seat-log.js' import { isRelayDatabaseTransientError } from './database.js' +import { printableCloseReason } from './control-close-reason-text.js' import { ControlRenewalBatch } from './control-renewal-batch.js' -import { RelayCredentialStore, type CredentialReservation } from './credential-store.js' +import { RelayCredentialStore, RelayStoreError, type CredentialReservation } from './credential-store.js' import { HostCloseReasonMemory } from './host-close-reason-memory.js' +import { MemoryResumeConfirmations } from './memory-resume-confirmations.js' +import { PhoneConnectionLog, type PhoneDatabaseCause } from './phone-connection-log.js' +import { + postProofCloseCause, + PostProofCloseLog, + type PostProofCloseCause +} from './post-proof-close-log.js' +import { describeRelayFailure, relayErrorSlug } from './relay-error-redaction.js' import { relayHostLogDigest } from './relay-host-log-digest.js' import type { RelayTokenClaims } from './relay-token-verifier.js' import { @@ -52,14 +63,6 @@ import type { PendingHostDataReservation } from './relay-connection-ledger.js' import { closeRelayWebSocket } from './relay-websocket-close.js' import { ProcessQueuedByteBudget, wireSplice } from './splice-forwarder.js' -// Peer-supplied close reasons are logged; keep them printable and short. -function printableCloseReason(reason: Buffer | string): string { - return reason - .toString() - .replace(/[^\x20-\x7e]/g, '') - .slice(0, 80) -} - type VerifyRelayToken = (token: string) => Promise type HostState = 'proving' | 'active' | 'orphaned' | 'drain-only' | 'closed' @@ -134,6 +137,8 @@ type PendingConnection = { client: WebSocket attachTimer: ReturnType credentialActivityId: string | null + // `memory`: a resume admitted from the seat, which writes no activity leases. + phonePath: 'db' | 'memory' capacityReservation?: PendingHostDataReservation timing: ClientAcceptTiming } @@ -249,6 +254,8 @@ export type CellReserveAdmission = { directorContact?: () => void // The switch file's flip-back pace, when set. reregisterInFlight?: () => number | undefined + // The switch file's phone path; it acts only while the cell is in reserve mode. + phonePath?: () => 'db' | 'memory' // The pool-pressure shed's condition: a hello memory cannot admit is shed, not queued. databaseShedding: () => boolean book: CellReserveBook @@ -299,6 +306,9 @@ export class HostSessionRegistry { // draining" is not the same question as "this host has been told to leave". private readonly drainSentHosts = new Set() private readonly seatLog: CellSeatLog + private readonly postProofCloses: PostProofCloseLog + private readonly phoneConnections: PhoneConnectionLog + private readonly memoryConfirmations: MemoryResumeConfirmations // Seat key -> the demoted control socket. Its own set, not drainSentHosts: it ends with // that socket (close, rebind or a new join), so a later booking here is a fresh seat. private readonly demotedSeats = new Map() @@ -442,6 +452,11 @@ export class HostSessionRegistry { private readonly reserveAdmission?: CellReserveAdmission ) { this.seatLog = new CellSeatLog(undefined, now) + this.postProofCloses = new PostProofCloseLog(observer) + this.phoneConnections = new PhoneConnectionLog(config.cellId) + this.memoryConfirmations = new MemoryResumeConfirmations(store, now, (error) => + this.recordRenewalFailure(error) + ) } // Reserve mode is only ever a cell's, and only while its applied switch says so. @@ -453,6 +468,12 @@ export class HostSessionRegistry { return this.reserveMode() } + // Step 6: the seat is authoritative only in reserve mode, so a dead-man trip or admitMode=db + // turns the memory phone path off for new phones too. + phonePathEffective(): 'db' | 'memory' { + return this.reserveMode() && this.reserveAdmission?.phonePath?.() === 'memory' ? 'memory' : 'db' + } + // Leaving reserve mode: every control admitted from memory gets the database lease it never // took, while it stays connected. Rescans until none lacks one, so a hello whose proof lands // after the flip is caught too. A control whose row never names this cell is closed (4409). @@ -833,34 +854,47 @@ export class HostSessionRegistry { stageMs[stage] = at - stageCursor stageCursor = at } - if (this.config.role === 'cell') { - // Each lookup is its own pooled round trip; stop between them once the phone - // has left instead of running the rest of the chain for nobody. - let outerIdentity = await this.store.resolveResume(hostId, credential) - if (abandonedByClient('assignment')) return - if (!outerIdentity) { - outerIdentity = await this.store.resolveInviteForMove(hostId, credential) - if (abandonedByClient('assignment')) return - } - const assignment = outerIdentity - ? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId }) - : null - if (!assignment || assignment.cellId !== this.config.cellId) { + // Step 6: a resume is admitted from one device read and the seat, writing no leases. + let memoryReservation: CredentialReservation | null = null + let memoryMissedDevice = false + if (this.config.role === 'cell' && this.phonePathEffective() === 'memory') { + markStage('assignment') + let found: Awaited> + try { + found = await this.store.lookupResume(hostId, credential) + } catch (error) { + // Never the invite path: an unread device is not a missing one. capacityReservation?.release() - this.observer.recordAuth(false) - this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL) + this.observer.recordPhoneAcceptDatabaseError?.() + this.phoneConnections.databaseError(databaseCause(error)) + this.rejectClient(socket, RELAY_CLOSE_CODE.DRAINING) return } - if (abandonedByClient('assignment')) return - } - markStage('assignment') - const reservation = await this.store.reserveCredential(hostId, credential) - if (!reservation) { - capacityReservation?.release() - this.observer.recordAuth(false) - this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL) - return + if (abandonedByClient('credential')) return + // The switch turned off during the read: today's whole path, never a lease-less memory phone. + if (this.phonePathEffective() === 'memory') { + if (found.outcome === 'refused') { + capacityReservation?.release() + this.observer.recordAuth(false) + this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL) + return + } + if (found.outcome === 'resume') memoryReservation = found.reservation + else memoryMissedDevice = true + } } + const reservation = + memoryReservation ?? + (await this.reserveCredentialFromDatabase( + socket, + hostId, + credential, + memoryMissedDevice, + capacityReservation, + abandonedByClient, + markStage + )) + if (!reservation) return this.observer.recordAuth(true) if (abandonedByClient('credential', () => this.failReservationBestEffort(reservation))) return markStage('credential') @@ -879,6 +913,10 @@ export class HostSessionRegistry { session.socket.readyState !== session.socket.OPEN ) { capacityReservation?.release() + if (memoryReservation) { + await this.rejectMissedSeat(socket, reservation, sessionKey) + return + } await this.store.failReservation(reservation) // The only rejection that can name a cause: the host is genuinely absent. // The attach-deadline 4404 below fires while control is still connected. @@ -900,7 +938,7 @@ export class HostSessionRegistry { const connTicket = randomBytes(32).toString('base64url') const identity = { userId: reservation.userId, relayHostId: hostId } const credentialActivityId = - this.config.role === 'cell' + this.config.role === 'cell' && !memoryReservation ? `${reservation.credentialKind === 'invite' ? 'invite' : 'confirmation'}:${connId}` : null if (credentialActivityId) { @@ -945,7 +983,13 @@ export class HostSessionRegistry { capacityReservation?.release() this.failReservationBestEffort(reservation) if (credentialActivityId) this.releaseActivityBestEffort(identity, credentialActivityId) - this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE) + this.recordAttachDeadline(session, reservation) + // OTA phones redial this cell once, quickly: the control is still seated here. + this.rejectClient( + socket, + RELAY_CLOSE_CODE.HOST_OFFLINE, + RELAY_PHONE_CLOSE_REASON.HOST_ATTACH_TIMEOUT + ) }, RELAY_PROTOCOL_LIMITS.hostAttachDeadlineMs) const pending: PendingConnection = { connId, @@ -954,6 +998,7 @@ export class HostSessionRegistry { client: socket, attachTimer, credentialActivityId, + phonePath: memoryReservation ? 'memory' : 'db', capacityReservation, // Attach starts where the activity stage ended, so the conn-open send is // charged to it and no wall-clock gap goes unattributed. @@ -982,6 +1027,74 @@ export class HostSessionRegistry { }) } + // Today's credential path: invites, and every resume while the phone path is db. `deviceMissed`: + // the memory path already found no device row, so only the invite lookup is left. + private async reserveCredentialFromDatabase( + socket: WebSocket, + hostId: string, + credential: string, + deviceMissed: boolean, + capacityReservation: PendingHostDataReservation | undefined, + abandonedByClient: (stage: RelayClientAcceptStage, cleanup?: () => void) => boolean, + markStage: (stage: RelayClientAcceptStage) => void + ): Promise { + if (this.config.role === 'cell') { + // Each lookup is its own pooled round trip; stop between them once the phone + // has left instead of running the rest of the chain for nobody. + let outerIdentity = deviceMissed ? null : await this.store.resolveResume(hostId, credential) + if (abandonedByClient('assignment')) return null + if (!outerIdentity) { + outerIdentity = await this.store.resolveInviteForMove(hostId, credential) + if (abandonedByClient('assignment')) return null + } + const assignment = outerIdentity + ? await this.assignments.resolve({ userId: outerIdentity.userId, relayHostId: hostId }) + : null + if (!assignment || assignment.cellId !== this.config.cellId) { + capacityReservation?.release() + this.observer.recordAuth(false) + this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL) + return null + } + if (abandonedByClient('assignment')) return null + } + markStage('assignment') + const reservation = await this.store.reserveCredential(hostId, credential) + if (!reservation) { + capacityReservation?.release() + this.observer.recordAuth(false) + this.rejectClient(socket, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL) + return null + } + return reservation + } + + // No seat for a memory resume: one assignment read tells "moved" from "offline". A row that + // names this cell may be a lost ledger write (stale up to the 10 min reconcile), so it is logged. + private async rejectMissedSeat( + socket: WebSocket, + reservation: CredentialReservation, + sessionKey: string + ): Promise { + let elsewhere: boolean + try { + const row = await this.assignments.resolve({ + userId: reservation.userId, + relayHostId: reservation.relayHostId + }) + elsewhere = row !== null && row.cellId !== this.config.cellId + if (row?.cellId === this.config.cellId) this.phoneConnections.missRowNamesHere(reservation.relayHostId) + } catch (error) { + this.phoneConnections.missReadError(databaseCause(error)) + elsewhere = true + } + if (elsewhere) { + this.rejectClient(socket, RELAY_CLOSE_CODE.WRONG_CELL) + return + } + this.rejectClient(socket, RELAY_CLOSE_CODE.HOST_OFFLINE, this.hostCloseReasons.read(sessionKey)) + } + async acceptHostData( socket: WebSocket, connId: string, @@ -1041,7 +1154,8 @@ export class HostSessionRegistry { userId: pending.reservation.userId, relayHostId: pending.reservation.relayHostId } - const spliceActivityId = this.config.role === 'cell' ? `splice:${connId}` : null + const spliceActivityId = + this.config.role === 'cell' && pending.phonePath === 'db' ? `splice:${connId}` : null try { if (spliceActivityId) { await this.assignments.acquireActivity(identity, { @@ -1050,12 +1164,16 @@ export class HostSessionRegistry { cellId: this.config.cellId }) } - await this.store.recordConnectionBasis({ - ...pending.reservation, - basisConnId: connId, - owningControlGeneration: session.generation, - deadline: basisDeadline - }) + if (pending.phonePath === 'memory') { + this.openMemoryBasis(pending, connId, session.generation, basisDeadline) + } else { + await this.store.recordConnectionBasis({ + ...pending.reservation, + basisConnId: connId, + owningControlGeneration: session.generation, + deadline: basisDeadline + }) + } } catch { session.activeConnIds.delete(connId) await this.store.failReservation(pending.reservation) @@ -1078,7 +1196,7 @@ export class HostSessionRegistry { ) { session.activeConnIds.delete(connId) pending.capacityReservation?.release() - this.deactivateBasisBestEffort(connId) + this.closeBasis(pending, connId) this.failReservationBestEffort(pending.reservation) if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId) if (pending.credentialActivityId) { @@ -1095,7 +1213,7 @@ export class HostSessionRegistry { onClose: () => { session.activeConnIds.delete(connId) session.activeSplices.delete(connId) - this.deactivateBasisBestEffort(connId) + this.closeBasis(pending, connId) if (spliceActivityId) this.releaseActivityBestEffort(identity, spliceActivityId) if (pending.credentialActivityId) { this.releaseActivityBestEffort(identity, pending.credentialActivityId) @@ -1161,13 +1279,14 @@ export class HostSessionRegistry { basis: nonNegativeMs(helloAt - attachedAt) } const totalMs = nonNegativeMs(helloAt - pending.timing.startedAt) - this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs }) + this.observer.recordClientAcceptCompleted?.({ totalMs, stageMs, phonePath: pending.phonePath }) console.log( JSON.stringify({ event: 'orca_relay_client_accept_completed', ...this.logIdentity(), ...this.sessionPlacementLogFields(session), credentialKind: pending.reservation.credentialKind, + phonePath: pending.phonePath, stageMs, totalMs, relayHostIdDigest: relayHostLogDigest(session.relayHostId) @@ -1627,9 +1746,15 @@ export class HostSessionRegistry { // Present when the hello was admitted from memory: no database registration. reserved?: { reservedBy?: string } ): Promise { + const refused = { relayHostId: identity.relayHostId, generation, appVersion } const release = this.beginIdleWork(identity.relayHostId) if (!release) { - socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'idle cutover in progress') + this.refuseProvenControl( + socket, + refused, + RELAY_CLOSE_CODE.WRONG_CELL, + 'idle cutover in progress' + ) return Promise.resolve() } const key = this.key(identity.sub, identity.relayHostId) @@ -1639,16 +1764,36 @@ export class HostSessionRegistry { let queueWaitExpired = false const queueWaitTimer = setTimeout(() => { queueWaitExpired = true - socket.close(RELAY_CLOSE_CODE.LIMIT_EXCEEDED, 'control activation queue stalled') + this.refuseProvenControl( + socket, + refused, + RELAY_CLOSE_CODE.LIMIT_EXCEEDED, + 'control activation queue stalled', + undefined, + 'host closed while queued' + ) }, ACTIVATION_QUEUE_WAIT_MS) queueWaitTimer.unref?.() const activation = previous .catch(() => undefined) .then(async () => { clearTimeout(queueWaitTimer) - if (queueWaitExpired || socket.readyState !== socket.OPEN) return + if (queueWaitExpired) return + if (socket.readyState !== socket.OPEN) { + this.postProofCloses.record({ + ...refused, + code: null, + reason: 'host closed while queued' + }) + return + } if ((this.sessions.get(key) ?? null) !== existing) { - socket.close(RELAY_CLOSE_CODE.PEER_DROPPED, 'control activation superseded') + this.refuseProvenControl( + socket, + refused, + RELAY_CLOSE_CODE.PEER_DROPPED, + 'control activation superseded' + ) return } await this.activateCurrent( @@ -1683,6 +1828,7 @@ export class HostSessionRegistry { connectionInclusionWatermark?: number, reserved?: { reservedBy?: string } ): Promise { + const refused = { relayHostId: identity.relayHostId, generation, appVersion } let controlActivityId: string | null = null // Admitted from memory, but the cell left reserve mode before the proof landed: lease now. if (this.config.role === 'cell' && !(reserved && this.reserveMode())) { @@ -1704,7 +1850,7 @@ export class HostSessionRegistry { { userId: identity.sub, relayHostId: identity.relayHostId }, { cellId: this.config.cellId, assignmentEpoch } ) - } catch { + } catch (error) { // A failure after activateControl succeeded must not strand the // acquired control activity until lease expiry. if (controlActivityId) { @@ -1713,7 +1859,13 @@ export class HostSessionRegistry { controlActivityId ) } - socket.close(RELAY_CLOSE_CODE.WRONG_CELL, 'assignment changed during host proof') + this.refuseProvenControl( + socket, + refused, + RELAY_CLOSE_CODE.WRONG_CELL, + 'assignment changed during host proof', + postProofCloseCause(error, isRelayDatabaseTransientError) + ) return } } @@ -1725,7 +1877,13 @@ export class HostSessionRegistry { ) } if (socket.readyState === socket.OPEN) { - socket.close(RELAY_CLOSE_CODE.DRAINING, 'relay draining') + this.refuseProvenControl(socket, refused, RELAY_CLOSE_CODE.DRAINING, 'relay draining') + } else { + this.postProofCloses.record({ + ...refused, + code: null, + reason: 'host closed during activation' + }) } return } @@ -1828,6 +1986,22 @@ export class HostSessionRegistry { this.appendSeatChange(session, 'join', undefined, reserved?.reservedBy) } + private refuseProvenControl( + socket: WebSocket, + refused: { relayHostId: string; generation: number; appVersion: string }, + code: number, + reason: string, + cause?: PostProofCloseCause, + hungUpReason = 'host closed during activation' + ): void { + // A host that already hung up got no close frame from us; log what happened instead. + const open = socket.readyState === socket.OPEN + socket.close(code, reason) + this.postProofCloses.record( + open ? { ...refused, code, reason, cause } : { ...refused, code: null, reason: hungUpReason } + ) + } + seatFeed(sinceSeq: number | null): CellSeatFeedPage { return { ...this.seatLog.read(sinceSeq), seats: this.seatLog.seatCount() } } @@ -2322,12 +2496,22 @@ export class HostSessionRegistry { } if (type === 'device-resume-confirm') { const request = DeviceResumeConfirmSchema.parse(payload(raw, type)) - const result = await this.store.confirmResume({ - ...identity, - ...request, - owningControlGeneration: session.generation - }) - await this.assignments.releaseActivity(identity, `confirmation:${request.basisConnId}`) + const confirm = { ...identity, ...request, owningControlGeneration: session.generation } + const phonePath = this.memoryConfirmations.owns(confirm) ? 'memory' : 'db' + let result + try { + result = + phonePath === 'memory' + ? await this.memoryConfirmations.confirm(confirm) + : await this.store.confirmResume(confirm) + } catch (error) { + this.recordConfirmFailure(error, phonePath) + throw error + } + // After a committed confirm: a failed release is not a failed confirm. + if (phonePath === 'db') { + await this.assignments.releaseActivity(identity, `confirmation:${request.basisConnId}`) + } send(session.socket, 'device-resume-confirmed', result) return } @@ -2345,28 +2529,51 @@ export class HostSessionRegistry { ? String(candidate.reqId) : undefined })() - this.sendControlError( - session, - reqId, - error instanceof Error ? error.message : 'control_operation_failed' - ) + this.sendControlError(session, reqId, controlErrorCode(type, error)) } } + private recordRenewalFailure(error: unknown): void { + this.observer.recordResumeRenewalFailure?.() + this.phoneConnections.renewalFailed(databaseCause(error)) + } + + // Never the raw message as a key: pg errors can carry the database URL. + private recordConfirmFailure(error: unknown, phonePath: 'db' | 'memory'): void { + const code = error instanceof RelayStoreError ? error.code : databaseCause(error) + this.observer.recordResumeConfirmFailure?.(code) + this.phoneConnections.confirmFailed(code, phonePath) + } + + private recordAttachDeadline(session: HostSession, reservation: CredentialReservation): void { + const now = this.now() + const msSinceLastPong = now - session.lastPongAt + this.observer.recordPhoneAttachDeadline?.() + this.phoneConnections.attachDeadline(session.relayHostId, { + credentialKind: reservation.credentialKind, + msSinceLastPong, + pingOutstanding: session.pendingPingAt !== null, + controlAgeMs: now - session.controlWiredAt, + controlRttMs: session.controlRttSamplesMs.at(-1) ?? null, + pong: msSinceLastPong > RELAY_PROTOCOL_LIMITS.controlPingIntervalMs ? 'stale' : 'fresh' + }) + } + private sendControlError(session: HostSession, reqId: string | undefined, code: string): void { if (session.socket) send(session.socket, 'control-error', { ...(reqId ? { reqId } : {}), code }) } - // hostCloseReason rides the WebSocket close reason, never relay-hello: every - // shipped phone parses relay-hello with a strict schema that rejects an - // unknown key, and none of them read the close reason at all. + // closeReason rides the WebSocket close reason, never relay-hello: every shipped + // phone parses relay-hello with a strict schema that rejects an unknown key, and + // ignores a close reason it does not know. private rejectClient( socket: WebSocket, code: number, - hostCloseReason?: RelayHostCloseReason | null + closeReason?: RelayHostCloseReason | RelayPhoneCloseReason | null ): void { + this.observer.recordPhoneRejection?.(code) send(socket, 'relay-hello', { ok: false, code }) - closeRelayWebSocket(socket, code, hostCloseReason ?? 'relay connection rejected') + closeRelayWebSocket(socket, code, closeReason ?? 'relay connection rejected') } private releaseControlActivity(session: HostSession): void { @@ -2393,6 +2600,34 @@ export class HostSessionRegistry { }) } + // A memory resume's basis is the cell's own record; every other basis is a Postgres row. + private openMemoryBasis( + pending: PendingConnection, + connId: string, + owningControlGeneration: number, + deadline: number + ): void { + const { reservation } = pending + if (reservation.acceptedCredentialVersion === undefined || reservation.acceptedAs === undefined) { + throw new Error('memory basis needs an accepted resume credential') + } + this.memoryConfirmations.open({ + basisConnId: connId, + userId: reservation.userId, + relayHostId: reservation.relayHostId, + relayDeviceId: reservation.relayDeviceId, + acceptedCredentialVersion: reservation.acceptedCredentialVersion, + acceptedAs: reservation.acceptedAs, + owningControlGeneration, + deadline + }) + } + + private closeBasis(pending: PendingConnection, connId: string): void { + if (pending.phonePath === 'memory') this.memoryConfirmations.close(connId) + else this.deactivateBasisBestEffort(connId) + } + private deactivateBasisBestEffort(connId: string): void { void this.store.deactivateBasis(connId).catch(() => { console.warn('[orca-relay] basis deactivation deferred to credential cleanup') @@ -2401,3 +2636,16 @@ export class HostSessionRegistry { } export type RelayIdentityKey = { userId: string; relayHostId: string } + +// Intentional throws are store codes or slugs; anything else could carry the database URL. +function controlErrorCode(type: string, error: unknown): string { + if (error instanceof RelayStoreError) return error.code + const slug = relayErrorSlug(error) + if (slug) return slug + console.warn(`[orca-relay] control ${type} failed: ${describeRelayFailure(error)}`) + return isRelayDatabaseTransientError(error) ? 'database_transient' : 'control_operation_failed' +} + +function databaseCause(error: unknown): PhoneDatabaseCause { + return isRelayDatabaseTransientError(error) ? 'database_transient' : 'error' +} diff --git a/cloud/apps/relay/src/idle-regional-rehome-store.test.ts b/cloud/apps/relay/src/idle-regional-rehome-store.test.ts index b83cd57f92f9..ce84bf78e531 100644 --- a/cloud/apps/relay/src/idle-regional-rehome-store.test.ts +++ b/cloud/apps/relay/src/idle-regional-rehome-store.test.ts @@ -217,6 +217,15 @@ describe('constrained idle regional assignment transaction', () => { expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1) }) + // sourceRequestUnits reads a reserve cell's missing phone leases as zero units; selection must drop it. + it.each(['source', 'target'])('drops every candidate while the %s cell is recorded as reserve', async (cellId) => { + const { store, safety } = await setup() + await store.setCellAdmitMode(cellId, 'reserve') + expect(await store.selectIdleRegionalRehomeCandidates(safety)).toEqual([]) + await store.setCellAdmitMode(cellId, 'db') + expect(await store.selectIdleRegionalRehomeCandidates(safety)).toHaveLength(1) + }) + it('leaves the candidate page offset untouched across a closed dispatch budget', async () => { const { store, database, safety } = await setup() const first = await store.selectIdleRegionalRehomeCandidates(safety) diff --git a/cloud/apps/relay/src/memory-resume-confirmations.ts b/cloud/apps/relay/src/memory-resume-confirmations.ts new file mode 100644 index 000000000000..c2de7a7be7a8 --- /dev/null +++ b/cloud/apps/relay/src/memory-resume-confirmations.ts @@ -0,0 +1,98 @@ +import type { DeviceResumeConfirmed } from '@orca-cloud/relay-contract' +import { RelayStoreError, type RelayCredentialStore, type RelayIdentity } from './credential-store.js' + +// Step 6: what a memory cell keeps instead of relay_connection_bases and relay_confirm_results. +// Both live and die with this process; a restart takes the splices with it. + +export type MemoryConnectionBasis = RelayIdentity & { + basisConnId: string + relayDeviceId: string + acceptedCredentialVersion: number + acceptedAs: 'current' | 'grace' + owningControlGeneration: number + deadline: number +} + +export type ResumeConfirmRequest = RelayIdentity & { + reqId: string + basisConnId: string + owningControlGeneration: number +} + +// A retried confirm of the same request answers from here, as relay_confirm_results does. +export const MEMORY_CONFIRM_RESULT_RETENTION_MS = 10 * 60 * 1000 + +type StoredResult = { basisConnId: string; result: DeviceResumeConfirmed; at: number } + +function resultKey(request: RelayIdentity & { reqId: string }): string { + return `${request.userId}\u0000${request.relayHostId}\u0000${request.reqId}` +} + +export class MemoryResumeConfirmations { + private readonly bases = new Map() + // Insertion order is age order, so expiry walks from the front. + private readonly results = new Map() + + constructor( + private readonly store: RelayCredentialStore, + private readonly now: () => number = Date.now, + private readonly onRenewalFailure?: (error: unknown) => void + ) {} + + open(basis: MemoryConnectionBasis): void { + this.bases.set(basis.basisConnId, basis) + } + + close(basisConnId: string): void { + this.bases.delete(basisConnId) + } + + // Routed by where the basis lives, not by the switch, so a flip between accept and confirm + // neither fails the confirm nor sends it to a database that holds no basis for it. + owns(request: ResumeConfirmRequest): boolean { + this.expireResults() + return this.bases.has(request.basisConnId) || this.results.has(resultKey(request)) + } + + async confirm(request: ResumeConfirmRequest): Promise { + this.expireResults() + const key = resultKey(request) + const prior = this.results.get(key) + if (prior) { + if (prior.basisConnId !== request.basisConnId) throw new RelayStoreError('confirmation_tuple_mismatch') + return prior.result + } + const basis = this.bases.get(request.basisConnId) + if ( + !basis || + basis.userId !== request.userId || + basis.relayHostId !== request.relayHostId || + basis.owningControlGeneration !== request.owningControlGeneration || + basis.deadline < this.now() + ) { + throw new RelayStoreError('confirmation_not_active') + } + const result = await this.store.confirmMemoryResume( + { + userId: basis.userId, + relayHostId: basis.relayHostId, + reqId: request.reqId, + basisConnId: basis.basisConnId, + relayDeviceId: basis.relayDeviceId, + acceptedCredentialVersion: basis.acceptedCredentialVersion, + acceptedAs: basis.acceptedAs + }, + { onRenewalFailure: this.onRenewalFailure } + ) + this.results.set(key, { basisConnId: basis.basisConnId, result, at: this.now() }) + return result + } + + private expireResults(): void { + const cutoff = this.now() - MEMORY_CONFIRM_RESULT_RETENTION_MS + for (const [key, stored] of this.results) { + if (stored.at > cutoff) break + this.results.delete(key) + } + } +} diff --git a/cloud/apps/relay/src/phone-connection-log.ts b/cloud/apps/relay/src/phone-connection-log.ts new file mode 100644 index 000000000000..52b8c3ace6fa --- /dev/null +++ b/cloud/apps/relay/src/phone-connection-log.ts @@ -0,0 +1,86 @@ +import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js' +import { relayHostLogDigest } from './relay-host-log-digest.js' + +// Phone-connection anomalies that repeat per dial (a stuck database, a lost ledger write, a +// half-open control): per kind, one line opens the window and one closing line carries the rest. +export const PHONE_CONNECTION_LOG_WINDOW_MS = 60_000 + +export type PhoneDatabaseCause = 'database_transient' | 'error' + +// What the cell knew of the desktop's control when a phone's attach deadline passed: a stale +// pong is a half-open control, a fresh one a slow or failing desktop data dial. +export type AttachDeadlineEvidence = { + credentialKind: 'invite' | 'resume' + msSinceLastPong: number + pingOutstanding: boolean + controlAgeMs: number + controlRttMs: number | null + pong: 'stale' | 'fresh' +} + +type PhoneConnectionAnomaly = + | { event: 'orca_relay_client_accept_db_error'; cause: PhoneDatabaseCause } + | { event: 'orca_relay_client_miss_row_names_here'; relayHostIdDigest: string } + | { event: 'orca_relay_client_miss_read_error'; cause: PhoneDatabaseCause } + | { event: 'orca_relay_resume_renewal_failed'; cause: PhoneDatabaseCause } + | ({ event: 'orca_relay_attach_deadline_offline'; relayHostIdDigest: string } & AttachDeadlineEvidence) + | { event: 'orca_relay_resume_confirm_failed'; code: string; phonePath: 'db' | 'memory' } + +export class PhoneConnectionLog { + private readonly window: AssignmentRejectionLogWindow + + constructor( + private readonly cellId: string, + private readonly warn: (line: string) => void = (line) => console.warn(line), + schedule?: (callback: () => void, delayMs: number) => () => void + ) { + this.window = new AssignmentRejectionLogWindow({ + windowMs: PHONE_CONNECTION_LOG_WINDOW_MS, + onWindowClosed: ({ suppressed, sample }) => this.write(sample, suppressed), + schedule + }) + } + + // The one device read failed; the phone was closed 4503 and never took the invite path. + databaseError(cause: PhoneDatabaseCause): void { + this.record(cause, { event: 'orca_relay_client_accept_db_error', cause }) + } + + // No seat, yet the row names this cell: the 4404 may be stale (a lost reserve ledger write). + missRowNamesHere(relayHostId: string): void { + this.record('', { + event: 'orca_relay_client_miss_row_names_here', + relayHostIdDigest: relayHostLogDigest(relayHostId) + }) + } + + // The no-seat assignment read failed; the phone was closed 4409 to re-resolve. + missReadError(cause: PhoneDatabaseCause): void { + this.record(cause, { event: 'orca_relay_client_miss_read_error', cause }) + } + + // A due renewal write failed; the confirm still answered the stored expiry. + renewalFailed(cause: PhoneDatabaseCause): void { + this.record(cause, { event: 'orca_relay_resume_renewal_failed', cause }) + } + + attachDeadline(relayHostId: string, evidence: AttachDeadlineEvidence): void { + this.record(`${evidence.credentialKind}:${evidence.pong}`, { + event: 'orca_relay_attach_deadline_offline', + relayHostIdDigest: relayHostLogDigest(relayHostId), + ...evidence + }) + } + + confirmFailed(code: string, phonePath: 'db' | 'memory'): void { + this.record(`${code}:${phonePath}`, { event: 'orca_relay_resume_confirm_failed', code, phonePath }) + } + + private record(kind: string, anomaly: PhoneConnectionAnomaly): void { + if (this.window.admit(`${anomaly.event}:${kind}`, anomaly)) this.write(anomaly, 0) + } + + private write(anomaly: PhoneConnectionAnomaly, suppressed: number): void { + this.warn(JSON.stringify({ ...anomaly, cellId: this.cellId, ...(suppressed > 0 ? { suppressed } : {}) })) + } +} diff --git a/cloud/apps/relay/src/post-proof-close-log.test.ts b/cloud/apps/relay/src/post-proof-close-log.test.ts new file mode 100644 index 000000000000..6113c8d72a8a --- /dev/null +++ b/cloud/apps/relay/src/post-proof-close-log.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' +import { POST_PROOF_CLOSE_LOG_WINDOW_MS, PostProofCloseLog } from './post-proof-close-log.js' + +const close = { + relayHostId: 'abcdefghijklmnop', + generation: 7, + appVersion: '1.4.150', + code: 4409, + reason: 'assignment changed during host proof', + cause: 'wrong_assignment' as const +} + +function createLog() { + const lines: string[] = [] + const scheduled: Array<{ callback: () => void; delayMs: number }> = [] + const observer = { + recordAuth: vi.fn(), + recordForwardedBytes: vi.fn(), + recordHttp: vi.fn(), + recordReconnect: vi.fn(), + recordSql: vi.fn(), + recordPostProofClose: vi.fn() + } + const log = new PostProofCloseLog( + observer, + (line) => lines.push(line), + (callback, delayMs) => { + scheduled.push({ callback, delayMs }) + return () => undefined + } + ) + return { log, lines, scheduled, observer } +} + +describe('post-proof close log', () => { + it('counts every close but writes one line per reason and cause per window', () => { + const { log, lines, scheduled, observer } = createLog() + for (let index = 0; index < 1_000; index++) log.record({ ...close, generation: index }) + log.record({ ...close, cause: 'database_transient' }) + log.record({ ...close, code: 4400, reason: 'relay draining', cause: undefined }) + + expect(observer.recordPostProofClose).toHaveBeenCalledTimes(1_002) + expect(lines).toHaveLength(3) + expect(scheduled.map((entry) => entry.delayMs)).toEqual( + Array.from({ length: 3 }, () => POST_PROOF_CLOSE_LOG_WINDOW_MS) + ) + + for (const { callback } of scheduled) callback() + // Only the window that held closes back writes a closing line, naming the latest. + expect(lines).toHaveLength(4) + expect(lines[3]).toContain('gen=999') + expect(lines[3]).toContain('suppressed=999') + }) + + it('names the host by digest and keeps peer-supplied versions printable', () => { + const { log, lines } = createLog() + log.record({ ...close, appVersion: '1.4.150\n\u0007injected' }) + expect(lines[0]).toMatch( + /^\[orca-relay\] control refused after proof host=\S+ gen=7 app="1\.4\.150injected" code=4409 reason="assignment changed during host proof" cause=wrong_assignment$/ + ) + expect(lines[0]).not.toContain(close.relayHostId) + }) +}) diff --git a/cloud/apps/relay/src/post-proof-close-log.ts b/cloud/apps/relay/src/post-proof-close-log.ts new file mode 100644 index 000000000000..8385509309d5 --- /dev/null +++ b/cloud/apps/relay/src/post-proof-close-log.ts @@ -0,0 +1,65 @@ +import { AssignmentRejectionLogWindow } from './assignment-rejection-log-window.js' +import { printableCloseReason } from './control-close-reason-text.js' +import { relayHostLogDigest } from './relay-host-log-digest.js' +import type { RelayRuntimeObserver } from './relay-observability.js' + +// A looping host hits a post-proof close about once a second; per reason and +// cause, one line opens the window and one closing line carries the rest. +export const POST_PROOF_CLOSE_LOG_WINDOW_MS = 60_000 + +// Why the assignment check failed: the close reason alone cannot tell a moved +// assignment from a database timeout. Never the raw message (pg errors carry URLs). +export type PostProofCloseCause = 'wrong_assignment' | 'database_transient' | 'error' + +export type PostProofClose = { + relayHostId: string + generation: number + appVersion: string + // Null when the host hung up itself: the cell sent no close frame. + code: number | null + reason: string + cause?: PostProofCloseCause +} + +// Controls closed between host proof and wiring never reach the 'control closed' listener. +export class PostProofCloseLog { + private readonly window: AssignmentRejectionLogWindow + + constructor( + private readonly observer: RelayRuntimeObserver, + private readonly warn: (line: string) => void = (line) => console.warn(line), + schedule?: (callback: () => void, delayMs: number) => () => void + ) { + this.window = new AssignmentRejectionLogWindow({ + windowMs: POST_PROOF_CLOSE_LOG_WINDOW_MS, + onWindowClosed: ({ suppressed, sample }) => this.warn(formatLine(sample, suppressed)), + schedule + }) + } + + record(close: PostProofClose): void { + this.observer.recordPostProofClose?.(close.reason) + if (!this.window.admit(`${close.reason}:${close.cause ?? ''}`, close)) return + this.warn(formatLine(close, 0)) + } +} + +export function postProofCloseCause( + error: unknown, + isTransient: (error: unknown) => boolean +): PostProofCloseCause { + if (error instanceof Error && error.message === 'wrong_assignment') return 'wrong_assignment' + return isTransient(error) ? 'database_transient' : 'error' +} + +function formatLine(close: PostProofClose, suppressed: number): string { + return ( + `[orca-relay] control refused after proof host=${relayHostLogDigest(close.relayHostId)}` + + ` gen=${close.generation}` + + ` app=${JSON.stringify(printableCloseReason(close.appVersion))}` + + ` code=${close.code ?? 'none'} reason=${JSON.stringify(close.reason)}` + + (close.cause ? ` cause=${close.cause}` : '') + + // Closing line only: how many more closes the window held back; the fields name the latest. + (suppressed > 0 ? ` suppressed=${suppressed}` : '') + ) +} diff --git a/cloud/apps/relay/src/relay-background-operation.ts b/cloud/apps/relay/src/relay-background-operation.ts index be3943b7b1ad..adf704106cd4 100644 --- a/cloud/apps/relay/src/relay-background-operation.ts +++ b/cloud/apps/relay/src/relay-background-operation.ts @@ -1,19 +1,9 @@ +import { describeRelayFailure } from './relay-error-redaction.js' + type Warn = (message: string) => void // A swallowed error hides which sweep step is failing (a silently dead -// cleanup chain stalled production rehoming for hours), but raw messages can -// embed secrets such as connection strings. Only two provably inert shapes -// are logged: this codebase's snake_case invariant slugs, and five-character -// SQLSTATE codes; everything else stays redacted. -function describeFailure(error: unknown): string { - const name = error instanceof Error ? error.name : typeof error - const message = error instanceof Error ? error.message : '' - const slug = /^[a-z0-9_]{1,64}$/.test(message) ? message : 'redacted' - const code = (error as { code?: unknown } | null)?.code - const sqlState = typeof code === 'string' && /^[0-9A-Z]{5}$/.test(code) ? ` code=${code}` : '' - return `${name}: ${slug}${sqlState}` -} - +// cleanup chain stalled production rehoming for hours). export async function runRelayBackgroundOperation( operation: () => Promise, failureMessage: string, @@ -23,6 +13,6 @@ export async function runRelayBackgroundOperation( await operation() } catch (error) { // Dependency outages must fail readiness without crashing liveness. - warn(`${failureMessage}: ${describeFailure(error)}`) + warn(`${failureMessage}: ${describeRelayFailure(error)}`) } } diff --git a/cloud/apps/relay/src/relay-error-redaction.ts b/cloud/apps/relay/src/relay-error-redaction.ts new file mode 100644 index 000000000000..f67642a04002 --- /dev/null +++ b/cloud/apps/relay/src/relay-error-redaction.ts @@ -0,0 +1,14 @@ +// Raw error messages can embed secrets such as connection strings. Only two +// provably inert shapes leave the process: this codebase's snake_case +// invariant slugs, and five-character SQLSTATE codes; everything else is redacted. +export function relayErrorSlug(error: unknown): string | null { + const message = error instanceof Error ? error.message : '' + return /^[a-z0-9_]{1,64}$/.test(message) ? message : null +} + +export function describeRelayFailure(error: unknown): string { + const name = error instanceof Error ? error.name : typeof error + const code = typeof error === 'object' && error !== null && 'code' in error ? error.code : null + const sqlState = typeof code === 'string' && /^[0-9A-Z]{5}$/.test(code) ? ` code=${code}` : '' + return `${name}: ${relayErrorSlug(error) ?? 'redacted'}${sqlState}` +} diff --git a/cloud/apps/relay/src/relay-observability.test.ts b/cloud/apps/relay/src/relay-observability.test.ts index 8e0e209cdf86..6d73c7871e78 100644 --- a/cloud/apps/relay/src/relay-observability.test.ts +++ b/cloud/apps/relay/src/relay-observability.test.ts @@ -508,6 +508,8 @@ describe('relay observability', () => { observability.recordControlClose(1006) observability.recordControlClose(1006) observability.recordControlClose(4402) + observability.recordPostProofClose('relay draining') + observability.recordPostProofClose('relay draining') observability.recordSpliceClose('host-oversize-frame') observability.recordSpliceClose('queue-limit') observability.recordClientAcceptAbandoned('activity', 14_250.4) @@ -518,18 +520,55 @@ describe('relay observability', () => { expect(entries[0]).toMatchObject({ controlClosesByCodeDelta: { 1006: 2, 4402: 1 }, + postProofClosesByReasonDelta: { 'relay draining': 2 }, spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 }, clientAcceptsAbandonedByStageDelta: { activity: 2, credential: 1 }, clientAcceptAbandonedMsMax: 14_250.4 }) expect(entries[1]).toMatchObject({ controlClosesByCodeDelta: {}, + postProofClosesByReasonDelta: {}, spliceClosesByTriggerDelta: {}, clientAcceptsAbandonedByStageDelta: {}, clientAcceptAbandonedMsMax: 0 }) }) + it('counts the phone path, phone refusals by code and the step-6 failures per window', () => { + const entries: Array> = [] + const observability = new RelayObservability( + { role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' }, + (entry) => entries.push(entry) + ) + const stageMs = { assignment: 0, credential: 170, activity: 0, attach: 300, basis: 0 } + observability.recordClientAcceptCompleted({ totalMs: 470, stageMs, phonePath: 'memory' }) + observability.recordClientAcceptCompleted({ totalMs: 3_600, stageMs, phonePath: 'db' }) + observability.recordPhoneRejection(4401) + observability.recordPhoneRejection(4404) + observability.recordPhoneRejection(4404) + observability.recordPhoneAttachDeadline() + observability.recordPhoneAcceptDatabaseError() + observability.recordResumeConfirmFailure('reject-revoked') + observability.recordResumeRenewalFailure() + observability.flush({ ...counts, resumeAuditDropped: 3 }) + observability.flush(counts) + expect(entries[0]).toMatchObject({ + clientAcceptsByPhonePathDelta: { memory: 1, db: 1 }, + phoneRejectionsByCodeDelta: { '4401': 1, '4404': 2 }, + phoneAttachDeadlinesDelta: 1, + phoneAcceptDatabaseErrorsDelta: 1, + resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1 }, + resumeRenewalFailuresDelta: 1, + resumeAuditDropped: 3 + }) + expect(entries[1]).toMatchObject({ + phoneRejectionsByCodeDelta: {}, + phoneAttachDeadlinesDelta: 0, + phoneAcceptDatabaseErrorsDelta: 0, + resumeConfirmFailuresByCodeDelta: {} + }) + }) + it('summarises completed client accepts and control round trips per window', () => { const entries: Array> = [] const observability = new RelayObservability( @@ -572,6 +611,7 @@ describe('relay observability', () => { clientAcceptsAbandonedByStageDelta: {}, clientAcceptAbandonedMsMax: 0 }) + expect(entries[0]).toMatchObject({ clientAcceptsByPhonePathDelta: {} }) // An empty window publishes counts only: a zero percentile point is // indistinguishable from a real zero once Cloud Logging aggregates it. expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 }) diff --git a/cloud/apps/relay/src/relay-observability.ts b/cloud/apps/relay/src/relay-observability.ts index 6f43248b4457..27ab9178dc8a 100644 --- a/cloud/apps/relay/src/relay-observability.ts +++ b/cloud/apps/relay/src/relay-observability.ts @@ -18,6 +18,8 @@ export type RelayRuntimeCounts = { splices: number pendingSplices: number queuedBytes: number + // Step 6: resume audit rows a memory cell dropped since boot; the watcher pages on any. + resumeAuditDropped?: number } export function observedRelayRequests(counts: RelayRuntimeCounts): number { @@ -101,9 +103,16 @@ export interface RelayRuntimeObserver { fallback: boolean }): void recordControlClose?(code: number): void + recordPostProofClose?(reason: string): void recordSpliceClose?(trigger: string): void recordClientAcceptAbandoned?(stage: RelayClientAcceptStage, elapsedMs: number): void recordClientAcceptCompleted?(sample: RelayClientAcceptSample): void + // Step 6: every phone refusal by close code, and the phone-path failures the watcher reads. + recordPhoneRejection?(code: number): void + recordPhoneAttachDeadline?(): void + recordPhoneAcceptDatabaseError?(): void + recordResumeConfirmFailure?(code: string): void + recordResumeRenewalFailure?(): void recordControlRtt?(rttMs: number): void recordHostPreAuth?(path: HostPreAuthPath, outcome: HostPreAuthOutcome, durationMs: number): void } @@ -162,6 +171,7 @@ export const RELAY_CLIENT_ACCEPT_TIMED_STAGES = [ export type RelayClientAcceptSample = { totalMs: number stageMs: Record + phonePath?: 'db' | 'memory' } type RelayMetricDeltas = { @@ -200,11 +210,18 @@ type RelayMetricDeltas = { regionFallbacks: Record unavailableRegions: Record controlClosesByCode: Record + postProofClosesByReason: Record spliceClosesByTrigger: Record clientAcceptsAbandonedByStage: Record clientAcceptAbandonedMsMax: number clientAcceptTotalsMs: number[] clientAcceptStageSamplesMs: Record + clientAcceptsByPhonePath: Record + phoneRejectionsByCode: Record + phoneAttachDeadlines: number + phoneAcceptDatabaseErrors: number + resumeConfirmFailuresByCode: Record + resumeRenewalFailures: number controlRttSamplesMs: number[] controlRttObserved: number controlRenewalLatenciesMs: number[] @@ -288,6 +305,7 @@ const emptyDeltas = (): RelayMetricDeltas => ({ regionFallbacks: {}, unavailableRegions: {}, controlClosesByCode: {}, + postProofClosesByReason: {}, spliceClosesByTrigger: {}, clientAcceptsAbandonedByStage: {}, clientAcceptAbandonedMsMax: 0, @@ -299,6 +317,12 @@ const emptyDeltas = (): RelayMetricDeltas => ({ attach: [], basis: [] }, + clientAcceptsByPhonePath: {}, + phoneRejectionsByCode: {}, + phoneAttachDeadlines: 0, + phoneAcceptDatabaseErrors: 0, + resumeConfirmFailuresByCode: {}, + resumeRenewalFailures: 0, controlRttSamplesMs: [], controlRttObserved: 0, controlRenewalLatenciesMs: [], @@ -529,6 +553,10 @@ export class RelayObservability implements RelayRuntimeObserver { this.deltas.controlClosesByCode[key] = (this.deltas.controlClosesByCode[key] ?? 0) + 1 } + recordPostProofClose(reason: string): void { + increment(this.deltas.postProofClosesByReason, reason) + } + recordSpliceClose(trigger: string): void { this.deltas.spliceClosesByTrigger[trigger] = (this.deltas.spliceClosesByTrigger[trigger] ?? 0) + 1 @@ -543,12 +571,33 @@ export class RelayObservability implements RelayRuntimeObserver { } recordClientAcceptCompleted(sample: RelayClientAcceptSample): void { + if (sample.phonePath) increment(this.deltas.clientAcceptsByPhonePath, sample.phonePath) this.deltas.clientAcceptTotalsMs.push(sample.totalMs) for (const stage of RELAY_CLIENT_ACCEPT_TIMED_STAGES) { this.deltas.clientAcceptStageSamplesMs[stage].push(sample.stageMs[stage]) } } + recordPhoneRejection(code: number): void { + increment(this.deltas.phoneRejectionsByCode, String(code)) + } + + recordPhoneAttachDeadline(): void { + this.deltas.phoneAttachDeadlines += 1 + } + + recordPhoneAcceptDatabaseError(): void { + this.deltas.phoneAcceptDatabaseErrors += 1 + } + + recordResumeConfirmFailure(code: string): void { + increment(this.deltas.resumeConfirmFailuresByCode, code) + } + + recordResumeRenewalFailure(): void { + this.deltas.resumeRenewalFailures += 1 + } + recordControlRtt(rttMs: number): void { const observedBefore = this.deltas.controlRttObserved++ keepInReservoir( @@ -670,10 +719,19 @@ export class RelayObservability implements RelayRuntimeObserver { regionFallbacksDelta: deltas.regionFallbacks, unavailableRegionsDelta: deltas.unavailableRegions, controlClosesByCodeDelta: deltas.controlClosesByCode, + // Closes after a host proof but before its control was wired; none is in the line above. + postProofClosesByReasonDelta: deltas.postProofClosesByReason, spliceClosesByTriggerDelta: deltas.spliceClosesByTrigger, clientAcceptsAbandonedByStageDelta: deltas.clientAcceptsAbandonedByStage, clientAcceptAbandonedMsMax: roundMs(deltas.clientAcceptAbandonedMsMax), clientAcceptCompletedDelta: deltas.clientAcceptTotalsMs.length, + clientAcceptsByPhonePathDelta: deltas.clientAcceptsByPhonePath, + // Phone refusals by close code; attach deadlines are the 4404s sent with the control still up. + phoneRejectionsByCodeDelta: deltas.phoneRejectionsByCode, + phoneAttachDeadlinesDelta: deltas.phoneAttachDeadlines, + phoneAcceptDatabaseErrorsDelta: deltas.phoneAcceptDatabaseErrors, + resumeConfirmFailuresByCodeDelta: deltas.resumeConfirmFailuresByCode, + resumeRenewalFailuresDelta: deltas.resumeRenewalFailures, // Accepts are sparse: publishing a zero percentile for every empty window // would pin the p50 at 0 forever and collapse the p95 at low accept rates. ...(deltas.clientAcceptTotalsMs.length === 0 diff --git a/cloud/apps/relay/src/relay-region-app.test.ts b/cloud/apps/relay/src/relay-region-app.test.ts index 8c4e87ab8c32..0e7685b5dedb 100644 --- a/cloud/apps/relay/src/relay-region-app.test.ts +++ b/cloud/apps/relay/src/relay-region-app.test.ts @@ -140,6 +140,68 @@ describe('Relay region API', () => { }) }) + it('gives a window request the in-region reserve fallback, and never a report', async () => { + const assignment = { + userId: 'user-1', + relayHostId: 'abcdefghijklmnop', + cellId: 'asia-c1', + cellUrl: 'https://asia-c1.relay.example.test', + region: 'asia-east2', + assignmentEpoch: 7, + leaseExpiresAt: Date.now() + 300_000 + } + const assign = vi.fn(async () => assignment) + const regionReserve = vi.fn(async () => null) + const plan = vi.fn(async () => ({ kind: 'database', regionReserve })) + const app = createRelayApp(config(), { + store: {} as never, + assignments: { + assign, + resolve: async () => assignment, + exchangeRegionCorrection: async () => ({ v: 1, reportStatus: 'accepted' }) + } as never, + reservePlacement: { plan } as never, + drain: vi.fn(), + ready: async () => true + }) + const response = await app.request( + '/v1/assign', + assignmentRequest('abcdefghijklmnop', { + reconnect: true, + preferredRegion: 'asia-east2', + regionCorrection: { v: 1, action: 'issue-window' } + }) + ) + expect(response.status).toBe(200) + expect(plan).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + { reconnect: true, region: 'asia-east2', regionCorrection: true } + ) + expect(assign).toHaveBeenCalledWith( + { userId: 'user-1', relayHostId: 'abcdefghijklmnop' }, + 'asia-east2', + 'asia-east2', + undefined, + { regionReserve, hostRequested: true } + ) + plan.mockClear() + await app.request( + '/v1/assign', + assignmentRequest('abcdefghijklmnop', { + regionCorrection: { + v: 1, + action: 'report', + generation: 2, + assignmentEpoch: 7, + policyVersion: 1, + outcome: 'inconclusive', + reason: 'timeout' + } + }) + ) + expect(plan).not.toHaveBeenCalled() + }) + it('returns successful placement when optional window storage is unavailable', async () => { const app = createRelayApp(config(), { store: {} as never, diff --git a/cloud/apps/relay/src/relay-region-placement.test.ts b/cloud/apps/relay/src/relay-region-placement.test.ts index 76cf5479a415..8754b15b0d5b 100644 --- a/cloud/apps/relay/src/relay-region-placement.test.ts +++ b/cloud/apps/relay/src/relay-region-placement.test.ts @@ -1,7 +1,8 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { RelayAssignmentStore } from './assignment-store.js' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { RelayAssignmentStore, type RelayAssignment } from './assignment-store.js' import type { RelayCellConfig } from './config.js' import { openInMemoryRelayDatabase, type RelayDatabase } from './database.js' +import { ReservePaceError } from './reserve-assignment.js' const CELLS: RelayCellConfig[] = [ { @@ -179,6 +180,68 @@ describe('Relay regional placement', () => { ).resolves.toBeUndefined() }) + describe('with a reserve cell in the region', () => { + const identity = { userId: 'spill-user', relayHostId: 'spillhost0000001' } + const onReserve: RelayAssignment = { + ...identity, + cellId: 'asia-reserve', + cellUrl: 'https://asia-reserve.relay.example.com', + assignmentEpoch: 1, + leaseExpiresAt: 1_000_000, + region: 'asia-east2' + } + const assignAsia = async (regionReserve?: () => Promise) => + await store.assign(identity, 'asia-east2', 'asia-east2', undefined, { + placeFresh: async () => null, + regionReserve + }) + const fillAsia = async () => { + for (const cell of CELLS.slice(1)) await store.configureCell(cell, 'migration-only') + } + + it('books the reserve cell rather than spill out of a full region', async () => { + await fillAsia() + const place = vi.fn(async () => onReserve) + await expect(assignAsia(place)).resolves.toEqual(onReserve) + expect(place).toHaveBeenCalledOnce() + // The refused in-region attempt wrote nothing. + expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([]) + }) + + it('answers a throttled reserve cell with its pace, not a cell abroad', async () => { + await fillAsia() + await expect( + assignAsia(async () => { + throw new ReservePaceError(4) + }) + ).rejects.toBeInstanceOf(ReservePaceError) + expect(await database.query(`SELECT cell_id FROM relay_assignments`)).toEqual([]) + }) + + it('spills as before when the reserve cell has no room either', async () => { + await fillAsia() + const place = vi.fn(async () => null) + await expect(assignAsia(place)).resolves.toMatchObject({ + cellId: 'us-c1', + region: 'us-central1' + }) + expect(place).toHaveBeenCalledOnce() + // No in-region reserve room at plan time: today's single spill. + await expect( + store.assign({ ...identity, relayHostId: 'spillhost0000002' }, 'asia-east2', 'asia-east2') + ).resolves.toMatchObject({ cellId: 'us-c1' }) + }) + + it('leaves a region with database room to the database lane', async () => { + const place = vi.fn(async () => onReserve) + await expect(assignAsia(place)).resolves.toMatchObject({ + cellId: 'asia-c1', + region: 'asia-east2' + }) + expect(place).not.toHaveBeenCalled() + }) + }) + it('expires identity-linked region preferences after 30 days', async () => { const identity = { userId: 'expiry-user', relayHostId: 'expiryhost000001' } await store.assign(identity, 'asia-east2') diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts index 7d98d8adb7b4..3824105810ac 100644 --- a/cloud/apps/relay/src/relay-server.ts +++ b/cloud/apps/relay/src/relay-server.ts @@ -372,6 +372,7 @@ export function createRelayServer( mode: effectiveAdmitMode, directorContact: () => reserveDeadMan?.contact(), reregisterInFlight: () => options.cellFlags?.().flags.reregisterInFlight, + phonePath: () => options.cellFlags?.().flags.phonePath ?? 'db', ticketEnforce: () => options.cellFlags?.().flags.ticketCheck === 'enforce', dryRunEnabled: () => options.cellFlags?.().flags.reserveDryRun === true, databaseShedding: () => databaseShedding(), @@ -444,6 +445,7 @@ export function createRelayServer( effectiveAdmitMode() === 'reserve' || sessions.reregistrationPending() > 0 ? 'reserve' : 'db', + cellPhonePathEffective: () => sessions.phonePathEffective(), cellDemote: (request: DemoteRequest) => sessions.demote(request), cellReserveCounts: () => sessions.reserveCounts(), cellPlacementCeiling: placementCeiling @@ -872,7 +874,12 @@ export function createRelayServer( controls.close() clients.close() dataSockets.close() + store.resumeAudit.stop() }) + const resumeAuditDropped = (): number => { + const drops = store.resumeAudit.drops() + return drops.overflow + drops.aged + } const runtimeCounts = () => { const ledgerCounts = connectionLedger?.counts() return { @@ -880,6 +887,7 @@ export function createRelayServer( preAuthConnections, ...sessions.runtimeCounts(), queuedBytes: queuedBytes.current(), + ...(config.role === 'cell' ? { resumeAuditDropped: resumeAuditDropped() } : {}), ...(ledgerCounts ? { inFlightConnections: ledgerCounts.inFlightConnections, diff --git a/cloud/apps/relay/src/relay.blackbox.test.ts b/cloud/apps/relay/src/relay.blackbox.test.ts index 84f8a87f80bc..35785ac6528a 100644 --- a/cloud/apps/relay/src/relay.blackbox.test.ts +++ b/cloud/apps/relay/src/relay.blackbox.test.ts @@ -1542,7 +1542,7 @@ describe('served relay URL', () => { expect(await failed).toMatchObject({ type: 'control-error', reqId: 'fault-install', - code: 'injected SQL failure' + code: 'control_operation_failed' }) const status = nextMessage(host.socket) host.socket.send( @@ -1642,7 +1642,7 @@ describe('served relay URL', () => { expect(await directFailure).toMatchObject({ type: 'control-error', reqId: 'fallback-install', - code: 'injected SQL failure' + code: 'control_operation_failed' }) const status = nextMessage(host.socket) host.socket.send( diff --git a/cloud/apps/relay/src/reserve-assignment.test.ts b/cloud/apps/relay/src/reserve-assignment.test.ts index 38a6aa27e08b..fe424f412dbc 100644 --- a/cloud/apps/relay/src/reserve-assignment.test.ts +++ b/cloud/apps/relay/src/reserve-assignment.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import type { RelayRegion } from '@orca-cloud/relay-contract' import type { CellReserveClient } from './cell-reserve-client.js' import { ReserveAssignment, @@ -13,7 +14,7 @@ import { ShadowSeatDirectory, type SeatFeedCell, type SeatFeedResponse } from '. const HOST = { userId: 'user-1', relayHostId: 'abcdefghijklmnop' } const US = 'us-central1' as const -function cellList(ids: string[], region = US): SeatFeedCell[] { +function cellList(ids: string[], region: RelayRegion = US): SeatFeedCell[] { return ids.map((cellId) => ({ cellId, cellUrl: `https://${cellId}.example`, @@ -322,6 +323,92 @@ describe('reserve assignment on a director', () => { } }) + describe('for a share-draw loser', () => { + const ASIA = 'asia-east2' as const + const OTHER = { ...HOST, relayHostId: 'qrstuvwxyzabcdef' } + const setupAsia = (reserveCell = feed('c1'), reserve?: (cellId: string) => ReserveAttempt) => + setup({ + ...(reserve ? { reserve } : {}), + cells: [...cellList(['c1', 'c2'], ASIA), ...cellList(['c3'])], + feeds: [ + reserveCell, + // Database room far above the reserve cell's, so the 0.25 draw loses. + feed('c2', { admitMode: 'db', counts: { seats: 0, controls: 0, bookings: 0, units: 0, ceiling: 5_000 } }), + feed('c3') + ] + }) + const regionReserveOf = async (assignment: ReserveAssignment, identity = HOST) => { + const plan = await assignment.plan(identity, { reconnect: false, region: ASIA }) + expect(plan).toMatchObject({ kind: 'database', epochFloor: 0 }) + return plan.kind === 'database' ? plan.regionReserve : undefined + } + + it('offers the reserve cell in its region only, only while it has a seat', async () => { + const { assignment, reserved, directory, now } = setupAsia() + const regionReserve = await regionReserveOf(assignment) + expect(reserved).toEqual([]) + await expect(regionReserve?.()).resolves.toMatchObject({ cellId: 'c1', region: ASIA }) + expect(reserved.map((call) => call.cellId)).toEqual(['c1']) + // A full reserve cell is not offered, and the US reserve cell never is. + const full = feed('c1', { counts: { seats: 500, controls: 500, bookings: 0, units: 500, ceiling: 500 } }) + directory.apply('c1', full, now.value, now.value) + expect(await regionReserveOf(assignment, OTHER)).toBeUndefined() + expect(reserved.map((call) => call.cellId)).toEqual(['c1']) + }) + + it('paces when the reserve cell has seats but no intake budget, rather than give up the region', async () => { + const { assignment, reserved } = setupAsia(feed('c1', { intake: { perSec: 1, burst: 20, tokens: 0 } })) + const regionReserve = await regionReserveOf(assignment) + await expect(regionReserve?.()).rejects.toBeInstanceOf(ReservePaceError) + expect(reserved).toEqual([]) + }) + + it('gives the region up when the reserve cell refuses with a seat still on the map', async () => { + for (const outcome of ['full', 'unreachable']) { + const { assignment, reserved } = setupAsia(feed('c1'), (cellId) => + cellId === 'c1' ? { outcome } : { outcome: 'ok' } + ) + const regionReserve = await regionReserveOf(assignment) + // Null sends the host to the database lane's cross-region fallback, not a 1 s retry. + await expect(regionReserve?.()).resolves.toBeNull() + expect(reserved.map((call) => call.cellId)).toEqual(['c1']) + } + }) + }) + + describe('for a region-correction request', () => { + const request = { reconnect: true, region: US, regionCorrection: true } + + it('leaves the answer to the database lane, offering only the in-region reserve fallback', async () => { + const row = vi.fn(async () => null) + // A memory answer would skip the window exchange; the share draw is not offered either. + const { assignment, reserved } = setup({ feeds: [feed('c1', { full: [seated(7)] }), feed('c2')], row }) + const plan = await assignment.plan(HOST, request) + expect(plan).toEqual({ kind: 'database', regionReserve: expect.any(Function) }) + expect(reserved).toEqual([]) + expect(row).not.toHaveBeenCalled() + await expect(plan.kind === 'database' ? plan.regionReserve?.() : null).resolves.toMatchObject({ + assignmentEpoch: 8 + }) + }) + + it('keeps today’s path for a host the map never saw, without reading its row', async () => { + const row = vi.fn(async () => ({ cellId: 'c1', assignmentEpoch: 3 })) + const { assignment, reserved } = setup({ row }) + expect(await assignment.plan(HOST, request)).toEqual({ kind: 'database' }) + expect(row).not.toHaveBeenCalled() + expect(reserved).toEqual([]) + }) + + it('keeps today’s path while the map is incomplete, and in dry run', async () => { + const gated = setup({ cells: cellList(['c1', 'c2']), feeds: [feed('c1', { full: [seated(4)] })], startedAt: 100_500 }) + expect(await gated.assignment.plan(HOST, request)).toEqual({ kind: 'database' }) + const dryRun = setup({ mode: 'dry-run', feeds: [feed('c1', { full: [seated(4)] }), feed('c2')] }) + expect(await dryRun.assignment.plan(HOST, request)).toEqual({ kind: 'database' }) + expect([...gated.reserved, ...dryRun.reserved]).toEqual([]) + }) + }) + it('paces with Retry-After when every reserve cell is out of budget and no database cell exists', async () => { const { assignment } = setup({ reserve: () => ({ outcome: 'intake' }) }) await expect(assignment.plan(HOST, { reconnect: false, region: US })).rejects.toBeInstanceOf( diff --git a/cloud/apps/relay/src/reserve-assignment.ts b/cloud/apps/relay/src/reserve-assignment.ts index a64174571103..eb5d0bc42ae3 100644 --- a/cloud/apps/relay/src/reserve-assignment.ts +++ b/cloud/apps/relay/src/reserve-assignment.ts @@ -4,7 +4,7 @@ import { RELAY_DEFAULT_REGION, type RelayRegion } from '@orca-cloud/relay-contract' -import type { RelayAssignment } from './assignment-store.js' +import type { AssignOptions, RelayAssignment } from './assignment-store.js' import { CELL_RESERVE_TTL_MS } from './cell-reserve-contract.js' import type { CellReserveClient } from './cell-reserve-client.js' import { relayHostLogDigest } from './relay-host-log-digest.js' @@ -45,11 +45,7 @@ type Identity = { userId: string; relayHostId: string } export type ReservePlan = | { kind: 'answer'; assignment: RelayAssignment; lane: 'sticky' | 'placement' } | { kind: 'retry'; retryAfterSeconds: number; reason: 'map-incomplete' | 'database' | 'paced' } - | { - kind: 'database' - epochFloor?: number - placeFresh?: () => Promise - } + | ({ kind: 'database' } & Pick) // A host whose seat the ledger reconcile demoted mints above its row for this long, so the // re-assign does not collide with the row again (the mirror refuses an equal epoch). @@ -141,10 +137,15 @@ export class ReserveAssignment { return this.input.directory.cellIds().some((cellId) => this.input.directory.placementModeOf(cellId) === 'reserve') } - async plan(identity: Identity, request: { reconnect: boolean; region: RelayRegion }): Promise { + async plan( + identity: Identity, + // regionCorrection: the reply carries a region window bound to the database row. + request: { reconnect: boolean; region: RelayRegion; regionCorrection?: boolean } + ): Promise { this.flushSummary() if (this.input.mode === 'off') return { kind: 'database' } if (this.input.mode === 'dry-run') { + if (request.regionCorrection) return { kind: 'database' } return { kind: 'database', placeFresh: async () => await this.dryRun(identity, request.region) } } if (!this.hasReserveCells()) return { kind: 'database' } @@ -171,7 +172,7 @@ export class ReserveAssignment { // above it on any director, not just the one that demoted it. const leftWrongCell = left?.closeCode === RELAY_CLOSE_CODE.WRONG_CELL && !(this.input.databaseBusy?.() ?? false) - if (known.length === 0 || leftWrongCell) { + if ((known.length === 0 && !request.regionCorrection) || leftWrongCell) { try { // Only the never-seen host needs the row; for the WRONG_CELL one it is an optimisation, // bounded so a database stall cannot hold the reconnect (the reconcile repairs a miss). @@ -191,6 +192,13 @@ export class ReserveAssignment { } const epochFloor = known.length === 0 ? 0 : Math.max(...known) const lastCellId = seats[0]?.cellId ?? left?.cellId ?? row?.cellId + // The database lane answers it, so the window exchange follows: no memory answer, no share + // draw, only the in-region fallback, once the map is complete and knows the host. + if (request.regionCorrection) { + return incomplete || known.length === 0 + ? { kind: 'database' } + : { kind: 'database', regionReserve: this.regionReserve(identity, request.region, epochFloor, seats, now) } + } if (incomplete) { if (lastCellId !== undefined && directory.cellState(lastCellId)?.status === 'pending') { this.summary.retries.mapIncomplete += 1 @@ -209,15 +217,36 @@ export class ReserveAssignment { } const placeFresh = async (): Promise => await this.placeFresh(identity, request.region, epochFloor, seats) + const regionReserve = this.regionReserve(identity, request.region, epochFloor, seats, now) // Where the host was last seen decides who owns it: a database-mode cell keeps today's // path, sticky included; only a fresh placement may be booked on a reserve-mode cell. const lastOnReserve = lastCellId !== undefined && directory.placementModeOf(lastCellId) === 'reserve' if (!lastOnReserve && (seats.length > 0 || left !== undefined || row !== null)) { - return { kind: 'database', epochFloor, placeFresh } + return { kind: 'database', epochFloor, placeFresh, regionReserve } } const placed = await placeFresh() if (placed) return { kind: 'answer', assignment: placed, lane: 'placement' } - return { kind: 'database', epochFloor } + return { kind: 'database', epochFloor, regionReserve } + } + + // For the share draw's losers: the database lane tries the region first, then this, then spills. + // Undefined while no in-region reserve cell has a seat, so the database lane runs as today. + private regionReserve( + identity: Identity, + region: RelayRegion, + epochFloor: number, + seats: readonly ShadowSeat[], + now: number + ): AssignOptions['regionReserve'] { + const hasRoom = this.placementCells().some( + (cell) => this.input.placer.isCandidate(cell, region, now) && this.input.placer.hasSeat(cell) + ) + if (!hasRoom) return undefined + return async () => { + const placed = await this.placeFresh(identity, region, epochFloor, seats, true) + if (placed) this.summary.regionKept += 1 + return placed + } } // The same epoch on the same cell: no booking, no database, no row lock. A cell that @@ -288,16 +317,19 @@ export class ReserveAssignment { identity: Identity, region: RelayRegion, epochFloor: number, - seats: readonly ShadowSeat[] = [] + seats: readonly ShadowSeat[] = [], + // The database lane found the region full: no share draw, and no other region. + regionFull = false ): Promise { const placements = this.placementCells() - if (this.random() >= this.input.placer.reservePoolShare(placements, region)) return null - const regions = region === RELAY_DEFAULT_REGION ? [region] : [region, RELAY_DEFAULT_REGION] + if (!regionFull && this.random() >= this.input.placer.reservePoolShare(placements, region)) return null + const regions = regionFull || region === RELAY_DEFAULT_REGION ? [region] : [region, RELAY_DEFAULT_REGION] for (const target of regions) { const result = await this.input.placer.place({ cells: placements, region: target, epoch: mintEpoch([epochFloor]), + regionFull, reserve: async (cellId, epoch) => { const cell = this.input.cells().find((entry) => entry.cellId === cellId) if (!cell) return { outcome: 'unreachable' } @@ -463,6 +495,8 @@ type PlacementSummary = { sticky: number stickyReserves: number placed: number + // Placements the database lane would have sent out of the region. + regionKept: number calls: number database: number supersedes: number @@ -488,6 +522,7 @@ function emptySummary(): PlacementSummary { sticky: 0, stickyReserves: 0, placed: 0, + regionKept: 0, calls: 0, database: 0, supersedes: 0, diff --git a/cloud/apps/relay/src/reserve-placement.ts b/cloud/apps/relay/src/reserve-placement.ts index 3e677c054f5c..2cba1baaf841 100644 --- a/cloud/apps/relay/src/reserve-placement.ts +++ b/cloud/apps/relay/src/reserve-placement.ts @@ -106,8 +106,12 @@ export class ReservePlacer { // Above every epoch this director knows for the host. epoch: number reserve: (cellId: string, epoch: number) => Promise + // The database lane found the region full: a throttled reserve cell paces instead. + regionFull?: boolean }): Promise { const tried = new Set() + // Refused for a reason other than intake: no seat to wait for this time. + const refused = new Set() let epoch = input.epoch let calls = 0 while (calls < RESERVE_MAX_TRIES) { @@ -120,23 +124,30 @@ export class ReservePlacer { if (answer.outcome === 'ok') return { kind: 'placed', cellId: cell.cellId, epoch, calls } this.uncountBooking(cell, countedAt) if (answer.outcome === 'intake') this.drain(cell) + else refused.add(cell.cellId) if (answer.outcome === 'seated-newer' && 'epoch' in answer && typeof answer.epoch === 'number') { epoch = Math.max(epoch, answer.epoch + 1) } } - return this.fallback(input.cells, input.region, calls) + return this.fallback(input.cells, input.region, calls, { regionFull: input.regionFull, refused }) } - // Database cells in the region come first, then pacing; a region with neither falls to the - // database path too, which spills to another region as today. - fallback(cells: readonly PlacementCell[], region: RelayRegion, calls: number): PlacementResult { + // Database cells in the region come first (unless known full), then pacing on a cell that + // only lacks budget; anything else falls to the database path, which spills as today. + fallback( + cells: readonly PlacementCell[], + region: RelayRegion, + calls: number, + options: { regionFull?: boolean; refused?: ReadonlySet } = {} + ): PlacementResult { const now = this.now() - const databaseCells = cells.some( - (cell) => !cell.reserve && cell.region === region && cell.general && !cell.draining - ) + const databaseCells = + !options.regionFull && + cells.some((cell) => !cell.reserve && cell.region === region && cell.general && !cell.draining) if (databaseCells) return { kind: 'database', calls } let refillMs: number | null = null for (const cell of cells) { + if (options.refused?.has(cell.cellId)) continue if (!this.isCandidate(cell, region, now) || !this.hasSeat(cell)) continue const estimate = this.refill(cell, now) const rate = this.ratePerSec(cell) diff --git a/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts b/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts new file mode 100644 index 000000000000..a213074f03c0 --- /dev/null +++ b/cloud/apps/relay/src/resume-confirmed-audit-buffer.test.ts @@ -0,0 +1,129 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { openInMemoryRelayDatabase, type RelayDatabase, type SqlRow } from './database.js' +import { + RESUME_AUDIT_BUFFER_MAX_ROWS, + RESUME_AUDIT_FLUSH_MS, + RESUME_AUDIT_FLUSH_ROWS, + RESUME_AUDIT_MAX_AGE_MS, + ResumeConfirmedAuditBuffer +} from './resume-confirmed-audit-buffer.js' +import { StatementRecordingDatabase } from './test-fixtures/statement-recording-database.js' + +const event = (reqId: string) => ({ + userId: 'user-1', + relayHostId: 'host000000000001', + relayDeviceId: 'device-1', + detail: { reqId, basisConnId: `basis-${reqId}`, renewed: true } +}) + +describe('the batched resume-confirmed audit', () => { + let inner: RelayDatabase | undefined + + afterEach(async () => { + await inner?.close() + inner = undefined + }) + + async function setup(now: () => number = Date.now) { + inner = await openInMemoryRelayDatabase() + const database = new StatementRecordingDatabase(inner) + const lines: Array> = [] + const buffer = new ResumeConfirmedAuditBuffer(database, now, (line) => lines.push(JSON.parse(line))) + const rows = async (): Promise => + await inner!.query(`SELECT type, relay_device_id, detail_json FROM relay_audit_events ORDER BY at, id`) + return { database, buffer, lines, rows } + } + + it('writes buffered rows as one multi-row insert', async () => { + const { database, buffer, rows } = await setup() + for (const reqId of ['a', 'b', 'c']) buffer.add(event(reqId)) + expect(await rows()).toEqual([]) + await buffer.flush() + expect(database.statements).toEqual([ + expect.stringMatching(/^INSERT INTO relay_audit_events .* ON CONFLICT \(id\) DO NOTHING$/) + ]) + expect((await rows()).map((row) => JSON.parse(String(row.detail_json)).reqId).sort()).toEqual(['a', 'b', 'c']) + expect((await rows())[0]).toMatchObject({ type: 'resume-confirmed', relay_device_id: 'device-1' }) + buffer.stop() + }) + + it('flushes at once when a batch fills, without waiting for the timer', async () => { + const { buffer, rows } = await setup() + for (let index = 0; index < RESUME_AUDIT_FLUSH_ROWS; index += 1) buffer.add(event(`r${index}`)) + await buffer.flush() + expect(await rows()).toHaveLength(RESUME_AUDIT_FLUSH_ROWS) + buffer.stop() + }) + + it('retries a flush whose commit outcome was lost without writing any row twice', async () => { + const { database, buffer, rows } = await setup() + buffer.add(event('a')) + buffer.add(event('b')) + // The insert commits, then the reply is lost. + const query = database.query.bind(database) + let lost = true + database.query = async (sql, params) => { + const result = await query(sql, params) + if (lost) { + lost = false + throw new Error('Connection terminated unexpectedly') + } + return result + } + await buffer.flush() + expect(buffer.pending()).toBe(2) + await buffer.flush() + expect(buffer.pending()).toBe(0) + expect(await rows()).toHaveLength(2) + buffer.stop() + }) + + it('caps the buffer through an outage and drops rows too old to retry, counting both', async () => { + let now = 1_000_000 + const { database, buffer, lines, rows } = await setup(() => now) + database.failing = () => true + for (let index = 0; index <= RESUME_AUDIT_BUFFER_MAX_ROWS; index += 1) buffer.add(event(`r${index}`)) + // The batch the 500th row sent fails and comes back: rows in flight count toward the cap. + await buffer.flush() + expect(buffer.pending()).toBe(RESUME_AUDIT_BUFFER_MAX_ROWS) + expect(lines).toEqual([{ event: 'orca_relay_resume_audit_dropped', overflow: 1, aged: 0 }]) + now += RESUME_AUDIT_MAX_AGE_MS + database.failing = null + await buffer.flush() + expect(buffer.drops()).toEqual({ overflow: 1, aged: RESUME_AUDIT_BUFFER_MAX_ROWS }) + expect(lines.at(-1)).toEqual({ + event: 'orca_relay_resume_audit_dropped', + overflow: 1, + aged: RESUME_AUDIT_BUFFER_MAX_ROWS + }) + expect(await rows()).toEqual([]) + // Quiet once nothing more is lost. + await buffer.flush() + expect(lines).toHaveLength(2) + buffer.stop() + }) + + it('sends one early batch on crossing the batch size, and leaves retries to the timer', async () => { + const { database, buffer } = await setup() + database.failing = () => true + for (let index = 0; index < RESUME_AUDIT_FLUSH_ROWS + 50; index += 1) buffer.add(event(`r${index}`)) + await buffer.flush() + // The 50 adds past the batch size sent nothing more: no insert per confirm into a failing database. + expect(database.statements).toHaveLength(1) + buffer.stop() + }) + + it('writes nothing more once stopped', async () => { + vi.useFakeTimers() + try { + const { database, buffer } = await setup() + buffer.stop() + buffer.add(event('late')) + await vi.advanceTimersByTimeAsync(RESUME_AUDIT_FLUSH_MS * 3) + expect(database.statements).toEqual([]) + } finally { + vi.useRealTimers() + } + }) +}) + diff --git a/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts b/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts new file mode 100644 index 000000000000..f23239126fa2 --- /dev/null +++ b/cloud/apps/relay/src/resume-confirmed-audit-buffer.ts @@ -0,0 +1,114 @@ +import { randomUUID } from 'node:crypto' +import type { RelayDatabase } from './database.js' + +// Step 6 (decision 4): a memory cell's resume-confirmed audit rows leave in batches, off the +// confirm's round trips. A crash or shutdown loses at most one flush interval of rows: accepted, +// since a drain at exit would race the database pool closing. +export const RESUME_AUDIT_FLUSH_MS = 5_000 +export const RESUME_AUDIT_FLUSH_ROWS = 500 +export const RESUME_AUDIT_BUFFER_MAX_ROWS = 5_000 +// An erasure deletes an account's audit rows by hand; a row this old is dropped rather than +// retried, so a database outage cannot insert it after the account is gone. +export const RESUME_AUDIT_MAX_AGE_MS = 10 * 60 * 1000 + +export type ResumeConfirmedAudit = { + userId: string + relayHostId: string + relayDeviceId: string + detail: { reqId: string; basisConnId: string; renewed: boolean } +} + +// The id is fixed when the event happens, so a retried flush after an unknown commit outcome +// cannot write a row twice (ON CONFLICT (id) DO NOTHING). +type BufferedRow = ResumeConfirmedAudit & { id: string; at: number } + +export type ResumeAuditDrops = { overflow: number; aged: number } + +export class ResumeConfirmedAuditBuffer { + private rows: BufferedRow[] = [] + private flushing: Promise | null = null + private timer: ReturnType | null = null + private readonly dropped: ResumeAuditDrops = { overflow: 0, aged: 0 } + private reportedDrops = 0 + private stopped = false + + constructor( + private readonly database: RelayDatabase, + private readonly now: () => number = Date.now, + private readonly warn: (line: string) => void = (line) => console.warn(line) + ) {} + + add(event: ResumeConfirmedAudit): void { + this.rows.push({ ...event, id: randomUUID(), at: this.now() }) + this.trimToCap() + if (!this.stopped && !this.timer) { + this.timer = setInterval(() => void this.flush(), RESUME_AUDIT_FLUSH_MS) + this.timer.unref?.() + } + // Only on crossing the batch size: retries of a failing database wait for the timer. + if (this.rows.length === RESUME_AUDIT_FLUSH_ROWS) void this.flush() + } + + // Rows lost since boot: the watcher pages on any. + drops(): ResumeAuditDrops { + return { ...this.dropped } + } + + pending(): number { + return this.rows.length + } + + // One flush in flight at a time; a failed batch goes back to the front with its ids. + async flush(): Promise { + if (this.flushing) return await this.flushing + this.flushing = this.flushOnce().finally(() => { + this.flushing = null + }) + return await this.flushing + } + + stop(): void { + this.stopped = true + if (this.timer) clearInterval(this.timer) + this.timer = null + } + + // The oldest rows go first; rows back from a failed batch count toward the cap. + private trimToCap(): void { + const excess = this.rows.length - RESUME_AUDIT_BUFFER_MAX_ROWS + if (excess <= 0) return + this.rows.splice(0, excess) + this.dropped.overflow += excess + } + + private async flushOnce(): Promise { + const aged = this.now() - RESUME_AUDIT_MAX_AGE_MS + const before = this.rows.length + this.rows = this.rows.filter((row) => row.at > aged) + this.dropped.aged += before - this.rows.length + const batch = this.rows.splice(0, RESUME_AUDIT_FLUSH_ROWS) + if (batch.length > 0) { + try { + await this.database.query( + `INSERT INTO relay_audit_events + (id, at, type, user_id, relay_host_id, relay_device_id, detail_json) + VALUES ${batch.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ')} + ON CONFLICT (id) DO NOTHING`, + batch.flatMap((row) => [ + row.id, row.at, 'resume-confirmed', row.userId, row.relayHostId, row.relayDeviceId, + JSON.stringify(row.detail) + ]) + ) + } catch { + this.rows.unshift(...batch) + this.trimToCap() + } + } + // At most one line per flush, carrying the totals since boot. + const total = this.dropped.overflow + this.dropped.aged + if (total > this.reportedDrops) { + this.reportedDrops = total + this.warn(JSON.stringify({ event: 'orca_relay_resume_audit_dropped', ...this.drops() })) + } + } +} diff --git a/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts b/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts index 0242f499a631..265e5bfaa615 100644 --- a/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts +++ b/cloud/apps/relay/src/test-fixtures/reserve-mode-cell.ts @@ -28,7 +28,7 @@ async function unusedPort(): Promise { return address.port } -function nextMessage(socket: WebSocket): Promise> { +export function nextMessage(socket: WebSocket): Promise> { return new Promise((resolve, reject) => { socket.once('message', (data: RawData) => { try { @@ -42,6 +42,19 @@ function nextMessage(socket: WebSocket): Promise> { }) } +// The next frame of this type, skipping pings and anything else the socket carries first. +export function nextFrame(socket: WebSocket, type: string): Promise> { + return new Promise((resolve) => { + const onMessage = (data: RawData) => { + const frame: unknown = JSON.parse(data.toString()) + if (!frame || typeof frame !== 'object' || !('type' in frame) || frame.type !== type) return + socket.off('message', onMessage) + resolve(Object.fromEntries(Object.entries(frame))) + } + socket.on('message', onMessage) + }) +} + export async function startReserveModeCell(input: { database: RelayDatabase dataDir: string @@ -199,6 +212,64 @@ export async function startReserveModeCell(input: { return { identity, ...reply } } + // A paired phone's device row, as an install would have written it. + const installDevice = async ( + relayHostId: string, + resumeToken: string, + options: { expiresAt?: number; revokedAt?: number; relayDeviceId?: string } = {} + ) => + await input.database.query( + `INSERT INTO relay_devices + (user_id, relay_host_id, relay_device_id, current_hash, current_version, + current_expires_at, revoked_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + 'user-1', relayHostId, options.relayDeviceId ?? 'device-1', + createHash('sha256').update(resumeToken).digest('base64url'), 1, + options.expiresAt ?? Date.now() + 20 * 24 * 60 * 60 * 1000, options.revokedAt ?? null, Date.now() + ] + ) + + // A phone's dial: its first relay-hello frame and its close. + let phoneSource = 0 + const dialPhone = async (relayHostId: string, credential: string) => { + phoneSource += 1 + const socket = new WebSocket(`ws://127.0.0.1:${port}/v1/connect/${relayHostId}`, { + headers: { 'x-forwarded-for': `spoofed, 192.0.2.${phoneSource}, 35.191.0.1` }, + perMessageDeflate: false + }) + await new Promise((resolve, reject) => { + socket.once('open', () => resolve()) + socket.once('error', reject) + }) + input.cleanup.push(() => socket.terminate()) + const closed = new Promise<{ code: number; reason: string }>((resolve) => + socket.once('close', (code, reason) => resolve({ code, reason: reason.toString() })) + ) + const hello = nextMessage(socket) + socket.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential })) + return { socket, hello, closed } + } + + // The desktop's data leg for a conn-open its control received. + const attachData = async (connOpen: Record, generation: unknown) => { + const socket = new WebSocket(`ws://127.0.0.1:${port}/v1/host/data/${String(connOpen.connId)}`, { + perMessageDeflate: false + }) + await new Promise((resolve, reject) => { + socket.once('open', () => resolve()) + socket.once('error', reject) + }) + input.cleanup.push(() => socket.terminate()) + socket.send(JSON.stringify({ type: 'host-data-auth', v: 1, connTicket: connOpen.connTicket, generation })) + return socket + } + + const activityLeases = async () => + await input.database.query( + `SELECT activity_id, activity_kind FROM relay_assignment_activity_leases ORDER BY activity_id` + ) + const controlLeases = async () => await input.database.query( `SELECT relay_host_id, cell_id, activity_id FROM relay_assignment_activity_leases @@ -221,6 +292,10 @@ export async function startReserveModeCell(input: { insertRow, bookedHost, controlLeases, + activityLeases, + installDevice, + dialPhone, + attachData, admitEffectiveRow, setAdmitMode: (admitMode: 'db' | 'reserve', flags: Partial = {}) => { applied = { generation: applied.generation + 1, flags: { ...CELL_FLAG_DEFAULTS, ...flags, admitMode } } diff --git a/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts b/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts new file mode 100644 index 000000000000..d195ed6c552c --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/statement-recording-database.ts @@ -0,0 +1,59 @@ +import type { + RelayDatabase, + RelayLockOptions, + RelayTransactionOptions, + SqlRow +} from '../database.js' + +// Every statement a relay path issues, in order, so its round trips are counted rather than +// described. BEGIN and COMMIT are recorded too: each is a trip to a remote database. +export class StatementRecordingDatabase implements RelayDatabase { + readonly dialect: RelayDatabase['dialect'] + // A statement this matches fails as a dropped connection would, before reaching the database. + failing: ((sql: string) => boolean) | null = null + + constructor( + private readonly inner: RelayDatabase, + readonly statements: string[] = [] + ) { + this.dialect = inner.dialect + } + + async query(sql: string, params: unknown[] = []): Promise { + this.statements.push(normalized(sql)) + if (this.failing?.(normalized(sql))) throw new Error('Connection terminated unexpectedly') + return await this.inner.query(sql, params) + } + + async queryLocked(sql: string, params: unknown[] = [], options: RelayLockOptions = {}): Promise { + this.statements.push(`${normalized(sql)} [locked]`) + return await this.inner.queryLocked(sql, params, options) + } + + async transaction( + operation: (transaction: RelayDatabase) => Promise, + options: RelayTransactionOptions = {} + ): Promise { + this.statements.push('BEGIN') + try { + return await this.inner.transaction( + async (transaction) => { + const recording = new StatementRecordingDatabase(transaction, this.statements) + recording.failing = this.failing + return await operation(recording) + }, + options + ) + } finally { + this.statements.push('COMMIT') + } + } + + async close(): Promise { + await this.inner.close() + } +} + +function normalized(sql: string): string { + return sql.replace(/\s+/g, ' ').trim() +} diff --git a/cloud/dev/fixtures/terraform-root-partition/families.json b/cloud/dev/fixtures/terraform-root-partition/families.json index 89d97154b0af..9067dda7dbec 100644 --- a/cloud/dev/fixtures/terraform-root-partition/families.json +++ b/cloud/dev/fixtures/terraform-root-partition/families.json @@ -206,6 +206,7 @@ "google_project_iam_member.github_staging_relay_capacity_mutation", "google_project_iam_member.github_staging_relay_capacity_viewer", "google_project_iam_member.github_staging_relay_deploy_compute_viewer", + "google_project_iam_member.github_staging_relay_deploy_logging_viewer", "google_project_iam_member.github_staging_relay_power", "google_project_iam_member.push_runtime_cloudsql_client", "google_project_iam_member.push_runtime_fcm_admin", diff --git a/cloud/dev/scripts/check-relay-same-cap-headroom.mjs b/cloud/dev/scripts/check-relay-same-cap-headroom.mjs index 95fad7b8543b..6cc0bebf64ea 100644 --- a/cloud/dev/scripts/check-relay-same-cap-headroom.mjs +++ b/cloud/dev/scripts/check-relay-same-cap-headroom.mjs @@ -1,5 +1,6 @@ import { pathToFileURL } from 'node:url' import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' +import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs' // Drained hosts that find no free slot keep redialling and pin the drained cell (c28, // 2026-10-01). Hosts are counted as controls, but each moved host also brings its splices; @@ -57,6 +58,8 @@ async function adminJson(fetchImpl, url, token, body, wait) { { wait } ) const json = await response.json().catch(() => ({})) + // null: the director refused the read because the cell is reserve. + if (isReserveCellRefusal(response.status, json)) return null if (!response.ok) throw new Error(`${new URL(url).pathname} returned ${response.status}`) return json } @@ -89,15 +92,18 @@ export async function checkSameCapHeadroom(config, overrides = {}) { const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') const cellStatus = async (cellId) => { - const { status } = await adminJson( + const answer = await adminJson( fetchImpl, `${config.directorOrigin}/v1/admin/cell-status`, token, { v: 1, cellId }, wait ) - if (status?.cellId !== cellId) throw new Error('director status does not match the cell') - return status + if (answer === null) return null + if (answer.status?.cellId !== cellId) { + throw new Error('director status does not match the cell') + } + return answer.status } const runtime = await adminJson( fetchImpl, @@ -106,16 +112,23 @@ export async function checkSameCapHeadroom(config, overrides = {}) { { v: 1 }, wait ) - if (runtime.role !== 'cell' || runtime.cellId !== config.cellId) { + if (runtime?.role !== 'cell' || runtime.cellId !== config.cellId) { throw new Error('runtime status does not match the cell') } const targetHosts = count(runtime.runtime?.controls, 'target host controls') - const targetRegion = (await cellStatus(config.cellId)).region ?? null + const targetStatus = await cellStatus(config.cellId) + if (targetStatus === null) throw reserveCellRollRefusal(config.cellId) + const targetRegion = targetStatus.region ?? null const cells = [] for (const cellId of config.generalCells) { if (cellId === config.cellId) continue const status = await cellStatus(cellId) - cells.push({ cellId, region: status.region ?? null, freeSlots: freeSlots(status) }) + // A reserve cell's free seats are not in the database: counted as none, never guessed. + cells.push( + status === null + ? { cellId, region: null, freeSlots: 0, reserve: true } + : { cellId, region: status.region ?? null, freeSlots: freeSlots(status) } + ) } const total = (filter) => cells.filter(filter).reduce((sum, cell) => sum + cell.freeSlots, 0) diff --git a/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs b/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs index 66b5eda3bacb..de6304c7037a 100644 --- a/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs +++ b/cloud/dev/scripts/check-relay-same-cap-headroom.test.mjs @@ -134,6 +134,46 @@ test('no general cells means no headroom', async () => { assert.equal(result.sufficient, false) }) +const reserveRefusal = () => Response.json({ error: 'cell_in_reserve_mode' }, { status: 404 }) + +test('a reserve cell that is not being rolled offers no slots and is named', async () => { + const base = harness({ controls: 160 }) + const result = await run(async (url, options) => + JSON.parse(options.body).cellId === 'production-gce-c29' + ? reserveRefusal() + : await base(url, options)) + assert.equal(result.freeSlots, 200) + assert.equal(result.sufficient, true) + assert.deepEqual(result.cells.find((entry) => entry.cellId === 'production-gce-c29'), { + cellId: 'production-gce-c29', + region: null, + freeSlots: 0, + reserve: true + }) +}) + +test('refuses to roll a reserve cell, naming the flip back to db', async () => { + const base = harness() + await assert.rejects( + run(async (url, options) => + JSON.parse(options.body ?? '{}').cellId === config.cellId + ? reserveRefusal() + : await base(url, options)), + /production-gce-c28 is in reserve mode .*flip it to admitMode=db/ + ) +}) + +test('any other 404 from cell-status still fails', async () => { + const base = harness() + await assert.rejects( + run(async (url, options) => + JSON.parse(options.body).cellId === 'production-gce-c29' + ? Response.json({ error: 'cell_not_found' }, { status: 404 }) + : await base(url, options)), + /cell-status returned 404/ + ) +}) + test('rejects a runtime or director answer for another cell', async () => { const base = harness() await assert.rejects( diff --git a/cloud/dev/scripts/deploy-relay-blue-green.mjs b/cloud/dev/scripts/deploy-relay-blue-green.mjs index 08a18475cabe..f9dfe213daf9 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.mjs @@ -30,6 +30,14 @@ export const DIRECTOR_ADMISSION_ENVIRONMENT = Object.freeze({ }) const DIRECTOR_STARTUP_PROBE = 'tcpSocket.port=8080,timeoutSeconds=120,periodSeconds=120,failureThreshold=1' +// Why: Cloud Run routes a new revision's floor instances their first request up to ~110 s after +// they start, so a one-step move lands every assignment on the one pre-started instance. +export const DIRECTOR_STAGED_TRAFFIC_PERCENT = 20 +// Floor instances have started up to ~220 s after the revision and taken ~110 s more to route. +const DIRECTOR_SPREAD_TIMEOUT_MS = 6 * 60 * 1000 +const DIRECTOR_SPREAD_POLL_INTERVAL_MS = 15_000 +// A hung read must not hold the 20/80 split past the deadline. +const DIRECTOR_SPREAD_READ_TIMEOUT_MS = 30_000 export function taggedRevisionOrigin(serviceOrigin, tag) { const url = new URL(serviceOrigin) @@ -461,8 +469,13 @@ export function parseArguments(argv) { return values } -function commandJson(args) { - const result = spawnSync('gcloud', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }) +function commandJson(args, { timeoutMs } = {}) { + const result = spawnSync('gcloud', args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: timeoutMs + }) + if (result.error) throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.error.message}`) if (result.status !== 0) { throw new Error(`gcloud ${args.slice(0, 4).join(' ')} failed: ${result.stderr.trim()}`) } @@ -727,6 +740,59 @@ function cellIdentifier(sourceCellId, releaseId) { return `${sourceCellId.slice(0, available)}--${suffix}` } +// Instances that served a request since `since`, read from Cloud Run's own request log. +function servingInstanceIds(config, revision, since) { + const filter = [ + 'resource.type="cloud_run_revision"', + `resource.labels.service_name="${config.service}"`, + `resource.labels.revision_name="${revision}"`, + `logName="projects/${config.project}/logs/run.googleapis.com%2Frequests"`, + `timestamp>="${since}"` + ].join(' AND ') + const entries = commandJson([ + 'logging', + 'read', + filter, + '--project', + config.project, + '--limit', + '1000', + '--format=json(labels.instanceId)' + ], { timeoutMs: DIRECTOR_SPREAD_READ_TIMEOUT_MS }) + return new Set(entries.map((entry) => entry.labels?.instanceId).filter(Boolean)) +} + +export async function waitForServingInstances( + config, + revision, + required, + since, + { + readInstances = servingInstanceIds, + pollIntervalMs = DIRECTOR_SPREAD_POLL_INTERVAL_MS, + timeoutMs = DIRECTOR_SPREAD_TIMEOUT_MS + } = {} +) { + const deadline = Date.now() + timeoutMs + let instances = 0 + for (;;) { + try { + instances = readInstances(config, revision, since).size + console.warn(JSON.stringify({ event: 'director_candidate_spread', revision, instances, required })) + if (instances >= required) return + } catch (error) { + // A throttled log read is not evidence either way; the deadline still bounds the wait. + console.warn(JSON.stringify({ event: 'director_candidate_spread_unread', revision, error: String(error) })) + } + if (Date.now() >= deadline) { + // Fewer than the floor here can mean Cloud Run packs the partial share onto fewer instances. + console.warn(JSON.stringify({ event: 'director_candidate_spread_timeout', revision, instances, required })) + throw new Error(`candidate ${revision} served from ${instances} of ${required} instances before the deadline`) + } + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)) + } +} + async function waitForHealth(origin, connectionCapacityProtocol) { const response = await fetch(`${origin}/health`, { signal: AbortSignal.timeout(15_000) }) const body = await response.json() @@ -952,6 +1018,7 @@ export async function deployDirector(config, tag, overrides = {}) { deleteRevision, updateTraffic, waitForHealth, + waitForServingInstances, assertRegionalRehomeDisabled, readReserveCells, readDirectorPlacement, @@ -962,7 +1029,8 @@ export async function deployDirector(config, tag, overrides = {}) { // 100% of traffic. Inheriting the serving floor keeps one owner for the number instead of // restating it here; public admission is per-instance, so losing it shrinks fleet capacity. const initialService = operations.describeService(config) - const servingRevision = operations.describeRevision(config, activeRevision(initialService)) + const servingRevisionName = activeRevision(initialService) + const servingRevision = operations.describeRevision(config, servingRevisionName) const shapeBaseline = directorShapeBaseline( config, initialService, @@ -1058,6 +1126,7 @@ export async function deployDirector(config, tag, overrides = {}) { } removeDirectorTrafficTags(config, operations) let deployed + let staged = false let promoted = false try { operations.deployCandidate( @@ -1178,6 +1247,18 @@ export async function deployDirector(config, tag, overrides = {}) { }) ) } + // A floor of one is the pre-started instance itself; nothing else needs to warm. + if (requiredMinimumInstances > 1) { + const since = new Date().toISOString() + staged = true + operations.updateTraffic(config, [`--to-tags=${tag}=${DIRECTOR_STAGED_TRAFFIC_PERCENT}`]) + await operations.waitForServingInstances( + config, + candidate.revision, + requiredMinimumInstances, + since + ) + } operations.updateTraffic(config, [`--to-tags=${tag}=100`]) promoted = true removeDirectorTrafficTags(config, operations, new Set([SELECTOR_ROLLBACK_TAG])) @@ -1195,6 +1276,13 @@ export async function deployDirector(config, tag, overrides = {}) { } catch (rollbackError) { recoveryErrors.push(rollbackError) } + } else if (staged) { + // The serving revision still holds most traffic and its warm floor; the cold rollback has neither. + try { + operations.updateTraffic(config, [`--to-revisions=${servingRevisionName}=100`]) + } catch (rollbackError) { + recoveryErrors.push(rollbackError) + } } try { removeDirectorTrafficTags( diff --git a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs index 0ba40b6f97b1..fac960c12e94 100644 --- a/cloud/dev/scripts/deploy-relay-blue-green.test.mjs +++ b/cloud/dev/scripts/deploy-relay-blue-green.test.mjs @@ -32,7 +32,8 @@ import { taggedRevisionOrigin, taggedTraffic, trafficTags, - waitForEvacuationCapacity + waitForEvacuationCapacity, + waitForServingInstances } from './deploy-relay-blue-green.mjs' // Terraform declares these values; audited blue/green stamps the same values without targeting @@ -443,7 +444,8 @@ function directorHarness({ placements = {}, callerEmail, // The revision template's Cloud SQL transport; a Terraform apply changes it under the serving one. - templateTransport + templateTransport, + spreadFailure } = {}) { const state = { activeRevision: 'relay-00001-old', @@ -476,6 +478,8 @@ function directorHarness({ } const removed = [] const healthProtocols = [] + const trafficMoves = [] + const spreadWaits = [] let pendingCleanupFailure = cleanupFailsBeforeRemoval const operations = { describeService: () => ({ @@ -563,19 +567,25 @@ function directorHarness({ if (cleanupReportsFailure) throw new Error('gcloud reported failed latest revision') return } - const promote = args.find((argument) => argument.startsWith('--to-tags=')) - assert.ok(promote) - const tag = promote.slice('--to-tags='.length).split('=')[0] - state.activeRevision = state.tags.get(tag) + const move = args.find((argument) => /^--to-(tags|revisions)=/.test(argument)) + assert.ok(move) + trafficMoves.push(move) + const [target, percent] = move.slice(move.indexOf('=') + 1).split('=') + if (percent !== '100') return + state.activeRevision = move.startsWith('--to-tags=') ? state.tags.get(target) : target }, waitForHealth: async (_origin, connectionCapacityProtocol) => { healthProtocols.push(connectionCapacityProtocol) }, + waitForServingInstances: async (_config, revision, required) => { + spreadWaits.push({ revision, required }) + if (spreadFailure) throw spreadFailure + }, readReserveCells: async (_config, origin) => reserveCells[origin], readDirectorPlacement: async (_config, origin) => placements[origin] ?? 'unreported', adminIdentityToken: () => adminToken(callerEmail) } - return { state, removed, healthProtocols, operations } + return { state, removed, healthProtocols, trafficMoves, spreadWaits, operations } } test('director deploy removes stale and promoted Cloud Run tags', async () => { @@ -701,6 +711,73 @@ test('director candidate inherits the serving warm-instance floor', async () => assert.equal(rollback.minimum, 0) }) +test('a multi-instance floor moves part of the traffic until every floor instance serves', async () => { + const harness = directorHarness({ servingMinimum: 5 }) + await deployDirector({}, 'candidate-new', harness.operations) + assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=20', '--to-tags=candidate-new=100']) + assert.deepEqual(harness.spreadWaits, [{ revision: 'relay-00003-new', required: 5 }]) + assert.equal(harness.state.activeRevision, 'relay-00003-new') +}) + +test('a floor of one moves all traffic at once', async () => { + const harness = directorHarness() + await deployDirector({}, 'candidate-new', harness.operations) + assert.deepEqual(harness.trafficMoves, ['--to-tags=candidate-new=100']) + assert.deepEqual(harness.spreadWaits, []) +}) + +test('a candidate that never spreads returns traffic to the warm serving revision', async () => { + const failure = new Error('candidate served from 1 of 5 instances before the deadline') + const harness = directorHarness({ servingMinimum: 5, spreadFailure: failure }) + await assert.rejects(deployDirector({}, 'candidate-new', harness.operations), failure) + assert.deepEqual(harness.trafficMoves, [ + '--to-tags=candidate-new=20', + '--to-revisions=relay-00001-old=100' + ]) + assert.equal(harness.state.activeRevision, 'relay-00001-old') + assert.equal(harness.state.tags.size, 0) +}) + +test('the spread wait counts distinct instances that served since the partial move', async () => { + const reads = [new Set(['a']), new Set(['a', 'b']), new Set(['a', 'b', 'c'])] + const seen = [] + await waitForServingInstances({}, 'relay-00003-new', 3, '2026-10-20T05:25:00.000Z', { + readInstances: (_config, revision, since) => { + seen.push([revision, since]) + return reads.shift() + }, + pollIntervalMs: 0, + timeoutMs: 60_000 + }) + assert.equal(seen.length, 3) + assert.deepEqual(seen[0], ['relay-00003-new', '2026-10-20T05:25:00.000Z']) +}) + +test('the spread wait retries a failed log read', async () => { + let reads = 0 + await waitForServingInstances({}, 'relay-00003-new', 2, '2026-10-20T05:25:00.000Z', { + readInstances: () => { + reads += 1 + if (reads === 1) throw new Error('RESOURCE_EXHAUSTED') + return new Set(['a', 'b']) + }, + pollIntervalMs: 0, + timeoutMs: 60_000 + }) + assert.equal(reads, 2) +}) + +test('the spread wait fails at its deadline', async () => { + await assert.rejects( + waitForServingInstances({}, 'relay-00003-new', 5, '2026-10-20T05:25:00.000Z', { + readInstances: () => new Set(['a']), + pollIntervalMs: 0, + timeoutMs: 0 + }), + /served from 1 of 5 instances before the deadline/ + ) +}) + test('director deploy rejects a serving maximum above the checked budget', async () => { const harness = directorHarness({ servingMaximum: 6 }) await assert.rejects( diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs index 6a0928041c99..0c324dda5ea3 100644 --- a/cloud/dev/scripts/deploy-relay-gce-candidate.mjs +++ b/cloud/dev/scripts/deploy-relay-gce-candidate.mjs @@ -6,6 +6,7 @@ import { selectorCellState, transitionAdmissionSelector } from './relay-admission-selector.mjs' +import { isReserveCellRefusal, reserveCellRollRefusal } from './relay-reserve-cell-refusal.mjs' const DEFAULT_POLL_INTERVAL_MS = 5_000 const DEFAULT_TIMEOUT_MS = 14 * 60 * 1_000 @@ -304,8 +305,9 @@ export function defaultIdentityToken(audience) { return result.stdout.trim() } -async function responseJson(response, label) { +async function responseJson(response, label, cellId) { const body = await response.json().catch(() => ({ error: `http_${response.status}` })) + if (isReserveCellRefusal(response.status, body)) throw reserveCellRollRefusal(cellId) if (!response.ok) throw new Error(`${label} failed: ${body.error ?? response.status}`) return body } @@ -349,7 +351,10 @@ export function createAdminPost(config, deps, token) { await deps.wait(deps.random() * ADMIN_RETRY_BASE_MS * 2 ** (attempt - 1)) continue } - return await responseJson(response, path) + // The director names no cell, so a two-cell request names both. + const cells = + body?.cellId ?? [body?.sourceCellId, body?.targetCellId].filter(Boolean).join(' or ') + return await responseJson(response, path, cells || null) } throw new Error(`${path} retry attempts exhausted`) } diff --git a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs index 3eccd1cf1912..3dc33d659532 100644 --- a/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs +++ b/cloud/dev/scripts/deploy-relay-gce-candidate.test.mjs @@ -62,6 +62,39 @@ test('retries evacuation-status HTTP 500 without widening other admin retries', assert.equal(calls, 1) }) +test('a reserve-cell refusal names the flip back to db; other 404s keep their error', async () => { + const post = (body) => createAdminPost({}, { + fetch: async () => Response.json(body, { status: 404 }), + emit: () => {}, + wait: async () => {}, + random: () => 0 + }, 'token') + await assert.rejects( + post({ error: 'cell_in_reserve_mode' })( + 'https://relay.example', + '/v1/admin/cell-status', + { v: 1, cellId: 'production-gce-c7' } + ), + /production-gce-c7 is in reserve mode .*flip it to admitMode=db/ + ) + await assert.rejects( + post({ error: 'cell_not_found' })( + 'https://relay.example', + '/v1/admin/cell-status', + { v: 1, cellId: 'production-gce-c7' } + ), + /cell-status failed: cell_not_found/ + ) + await assert.rejects( + post({ error: 'cell_in_reserve_mode' })( + 'https://relay.example', + '/v1/admin/evacuation-status', + { v: 1, sourceCellId: 'production-gce-c7', targetCellId: 'production-gce-c8' } + ), + /production-gce-c7 or production-gce-c8 is in reserve mode/ + ) +}) + test('verifies a 1,000-cap cell against both runtime and director telemetry', () => { const expected = deployment( { diff --git a/cloud/dev/scripts/load-relay-controls.mjs b/cloud/dev/scripts/load-relay-controls.mjs index 409f83d8f6e2..8b736e2f25d0 100644 --- a/cloud/dev/scripts/load-relay-controls.mjs +++ b/cloud/dev/scripts/load-relay-controls.mjs @@ -32,10 +32,16 @@ import { } from './relay-load-run-lifecycle.mjs' import { createRelayLoadReaderEvidence } from './relay-load-reader-evidence.mjs' import { holdRelayLoadSplice, relayLoadSpliceReopenSummary } from './relay-load-splice-retry.mjs' +import { + installRelayLoadResumeDevice, + relayLoadResumeSummary, + resumeRelayLoadPhoneOnce +} from './relay-load-resume-phone.mjs' import { parseRelayLoadArguments, relayLoadPrincipalIndex, relayLoadReaderEvidenceError, + relayLoadResumePhoneIndexes, relayLoadSpliceIndexes, relayLoadSpliceProfile, relayLoadSpliceStartDelayMs @@ -119,6 +125,7 @@ function report(state, final = false) { ...readerQueueEvidence.map(({ increaseBytes }) => increaseBytes) ), readerClosesByCode: state.readerClosesByCode, + resumePhones: relayLoadResumeSummary(state.resumeAttempts), // Every cell a control connected to, so a canary can prove its load reached the target. assignedCellOrigins: [...state.assignedCellOrigins].sort(), controlHeadroom: Math.max(0, state.controls - state.active.size), @@ -251,6 +258,7 @@ const state = { spliceRetries: 0, spliceFailuresByReason: {}, spliceFailedAt: new Map(), + resumeAttempts: [], spliceReopenGapsMs: [], slowReaderSplicesCompleted: 0, wedgedReaderSplicesClosed: 0, @@ -411,6 +419,42 @@ for (let localIndex = 0; localIndex < config.controls; localIndex++) { const offset = config.rampStartDelayMs + rampOffset initialConnections.push(delay(offset).then(() => connect(peer))) } +function recordResumeAttempt(peer, attempt) { + state.resumeAttempts.push(attempt) + console.log(JSON.stringify({ event: 'relay_load_resume_attempt', index: peer.index, ...attempt })) +} +// A paired phone that reconnects every interval until the run stops; every attempt is a line. +async function runResumePhone(peer, localIndex, endsAt) { + const relayDeviceId = `load-resume-device-${peer.index}` + // Spread the phones across one interval. + await delay(Math.floor((localIndex / Math.max(1, config.resumePhones)) * config.resumeIntervalMs)) + let token + while (!state.stopping && Date.now() < endsAt) { + const startedAt = Date.now() + if (!token) { + try { + token = await installRelayLoadResumeDevice(peer, relayDeviceId) + } catch (error) { + recordResumeAttempt(peer, { + at: startedAt, + ok: false, + failure: 'install', + error: String(error.message).slice(0, 160) + }) + } + } else { + recordResumeAttempt(peer, await resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, { + holdMs: config.resumeHoldMs, + stopping: () => state.stopping, + delay + })) + } + await delay(Math.min( + Math.max(1_000, config.resumeIntervalMs - (Date.now() - startedAt)), + Math.max(0, endsAt - Date.now()) + )) + } +} const progressTimer = setInterval(() => report(state), 10_000) progressTimer.unref() await runRelayLoadWithShutdown(async () => { @@ -593,7 +637,11 @@ await runRelayLoadWithShutdown(async () => { }) ) ) - await Promise.all([...splicePromises, delay(config.durationMs)]) + const resumeEndsAt = Date.now() + config.durationMs + const resumePromises = relayLoadResumePhoneIndexes(config).map((index, localIndex) => + runResumePhone(peers.get(index), localIndex, resumeEndsAt) + ) + await Promise.all([...splicePromises, ...resumePromises, delay(config.durationMs)]) }, async () => { state.stopping = true clearInterval(progressTimer) diff --git a/cloud/dev/scripts/operate-relay-cell-flags.mjs b/cloud/dev/scripts/operate-relay-cell-flags.mjs index 6eebd7cd2ed7..5456cc59fca6 100644 --- a/cloud/dev/scripts/operate-relay-cell-flags.mjs +++ b/cloud/dev/scripts/operate-relay-cell-flags.mjs @@ -43,7 +43,9 @@ export const CELL_FLAG_SPECS = { // Absent keeps the cell's boot value. readTimeoutMarginMs: { parse: numberIn(1_000, 60_000, true) }, // Flip-back leases in flight; absent is a third of the cell's pool. - reregisterInFlight: { parse: numberIn(1, 16, true) } + reregisterInFlight: { parse: numberIn(1, 16, true) }, + // Step 6: acts only while the cell is reserve. + phonePath: { parse: enumOf('db', 'memory'), default: 'db' } } export const RESERVE_CONFIRMATION = 'RESERVE' @@ -136,6 +138,24 @@ export function assertReserveKeepsLocalReadiness(flags, changes) { ) } +// The cell ignores phonePath=memory outside reserve, so writing it there only arms a later +// admitMode=reserve flip to turn phones over unnamed. Rolling back with admitMode=db stays allowed. +export function assertPhonePathNeedsReserve(flags, changes) { + if ((flags.phonePath ?? 'db') !== 'memory') return + if ((flags.admitMode ?? 'db') === 'reserve') { + if (changes.admitMode === 'reserve' && !('phonePath' in changes)) { + throw new Error( + 'the object still says phonePath=memory: name phonePath in the same write ' + + '(e.g. --set admitMode=reserve,phonePath=default), so a reserve flip never turns phones over unnamed' + ) + } + return + } + if ('phonePath' in changes) { + throw new Error("phonePath=memory requires admitMode=reserve on the cell, already written or in the same write") + } +} + // `key=value,key=value`; `key=default` removes the key so the cell uses its default. export function parseFlagChanges(text) { const changes = {} @@ -152,7 +172,8 @@ export function parseFlagChanges(text) { } const value = spec.parse(raw) if (value === undefined) throw new Error(`switch ${key} cannot be ${JSON.stringify(raw)}`) - changes[key] = value + // phonePath=db is the default, stored as no key: an image without the switch refuses the key. + changes[key] = key === 'phonePath' && value === 'db' ? null : value } if (Object.keys(changes).length === 0) throw new Error('set names no switch to change') return changes @@ -418,7 +439,8 @@ async function breakGlassToDatabase(request, dependencies, { fetchImpl, log, sle if (current.generation !== request.expectedGeneration) { throw generationConflict(current.generation, request.expectedGeneration) } - const object = desiredObject(request, current.object) + // phonePath goes too: a dead cell comes back with phones on the database path as well. + const object = desiredObject({ ...request, changes: { ...request.changes, phonePath: null } }, current.object) const generation = await writeObject(fetchImpl, request, object, accessToken, audit) await directorAdmitMode(fetchImpl, request, idToken, sleep, 'db') const result = { event: 'orca_relay_cell_admit_mode_break_glass', cellId: request.cellId, generation, audit } @@ -453,6 +475,7 @@ export async function operateCellFlags(request, dependencies) { ) } assertReserveKeepsLocalReadiness(object.flags, request.changes) + assertPhonePathNeedsReserve(object.flags, request.changes) const plan = { event: 'orca_relay_cell_flags_plan', mode: request.mode, diff --git a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs index 1b72cb6aebf7..5d30e7e9483f 100644 --- a/cloud/dev/scripts/operate-relay-cell-flags.test.mjs +++ b/cloud/dev/scripts/operate-relay-cell-flags.test.mjs @@ -36,7 +36,8 @@ const SUPPORTED = { reserveDryRun: { type: 'boolean' }, rejectionFence: { type: 'boolean' }, readTimeoutMarginMs: { type: 'number', min: 1_000, max: 60_000, integer: true }, - reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true } + reregisterInFlight: { type: 'number', min: 1, max: 16, integer: true }, + phonePath: { type: 'enum', values: ['db', 'memory'] } } function fakeGoogleAndCell({ @@ -499,6 +500,54 @@ test('a reserve cell refuses readinessLocal=false, but takes it together with th assert.deepEqual(fake.state.writes[0].object.flags, { readinessLocal: false, admitMode: 'db' }) }) +test('phonePath=memory needs reserve, and a reserve flip never turns phones over unnamed', async () => { + const onDatabase = { generation: '7', object: { v: 1, cellId: CELL, flags: { readinessLocal: true } } } + const refused = fakeGoogleAndCell({ stored: onDatabase }) + await assert.rejects( + run(parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })), refused).result, + /phonePath=memory requires admitMode=reserve/ + ) + assert.deepEqual(refused.state.writes, []) + + const together = fakeGoogleAndCell({ stored: onDatabase }) + const both = parseCellFlagsRequest( + values({ set: 'admitMode=reserve,phonePath=memory', 'expected-generation': '7', ...RESERVE }) + ) + assert.equal((await run(both, together).result).written, true) + assert.deepEqual(together.state.writes[0].object.flags, { + readinessLocal: true, + admitMode: 'reserve', + phonePath: 'memory' + }) + + const reserve = { readinessLocal: true, admitMode: 'reserve' } + const onReserve = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: reserve } } }) + const alone = parseCellFlagsRequest(values({ set: 'phonePath=memory', 'expected-generation': '7' })) + assert.equal((await run(alone, onReserve).result).written, true) + + // Rolling back with admitMode=db leaves phonePath=memory in the object, which the cell ignores. + const memory = { ...reserve, phonePath: 'memory' } + const rollback = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } }) + rollback.state.directorAdmitMode = 'reserve' + const toDatabase = parseCellFlagsRequest(values({ ...DIRECTOR, set: 'admitMode=db', 'expected-generation': '7' })) + assert.equal((await run(toDatabase, rollback).result).written, true) + assert.deepEqual(rollback.state.writes[0].object.flags, { ...memory, admitMode: 'db' }) + + // phonePath=db removes the key, so an image without the switch still takes later writes. + const off = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: memory } } }) + assert.equal((await run(parseCellFlagsRequest(values({ set: 'phonePath=db', 'expected-generation': '7' })), off).result).written, true) + assert.deepEqual(off.state.writes[0].object.flags, reserve) + + const lingering = { readinessLocal: true, admitMode: 'db', phonePath: 'memory' } + const unnamed = fakeGoogleAndCell({ stored: { ...onDatabase, object: { ...onDatabase.object, flags: lingering } } }) + await assert.rejects( + run(parseCellFlagsRequest(values({ set: 'admitMode=reserve', 'expected-generation': '7', ...RESERVE })), unnamed) + .result, + /still says phonePath=memory/ + ) + assert.deepEqual(unnamed.state.writes, []) +}) + test('a migration opened during the cache wait puts Postgres back to db', async () => { const fake = fakeGoogleAndCell({ refuseReservePost: { post: 2, error: 'cell_has_open_migration' } }) const request = parseCellFlagsRequest(values({ ...RESERVE, set: 'readinessLocal=true,admitMode=reserve' })) @@ -537,7 +586,7 @@ test('a flip back that outlasts its wait keeps Postgres on reserve and says how assert.equal(lines.find((line) => line.event === 'orca_relay_cell_reregistration_wait')?.waitMs, 1_666_667) }) -test('break glass records db for a cell that cannot answer, typed for that cell only', async () => { +test('break glass records db (and drops phonePath) for a cell that cannot answer, typed for that cell only', async () => { const glass = { ...RESERVE, set: 'admitMode=default', 'expected-generation': '7' } assert.throws(() => parseCellFlagsRequest(values({ ...glass, 'break-glass': 'BREAK GLASS production-gce-c27' })), /break glass requires/) assert.throws( @@ -545,7 +594,7 @@ test('break glass records db for a cell that cannot answer, typed for that cell /on its own/ ) const fake = fakeGoogleAndCell({ - stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true } } } + stored: { generation: '7', object: { v: 1, cellId: CELL, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } } }) fake.state.directorAdmitMode = 'reserve' const inner = fake.fetchImpl diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs index d4338494b5f1..96c6ccf2f22d 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs @@ -6,6 +6,8 @@ import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/ const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' +const SOURCE_TRANSIENT_RETRIES = 12 +const SOURCE_TRANSIENT_RETRY_DELAY_MS = 10_000 export function parseRehomeTrustProbeArguments(argv, environment = process.env) { const values = {} @@ -56,11 +58,19 @@ export async function probeRehomeTrust(config, dependencies = {}) { }, { wait: dependencies.wait } ) + const wait = dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) let response = await request() let body = await response.json().catch(() => ({})) - // The director wraps source HTTP failures in 409; retry only explicit transient statuses. - if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) { - await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000) + // The director wraps source HTTP failures in 409; retry only explicit transient statuses. The + // load balancer can answer 503 for a minute after a fresh instance is ready (c10, 2026-10-20). + for ( + let retry = 0; + retry < SOURCE_TRANSIENT_RETRIES && + response.status === 409 && + /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? ''); + retry++ + ) { + await wait(SOURCE_TRANSIENT_RETRY_DELAY_MS) response = await request() body = await response.json().catch(() => ({})) } diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs index f70087a4aeca..9e31864956df 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs @@ -165,13 +165,27 @@ test('reports safe trust reasons, keeps rejection final, and redacts arbitrary e } }) -test('stops after the second wrapped transient failure', async () => { +test('outlasts a load-balancer warm-up of consecutive wrapped source 503s', async () => { + let calls = 0 + const waits = [] + const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { + wait: async (ms) => { waits.push(ms) }, + fetch: async () => ++calls <= 6 + ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) + : Response.json(provenProbe) + }) + assert.equal(calls, 7) + assert.deepEqual(waits, Array(6).fill(10_000)) + assert.equal(result.proven, true) +}) + +test('stops after two minutes of wrapped transient failures', async () => { let calls = 0 await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { wait: async () => {}, fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) } }), /returned 409.*source_503/) - assert.equal(calls, 2) + assert.equal(calls, 13) }) // The probe allowlist is the committed rehome source list, so a declared source can always probe. diff --git a/cloud/dev/scripts/relay-evidence-code-provenance.mjs b/cloud/dev/scripts/relay-evidence-code-provenance.mjs index 3e6ccb913103..dba11559e6fa 100644 --- a/cloud/dev/scripts/relay-evidence-code-provenance.mjs +++ b/cloud/dev/scripts/relay-evidence-code-provenance.mjs @@ -33,6 +33,8 @@ export const TRUSTED_EVIDENCE_CODE_PATHS = [ relayTreePath('dev/scripts/probe-relay-rehome-trust.mjs'), relayTreePath('dev/scripts/validate-relay-capacity-plan.mjs'), relayTreePath('dev/scripts/verify-relay-capacity-transition.mjs'), + relayTreePath('dev/scripts/relay-admin-transient-retry.mjs'), + relayTreePath('dev/scripts/relay-reserve-cell-refusal.mjs'), // The exit metric filter the same-cap pre-drain sample's crash rule reads. relayTreePath('infra/terraform/relay-observability.tf'), // The monitor, the live preflight recheck, and the same-cap pre-drain sample, plus anything that diff --git a/cloud/dev/scripts/relay-load-control-peer.mjs b/cloud/dev/scripts/relay-load-control-peer.mjs index e250c0f65447..d58fc0c10b64 100644 --- a/cloud/dev/scripts/relay-load-control-peer.mjs +++ b/cloud/dev/scripts/relay-load-control-peer.mjs @@ -20,7 +20,7 @@ const REPORTABLE_ASSIGNMENT_ERRORS = [ 'relay_home_cell_unavailable' ] -function waitForOpen(socket, timeoutMs = 10_000) { +export function waitForOpen(socket, timeoutMs = 10_000) { return new Promise((resolve, reject) => { const timer = setTimeout(() => finish(new Error('control open timeout')), timeoutMs) const finish = (error) => { @@ -40,7 +40,7 @@ function waitForOpen(socket, timeoutMs = 10_000) { }) } -function nextJson(socket, timeoutMs = 10_000) { +export function nextJson(socket, timeoutMs = 10_000) { return new Promise((resolve, reject) => { const timer = setTimeout(() => finish(new Error('control response timeout')), timeoutMs) const finish = (error, value) => { @@ -66,7 +66,7 @@ function nextJson(socket, timeoutMs = 10_000) { }) } -function nextFrame(socket, timeoutMs = 10_000) { +export function nextFrame(socket, timeoutMs = 10_000) { return new Promise((resolve, reject) => { const timer = setTimeout(() => finish(new Error('relay frame timeout')), timeoutMs) const finish = (error, value) => { diff --git a/cloud/dev/scripts/relay-load-profile.mjs b/cloud/dev/scripts/relay-load-profile.mjs index 076a7407b0f2..7d01a0501d9e 100644 --- a/cloud/dev/scripts/relay-load-profile.mjs +++ b/cloud/dev/scripts/relay-load-profile.mjs @@ -81,6 +81,9 @@ export function parseRelayLoadArguments(argv) { values.get('--duration-seconds') ?? 900 ) * 1000 const splices = integer('--splices', 0) + const resumePhones = integer('--resume-phones', 0) + const resumeIntervalMs = integer('--resume-interval-seconds', 20) * 1000 + const resumeHoldMs = integer('--resume-hold-seconds', 10) * 1000 const spliceRampMs = integer('--splice-ramp-seconds', 0) * 1000 const slowReaderSplices = integer('--slow-reader-splices', 0) const wedgedReaderSplices = integer('--wedged-reader-splices', 0) @@ -112,6 +115,12 @@ export function parseRelayLoadArguments(argv) { if (controls < 1 || controls > 10_000) throw new Error('--controls must be between 1 and 10000') if (rebindProbes > controls) throw new Error('--rebind-probes cannot exceed --controls') if (splices > controls) throw new Error('--splices cannot exceed --controls') + if (splices + resumePhones > controls) { + throw new Error('--splices plus --resume-phones cannot exceed --controls') + } + if (resumePhones > 0 && resumeIntervalMs < 1_000) { + throw new Error('--resume-interval-seconds must be at least 1') + } if (shardCount > 1 && capacityHardCap !== undefined) { if (!values.has('--aggregate-controls') || !values.has('--aggregate-splices')) { throw new Error('capacity-bound sharding requires explicit aggregate controls and splices') @@ -320,6 +329,9 @@ export function parseRelayLoadArguments(argv) { aggregateReaderBytes, signingKeyFile: values.get('--signing-key-file'), splices, + resumePhones, + resumeIntervalMs, + resumeHoldMs, spliceRampMs, splicePayloadBytes, slowReaderSplices, @@ -348,6 +360,14 @@ export function relayLoadSpliceIndexes(config) { ) } +// Resume phones ride the controls after the splice controls, so the two never share a peer. +export function relayLoadResumePhoneIndexes(config) { + return Array.from( + { length: config.resumePhones }, + (_, localIndex) => (config.splices + localIndex) * config.shardCount + config.shardIndex + ) +} + export function relayLoadSpliceStartDelayMs(config, localIndex) { if (!Number.isSafeInteger(localIndex) || localIndex < 0 || localIndex >= config.splices) { throw new Error('invalid local splice index') diff --git a/cloud/dev/scripts/relay-load-profile.test.mjs b/cloud/dev/scripts/relay-load-profile.test.mjs index 9763d6a699f3..c5508453e765 100644 --- a/cloud/dev/scripts/relay-load-profile.test.mjs +++ b/cloud/dev/scripts/relay-load-profile.test.mjs @@ -4,6 +4,7 @@ import { parseRelayLoadArguments, relayLoadPrincipalIndex, relayLoadReaderEvidenceError, + relayLoadResumePhoneIndexes, relayLoadSpliceIndexes, relayLoadSpliceProfile, relayLoadSpliceStartDelayMs @@ -214,6 +215,20 @@ test('maps splice ownership deterministically within a shard', () => { assert.deepEqual(relayLoadSpliceIndexes(config), [2, 6, 10]) }) +test('puts resume phones on the controls after the splice controls', () => { + const config = parseRelayLoadArguments([ + ...required, '--controls', '4', '--splices', '1', '--resume-phones', '2', + '--shard-count', '4', '--shard-index', '2' + ]) + + assert.deepEqual(relayLoadSpliceIndexes(config), [2]) + assert.deepEqual(relayLoadResumePhoneIndexes(config), [6, 10]) + assert.throws( + () => parseRelayLoadArguments([...required, '--controls', '2', '--splices', '1', '--resume-phones', '2']), + /--splices plus --resume-phones cannot exceed --controls/ + ) +}) + test('staggered shards form one deterministic splice ramp', () => { const config = parseRelayLoadArguments([ ...required, diff --git a/cloud/dev/scripts/relay-load-resume-phone.mjs b/cloud/dev/scripts/relay-load-resume-phone.mjs new file mode 100644 index 000000000000..c010f27b9e31 --- /dev/null +++ b/cloud/dev/scripts/relay-load-resume-phone.mjs @@ -0,0 +1,220 @@ +import { createHash, randomBytes } from 'node:crypto' +import { nextFrame, nextJson, waitForOpen } from './relay-load-control-peer.mjs' + +// A paired phone's reconnects: one device install over an invite splice, then real device +// resumes (accept, then the desktop's device-resume-confirm), timed from the phone's side. + +const ECHO_INTERVAL_MS = 5_000 + +const resumeTokenHash = (token) => createHash('sha256').update(token).digest('base64url') + +function controlAnswer(peer, reqId, type) { + return peer.waitForControlMessage( + (message) => + message.reqId === reqId && (message.type === type || message.type === 'control-error') + ) +} + +function closeCode(socket, timeoutMs = 2_000) { + if (socket.readyState === socket.CLOSED) return Promise.resolve(undefined) + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(undefined), timeoutMs) + socket.once('close', (code) => { + clearTimeout(timer) + resolve(code) + }) + }) +} + +// The phone dials, the desktop's control answers conn-open, and the desktop attaches its data leg. +async function dialPhone(peer, credential, relayDeviceId) { + if (!peer.socket || peer.socket.readyState !== peer.socket.OPEN || peer.generation === undefined) { + throw Object.assign(new Error('active control required for a phone'), { reason: 'no-control' }) + } + const assignment = peer.lastAssignment + const phone = peer.createClientSocket(assignment) + peer.trackSpliceSocket(phone) + let data + try { + const setData = (socket) => { + data = socket + } + return await attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData }) + } catch (error) { + // A close before any hello names its code, e.g. 4503 while the database is down. + const closed = /closed: (\d+)/.exec(String(error.message)) + throw Object.assign(error, { + reason: error.reason ?? (closed ? `hello-${closed[1]}` : undefined), + phone, + data + }) + } +} + +async function attachPhone(peer, { assignment, phone, credential, relayDeviceId, setData }) { + await waitForOpen(phone) + const connectionPromise = peer.waitForControlMessage( + (message) => message.type === 'conn-open' && message.relayDeviceId === relayDeviceId + ) + connectionPromise.catch(() => undefined) + const hello = nextJson(phone) + const startedAt = performance.now() + phone.send(JSON.stringify({ type: 'relay-auth', v: 1, mode: 'connect', credential })) + // A refused phone gets a failed hello and no conn-open. + const first = await Promise.race([ + connectionPromise.then((connection) => ({ connection })), + hello.then((answer) => ({ answer })) + ]) + if (first.answer) { + const code = first.answer.code ?? (await closeCode(phone)) + throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` }) + } + const { connection } = first + const data = peer.createHostDataSocket(assignment, connection.connId) + setData(data) + peer.trackSpliceSocket(data) + await waitForOpen(data) + data.send( + JSON.stringify({ + type: 'host-data-auth', + v: 1, + connTicket: connection.connTicket, + generation: peer.generation + }) + ) + const answer = await hello + const acceptMs = performance.now() - startedAt + if (answer.ok !== true) { + const code = answer.code ?? (await closeCode(phone)) + throw Object.assign(new Error(`phone refused: ${code}`), { reason: `hello-${code}` }) + } + return { phone, data, connId: connection.connId, acceptMs } +} + +async function echo(phone, data, payload) { + const toHost = nextFrame(data) + phone.send(payload) + if ((await toHost).bytes.toString() !== payload) throw new Error('relay changed phone payload') + const toPhone = nextFrame(phone) + data.send(Buffer.from(payload)) + if ((await toPhone).bytes.toString() !== payload) throw new Error('relay changed host payload') +} + +function rttMs(socket, timeoutMs = 5_000) { + return new Promise((resolve) => { + const startedAt = performance.now() + const timer = setTimeout(() => finish(undefined), timeoutMs) + const finish = (value) => { + clearTimeout(timer) + socket.off('pong', onPong) + resolve(value) + } + const onPong = () => finish(performance.now() - startedAt) + socket.on('pong', onPong) + socket.ping() + }) +} + +export async function installRelayLoadResumeDevice(peer, relayDeviceId) { + const inviteReqId = `load-resume-invite-${peer.index}-${peer.spliceSequence++}` + const invite = controlAnswer(peer, inviteReqId, 'invite-created') + peer.socket.send(JSON.stringify({ type: 'invite-create', reqId: inviteReqId, relayDeviceId })) + const created = await invite + if (created.type !== 'invite-created') throw new Error(`invite failed: ${created.code}`) + const splice = await dialPhone(peer, created.inviteToken, relayDeviceId) + try { + const token = randomBytes(32).toString('base64url') + const reqId = `load-resume-install-${peer.index}-${peer.spliceSequence++}` + const installed = controlAnswer(peer, reqId, 'device-credential-installed') + peer.socket.send( + JSON.stringify({ + type: 'device-credential-install', + v: 1, + reqId, + relayDeviceId, + newResumeTokenHash: resumeTokenHash(token), + authorization: { mode: 'relay-basis', basisConnId: splice.connId } + }) + ) + const result = await installed + if (result.type !== 'device-credential-installed') { + throw new Error(`device install failed: ${result.code}`) + } + return token + } finally { + await Promise.all([peer.closeSpliceSocket(splice.phone), peer.closeSpliceSocket(splice.data)]) + } +} + +// One resume: accept, confirm, then hold the splice with periodic echoes. Never throws. +export async function resumeRelayLoadPhoneOnce(peer, { token, relayDeviceId }, { holdMs, stopping, delay }) { + const attempt = { at: Date.now(), cellUrl: peer.lastAssignment?.cellUrl } + let splice + try { + splice = await dialPhone(peer, token, relayDeviceId) + attempt.acceptMs = Math.round(splice.acceptMs) + const reqId = `load-resume-confirm-${peer.index}-${peer.spliceSequence++}` + const confirmed = controlAnswer(peer, reqId, 'device-resume-confirmed') + const confirmStartedAt = performance.now() + peer.socket.send( + JSON.stringify({ type: 'device-resume-confirm', v: 1, reqId, basisConnId: splice.connId }) + ) + const result = await confirmed + attempt.confirmMs = Math.round(performance.now() - confirmStartedAt) + if (result.type !== 'device-resume-confirmed') { + attempt.confirmFailure = String(result.code) + } else { + attempt.renewed = result.renewed + } + const rtt = await rttMs(splice.phone) + if (rtt !== undefined) attempt.rttMs = Math.round(rtt) + const holdUntil = Date.now() + holdMs + attempt.echoes = 0 + while (!stopping() && Date.now() < holdUntil) { + await echo(splice.phone, splice.data, `orca-relay-resume:${peer.index}:${attempt.echoes}`) + attempt.echoes++ + await delay(Math.min(ECHO_INTERVAL_MS, Math.max(0, holdUntil - Date.now()))) + } + attempt.ok = attempt.confirmFailure === undefined + } catch (error) { + attempt.ok = false + attempt.failure = error.reason ?? (splice ? 'held-splice' : 'accept') + attempt.error = String(error.message).slice(0, 160) + if (splice) attempt.heldCloseCode = await closeCode(splice.phone, 100) + splice ??= { phone: error.phone, data: error.data } + } finally { + await Promise.all([peer.closeSpliceSocket(splice?.phone), peer.closeSpliceSocket(splice?.data)]) + } + return attempt +} + +const percentile = (values, fraction) => { + if (values.length === 0) return undefined + const sorted = [...values].sort((left, right) => left - right) + return sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))] +} + +export function relayLoadResumeSummary(attempts) { + const accepts = attempts.flatMap((attempt) => + attempt.acceptMs === undefined ? [] : [attempt.acceptMs] + ) + const confirms = attempts.flatMap((attempt) => + attempt.confirmMs === undefined || attempt.confirmFailure ? [] : [attempt.confirmMs] + ) + const rtts = attempts.flatMap((attempt) => (attempt.rttMs === undefined ? [] : [attempt.rttMs])) + const failures = {} + for (const attempt of attempts) { + const reason = attempt.confirmFailure ? `confirm-${attempt.confirmFailure}` : attempt.failure + if (reason) failures[reason] = (failures[reason] ?? 0) + 1 + } + return { + attempts: attempts.length, + ok: attempts.filter((attempt) => attempt.ok).length, + acceptMsP50: percentile(accepts, 0.5), + acceptMsP95: percentile(accepts, 0.95), + confirmMsP50: percentile(confirms, 0.5), + confirmMsP95: percentile(confirms, 0.95), + rttMsP50: percentile(rtts, 0.5), + failures + } +} diff --git a/cloud/dev/scripts/relay-load-resume-phone.test.mjs b/cloud/dev/scripts/relay-load-resume-phone.test.mjs new file mode 100644 index 000000000000..26fd3a1938b0 --- /dev/null +++ b/cloud/dev/scripts/relay-load-resume-phone.test.mjs @@ -0,0 +1,24 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { relayLoadResumeSummary } from './relay-load-resume-phone.mjs' + +test('summarizes resume latency from completed steps and failures by reason', () => { + const summary = relayLoadResumeSummary([ + { ok: true, acceptMs: 100, confirmMs: 20, rttMs: 10 }, + { ok: true, acceptMs: 300, confirmMs: 40, rttMs: 10 }, + { ok: false, acceptMs: 200, confirmMs: 900, confirmFailure: 'database_unavailable' }, + { ok: false, failure: 'hello-4503' }, + { ok: false, failure: 'hello-4503' } + ]) + + assert.deepEqual(summary, { + attempts: 5, + ok: 2, + acceptMsP50: 200, + acceptMsP95: 300, + confirmMsP50: 40, + confirmMsP95: 40, + rttMsP50: 10, + failures: { 'confirm-database_unavailable': 1, 'hello-4503': 2 } + }) +}) diff --git a/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json new file mode 100644 index 000000000000..0593c68ab634 --- /dev/null +++ b/cloud/dev/scripts/relay-phone-watch-recorded-quiet-minutes.json @@ -0,0 +1 @@ +{"source":"production cell runtime metrics, read-only, 2026-10-18T06:00Z to 2026-10-19T08:00Z; per-minute sums of authFailuresDelta and clientAcceptCompletedDelta by the minute each 30 s line ends in","cells":{"production-gce-c7":{"authFailuresPerMinute":[0,1,0,1,2,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,1,1,0,2,1,1,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,2,1,0,1,1,0,2,2,1,0,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,2,1,1,0,3,2,2,1,3,5,1,0,1,2,1,2,1,2,0,5,4,1,1,4,0,1,0,1,3,2,4,2,1,3,2,5,1,3,1,2,1,1,2,2,1,1,3,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,4,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,2,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,2,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,2,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,1,0,2,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,1,0,2,2,2,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,2,0,1,0,1,0,0,0,0,0,1,0,3,0,1,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,4,1,1,1,3,1,1,1,0,1,1,0,1,2,2,2,0,0,1,6,0,1,1,1,1,2,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,1,0,4,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0],"completedAcceptsPerMinute":[2,0,1,2,0,2,1,5,1,1,2,3,1,4,0,0,2,4,1,0,0,0,1,1,2,2,2,1,2,2,1,1,0,0,2,0,0,2,1,1,1,2,0,0,1,2,1,0,2,1,0,1,1,0,1,0,1,0,1,0,0,0,1,1,4,0,1,0,1,0,2,0,0,0,0,0,2,2,0,1,1,1,1,1,0,0,0,1,2,1,0,0,0,1,0,1,3,0,0,3,0,0,1,1,0,2,1,1,2,1,2,0,1,1,2,1,3,2,1,1,0,1,0,1,2,4,2,3,4,0,1,1,3,2,0,1,2,1,2,2,2,3,4,2,3,1,3,1,0,2,2,0,0,3,1,1,2,1,3,2,2,2,2,3,1,3,1,3,3,1,1,2,0,0,3,0,4,2,3,3,1,2,2,4,3,3,2,3,4,4,0,4,4,3,2,1,0,2,1,2,3,2,3,3,1,4,2,2,2,3,3,1,4,2,1,4,2,3,4,2,0,1,4,2,1,0,0,2,0,1,4,2,1,4,3,3,3,0,2,3,3,3,2,3,5,2,5,3,1,3,0,5,2,1,4,1,6,1,3,7,4,4,1,4,3,1,1,2,2,3,3,4,0,3,0,2,2,3,2,1,2,3,0,2,0,2,2,1,6,2,4,1,2,4,1,1,3,1,1,3,3,5,3,1,2,2,2,3,2,2,3,2,5,6,2,3,3,2,5,3,2,1,5,3,2,3,6,8,6,1,2,2,1,5,6,8,1,2,7,6,4,3,5,4,6,2,5,5,2,3,2,1,5,1,2,3,6,4,1,4,1,1,5,2,1,3,2,4,3,2,1,1,4,4,2,3,4,2,3,3,1,2,0,4,1,4,3,2,0,4,2,2,1,2,2,1,1,6,2,4,3,1,0,1,1,4,0,1,1,4,1,4,4,1,1,3,2,2,3,3,1,0,1,2,0,2,2,1,2,1,2,2,1,7,4,1,3,0,0,2,2,1,1,1,0,3,7,2,2,0,0,3,0,2,2,1,1,1,4,2,1,3,2,4,0,0,1,5,3,4,2,5,4,2,1,1,2,7,4,6,1,4,2,1,5,0,2,5,2,0,2,3,4,4,1,0,1,0,4,1,4,3,2,2,3,3,2,3,3,3,2,0,1,3,4,3,2,5,3,1,6,2,4,2,3,2,2,2,3,1,2,2,0,6,2,0,2,4,5,1,1,0,2,5,1,2,4,3,4,2,0,2,1,3,4,3,3,3,2,3,5,2,6,2,3,1,3,4,3,1,2,1,3,5,4,0,2,1,1,0,1,1,3,3,3,2,3,2,2,0,2,2,1,4,1,2,4,2,3,1,2,2,3,1,5,0,4,6,1,0,2,5,3,3,5,0,3,1,3,3,7,3,3,1,5,4,4,6,1,6,1,5,2,1,1,1,1,1,4,2,3,5,2,2,1,4,1,1,2,2,1,2,0,0,3,4,2,0,2,4,5,3,3,1,1,1,2,3,1,2,1,4,2,1,3,0,0,2,2,0,0,0,1,1,0,0,0,0,4,4,3,1,1,1,2,1,0,1,1,1,2,4,2,3,3,1,1,2,4,0,1,1,1,2,2,4,2,2,3,2,2,3,1,1,1,5,4,3,4,1,3,3,1,1,1,0,4,3,5,3,1,4,2,3,1,2,3,5,2,3,2,2,4,1,2,5,2,1,4,1,2,3,4,3,2,3,2,2,3,1,0,0,0,1,2,2,0,3,1,2,1,0,0,4,1,0,4,0,0,1,3,0,0,1,1,1,2,0,3,2,0,2,1,3,3,0,2,0,1,2,4,4,3,1,3,5,1,3,0,1,1,3,4,1,0,3,1,3,4,1,3,2,3,3,1,3,3,3,4,7,2,4,3,4,1,2,0,2,3,1,2,3,2,3,0,2,2,2,7,2,3,1,1,3,2,1,3,3,1,1,1,2,4,1,2,6,1,0,0,3,0,4,0,0,2,2,3,3,1,2,0,3,1,1,3,1,5,1,2,4,2,1,1,3,6,5,2,0,4,2,5,0,2,1,4,2,2,3,2,1,7,3,5,0,4,3,0,2,1,3,2,2,1,1,0,2,1,4,2,2,1,1,2,4,3,1,0,1,2,1,2,0,1,5,0,3,3,2,2,1,0,2,1,5,0,5,1,3,2,3,2,1,3,2,3,0,0,2,2,1,1,2,0,0,3,3,2,2,0,5,2,5,2,2,4,1,2,5,0,3,4,2,1,5,3,1,5,3,0,1,3,4,5,2,2,2,4,1,1,1,3,0,2,1,5,1,1,1,1,1,4,3,1,1,1,1,4,3,0,2,5,2,4,3,1,0,3,2,4,5,1,4,2,2,4,2,3,3,4,3,1,3,1,1,4,1,3,1,2,0,4,4,3,3,2,1,2,1,1,2,0,3,1,2,3,2,1,1,0,1,1,0,1,2,3,1,3,3,1,0,2,4,1,2,3,2,1,1,2,2,1,0,1,1,3,0,0,0,3,3,3,1,0,4,0,3,1,3,1,1,4,2,3,0,1,2,2,1,0,0,4,1,3,0,3,2,2,2,0,1,1,1,1,2,3,2,0,1,2,0,2,2,1,0,1,0,2,3,2,2,1,3,3,0,2,3,2,2,6,3,1,1,0,2,2,1,0,2,3,1,0,0,1,2,1,1,0,0,1,0,1,0,2,2,3,1,2,0,1,0,0,3,0,0,2,1,2,0,2,0,2,1,3,0,0,2,3,0,1,0,0,0,0,2,1,0,4,2,0,1,0,3,3,2,0,1,2,2,1,1,1,0,1,2,2,1,3,2,1,3,0,1,1,2,2,0,2,0,2,2,1,1,2,1,0,0,1,1,2,1,0,2,1,3,2,0,2,4,2,1,0,2,1,1,1,0,2,2,1,2,2,3,0,1,2,2,3,3,2,1,2,2,5,0,0,3,1,3,0,1,1,3,3,2,2,1,0,2,3,4,2,4,2,1,1,2,1,0,2,1,0,1,2,1,3,1,4,2,4,1,3,3,2,1,3,0,3,1,3,0,1,3,4,1,5,0,4,2,4,6,3,1,1,1,4,2,2,5,3,0,0,2,1,1,0,1,2,3,0,3,0,1,2,0,4,1,2,2,2,4,1,1,0,4,1,1,1,3,0,1,2,3,2,2,4,1,2,1,1,2,3,2,2,2,1,0,0,0,0,3,0,1,2,0,1,2,1,2,1,0,2,4,3,4,4,3,4,3,3,3,2,0,0,0,1,3,1,2,4,0,4,4,1,1,3,1,1,1,0,1,1,2,2,3,0,1,0,3,0,0,1,2,1,2,2,4,1,1,3,1,0,0,3,1,2,1,1,2,3,2,3,0,0,7,2,2,0,3,1,0,3,1,1,1,2,1,3,0,4,1,1,0,2,4,1,2,1,0,4,2,2,0,1,1,2,1,1,0,1,1,1,3,0,1,1,1,4,2,0,2,2,2,2,4,0,2,1,1,3,0]},"production-gce-c28":{"authFailuresPerMinute":[2,0,0,0,0,0,6,4,3,3,5,3,2,5,5,3,2,1,0,1,1,0,0,1,1,3,2,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,2,0,0,0,0,0,2,0,1,0,0,0,1,0,0,0,1,1,0,2,0,1,0,0,0,0,1,1,3,0,1,1,2,1,1,0,0,0,1,7,3,5,4,5,6,3,0,1,3,2,2,1,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,2,0,0,0,0,0,3,0,0,1,1,0,0,2,0,0,1,3,1,4,5,1,5,6,3,5,2,0,0,0,0,0,8,4,4,8,8,10,7,7,4,7,0,1,1,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,1,1,5,12,8,5,4,3,5,5,3,3,0,1,3,3,0,0,1,1,2,2,1,0,0,0,1,0,2,0,1,0,1,0,0,0,0,0,0,1,1,0,0,4,2,0,2,0,0,0,0,0,0,0,0,0,3,0,0,1,0,0,1,0,0,2,5,3,0,0,0,0,1,0,0,0,1,1,2,2,2,1,0,0,0,3,0,0,1,0,0,0,0,0,0,0,1,0,2,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,1,0,1,0,2,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,2,0,1,0,1,0,0,0,0,0,0,0,0,4,3,2,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,2,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,2,8,4,3,2,1,1,0,1,0,0,2,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,7,2,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,2,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,1,1,1,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,2,1,0,0,2,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,1,2,1,0,0,0,0,1,0,0,0,0,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,0,3,7,3,2,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,5,0,0,0,2,0,0,1,1,0,0,0,0,0,0,2,4,2,6,2,3,2,3,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,1,0,1,1,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,6,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1,2,0,0,1,1,1,0,2,1,0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,2,0,1,1,2,0,0,0,2,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,3,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,2,0,0,0,0,1,1,0,0,0,0,1,0,1,1,2,1,1,1,0,1,0,0,1,0,0,1,0,0,1,0,1,1,0,2,1,0,0,0,1,0,0,2,0,0,1,0,0,0,0,2,2,0,0,0,1,0,0,1,2,0,0,0,2,0,0,1,0,2,0,1,3,0,0,0,1,0,1,0,0,0,0,1,2,3,0,0,0,0,0,1,0,0,1,0,1,0,0,6,2,0,5,2,0,0,0,4,5,3,1,0,5,4,0,0,3,0,1,0,1,0,5,0,2,3,6,3,6,2,0,5,3,9,0,0,3,0,1,6,3,0,2,0,5,1,1,1,4,1,1,1,0,0,0,0,1,0,1,2,1,0,5,4,5,2,0,1,0,0,0,1,1,6,5,3,6,8,6,4,9,3,6,3,3,0,3,14,5,11,6,5,11,8,5,6,10,4,1,9,14,7,4,14,6,5,4,4,1,9,9,11,6,1,0,0,2,0,0,0,1,1,0,0,0,1,0,0,1,3,3,1,0,2,2,1,1,0,0,0,0,3,4,0,3,0,2,0,3,1,2,0,0,0,1,1,0,0,1,2,1,2,1,11,5,3,0,0,1,8,8,1,4,4,0,4,0,1,6,7,6,5,9,5,3,3,5,2,7,1,2,3,1,0,1,0,1,0,2,1,5,4,5,2,1,0,0,4,2,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0,0,5,6,0,1,5,2,1,0,0,0,1,2,2,1,0,0,0,2,5,2,0,0,0,0,0,0,0,0,1,2,0,0,0,1,1,0,0,0,2,0,0,0,1,0,4,2,0,1,0,2,1,1,9,2,0,0,0,1,1,0,0,1,3,2,4,3,6,2,2,1,1,0,0,2,1,0,3,6,0,2,1,0,0,0,2,1,2,5,1,2,0,3,0,0,1,0,0,0,1,0,2,2,0,0,0,1,1,0,0,0,0,0,0,1,0,1],"completedAcceptsPerMinute":[17,10,8,8,13,10,8,10,16,16,31,20,15,17,14,14,17,19,22,20,20,12,15,23,12,16,11,20,21,19,23,17,19,25,18,19,13,12,16,8,11,10,15,15,19,6,9,7,7,11,14,16,15,14,9,10,11,7,10,5,12,12,11,12,6,7,10,6,10,4,10,7,10,7,7,13,14,5,11,4,9,7,9,9,10,17,11,14,16,9,15,12,12,1,8,8,9,7,10,17,14,13,7,10,16,13,8,21,18,13,14,7,8,14,10,14,12,14,16,19,16,22,21,15,19,18,16,16,9,8,14,9,10,14,10,12,14,8,7,13,9,9,10,8,9,7,14,16,15,11,17,6,8,18,6,12,10,5,11,11,8,10,8,7,13,16,8,9,13,9,15,8,18,14,13,9,9,5,7,9,3,8,8,6,7,13,15,17,12,8,9,7,10,11,15,16,12,15,16,16,14,9,6,6,16,11,10,12,16,14,18,11,11,16,11,8,8,2,12,7,10,8,8,13,3,8,8,15,11,6,9,7,11,14,7,15,7,6,6,10,15,8,10,10,10,10,8,7,5,5,11,9,18,12,10,19,16,12,14,8,9,6,10,20,11,10,12,22,11,20,10,17,6,14,19,12,14,7,18,12,9,8,8,11,10,9,7,1,6,6,11,6,6,9,8,13,5,8,16,17,10,8,12,11,8,16,8,9,12,10,7,5,6,8,13,12,8,7,13,6,5,12,11,8,10,14,6,13,11,11,7,10,8,2,2,6,4,9,9,9,7,8,13,6,6,10,6,6,6,5,2,9,7,4,9,10,5,9,7,9,7,4,6,7,13,10,8,6,10,7,8,14,7,9,6,5,7,13,9,14,9,5,12,17,13,19,21,8,15,12,10,18,13,12,11,13,9,8,9,13,8,12,7,6,8,3,4,4,4,9,5,10,8,4,12,13,19,10,8,9,13,4,5,4,4,1,8,9,2,5,7,7,12,10,6,7,4,9,5,9,8,13,11,10,6,11,10,8,6,15,14,9,20,24,17,12,10,6,10,8,10,6,8,5,4,3,12,9,3,7,9,5,5,4,11,6,8,7,6,7,13,4,5,5,9,6,8,4,14,9,11,12,9,5,10,6,6,7,9,7,8,11,5,6,6,9,5,8,11,7,3,5,6,7,7,9,5,12,8,8,7,5,8,5,5,10,4,10,6,6,6,2,3,7,5,12,4,3,7,6,5,5,7,10,2,7,8,12,5,7,11,5,6,8,6,12,10,4,10,7,9,7,12,14,11,14,7,11,4,9,5,7,5,4,6,13,9,3,5,6,6,12,3,4,5,9,11,5,5,4,9,11,16,9,9,13,7,4,5,9,4,5,5,7,4,4,4,1,3,4,11,7,4,3,4,3,5,2,4,3,4,4,3,3,5,4,5,6,7,3,3,2,5,2,6,4,4,3,2,2,4,6,4,1,6,7,2,2,3,4,4,1,2,3,3,4,3,3,6,2,2,4,3,3,1,6,3,1,3,0,7,3,3,2,1,1,1,3,4,1,0,1,1,4,1,1,2,3,4,3,2,2,3,1,4,4,2,0,2,1,0,2,3,1,3,1,1,3,5,3,3,3,2,1,2,6,1,0,1,4,0,5,6,3,2,0,5,1,4,1,0,0,1,4,2,2,4,0,4,0,1,1,1,1,0,1,2,2,2,1,0,1,2,2,5,3,0,1,3,0,3,2,2,4,1,2,1,1,1,2,1,1,2,1,2,4,1,2,0,1,1,0,1,0,1,3,1,1,1,0,2,0,1,0,1,0,0,0,0,3,0,3,3,0,2,1,0,1,0,2,4,6,1,0,3,1,0,0,2,0,2,1,1,2,2,1,0,0,4,1,0,4,2,2,3,1,0,3,1,2,4,2,3,3,1,1,3,5,0,2,3,4,2,0,0,1,1,0,2,1,2,1,0,1,0,1,0,1,1,0,0,2,1,2,2,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,3,0,1,4,0,1,2,0,0,2,0,0,0,1,0,0,0,1,1,0,1,5,1,0,0,0,0,1,2,1,1,1,1,1,1,0,1,1,3,2,1,1,2,0,2,0,3,0,2,1,0,1,1,5,4,3,5,1,0,2,4,1,1,3,5,0,2,0,2,0,2,3,1,1,3,2,2,3,4,3,2,1,4,2,1,2,2,2,2,1,4,1,3,4,2,2,4,2,3,3,3,0,3,2,2,3,3,6,5,7,8,3,2,3,3,2,2,4,2,4,5,1,1,2,2,4,3,4,2,2,4,6,7,5,4,2,3,2,3,6,4,5,7,5,2,9,4,5,7,7,6,5,8,9,7,6,6,6,5,9,9,7,8,4,6,7,4,2,2,5,5,9,9,4,4,3,6,2,8,4,6,8,3,2,3,6,6,11,6,8,1,5,2,3,11,7,0,6,5,3,5,8,6,6,8,6,5,5,9,11,6,7,6,5,5,5,6,7,6,2,5,7,6,7,5,9,2,6,19,4,2,4,6,7,8,6,11,5,11,7,5,7,10,7,7,1,10,10,5,7,12,8,9,12,11,9,7,11,8,8,5,12,10,8,5,8,5,11,3,7,7,5,5,5,7,3,8,5,7,11,9,9,5,8,10,8,11,20,16,16,20,15,14,13,16,13,14,15,12,16,13,13,12,8,6,6,6,7,6,8,6,12,11,5,5,7,10,3,5,9,8,4,8,5,9,4,11,7,10,11,12,7,7,14,9,14,8,6,10,7,7,10,6,11,11,6,5,11,5,9,11,10,9,8,10,10,10,7,8,10,11,10,10,7,12,9,13,11,12,9,13,8,16,10,10,5,5,7,13,10,7,10,6,10,3,9,9,14,6,9,11,4,14,12,11,8,7,15,6,16,12,9,13,14,10,21,13,8,16,11,10,13,4,8,12,7,16,14,9,12,11,10,7,13,13,11,11,13,6,7,9,13,8,9,7,11,12,7,9,9,10,13,9,12,6,8,8,8,12,8,12,2,6,1,10,9,13,13,10,2,11,12,11,7,13,10,3,8,11,7,12,8,10,7,16,17,11,7,14,7,5,8,3,8,7,11,8,6,9,11,11,7,7,5,10,9,14,4,8,10,8,17,9,6,6,5,9,11,5,10,11,10,11,10,12,16,8,8,2,9,4,9,6,13,6,7,4,11,10,4,9,9,11,6,13,10,6,10,12,12,13,16,6,8,5,7,6,4,7,13,5,10,6,5,5,9,7,6,7,11,10,5,6,7,9,3,8,8,14,7,10,11,5,8,10,14,15,16,17,9,9,9,14,14,20,13,14,8,9,5,8,14,13,19,24,13,14,15,14,11,14,13,15,14,7,7,10,9,8,17,16,14,20,12,20,14,15,15,9,5,10,5,10,11,7,9,8,4,5,6,5,9,11,5,6,2,5,6,8,13,4,9,4,8,4,4,8,5,10,9,9,5,14,14,11,11,8,8,4,12,10,13,13,9,10,10,9,5,4,7]},"production-gce-c29":{"authFailuresPerMinute":[0,1,0,1,0,0,4,3,2,5,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,2,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,9,6,0,0,4,3,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,3,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,2,4,0,0,4,5,0,0,0,0,1,5,1,5,2,4,4,5,2,3,1,1,6,3,4,4,1,0,0,0,0,3,7,1,0,0,0,0,0,6,1,1,7,0,6,5,2,0,0,1,0,0,0,0,0,0,3,4,0,1,4,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,1,1,0,2,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,2,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,3,0,1,0,0,0,2,0,0,3,4,5,3,1,0,0,0,0,0,2,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,4,3,0,0,2,2,0,0,2,2,0,0,0,1,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,2,2,0,2,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,2,0,0,0,0,1,0,0,1,0,0,0,0,3,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,4,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,2,0,6,4,2,0,0,1,0,2,1,0,2,0,0,0,0,0,0,0,0,0,0,2,0,0,1,0,1,0,0,1,4,1,0,0,3,1,0,1,0,0,0,6,2,6,5,8,4,0,0,0,6,4,8,4,3,0,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,2,3,0,0,0,0,0,0,2,0,1,0,1,0,1,0,0,1,0,2,0,0,2,0,0,1,0,0,0,0,0,0,2,6,3,7,0,1,0,0,0,1,0,2,2,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,6,4,0,0,0,0,0,3,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,2,0,0,0,2,0,0,1,1,0,0,0,0,0,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,2,1,1,3,0,2,0,0,0,1,2,0,0,0,1,0,0,1,2,0,0,0,0,3,0,0,0,0,3,0,0,0,2,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,3,2,0,0,2,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,2,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,0,0,1,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,2,1,0,0,0,0,1,0,0,1,0,0,2,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,2,0,0,0,2,0,1,0,1,0,0,1,0,0,0,2,0,0,0],"completedAcceptsPerMinute":[3,7,7,8,8,7,4,8,12,4,9,8,10,8,6,5,11,8,10,12,6,7,11,7,12,7,6,11,10,7,9,8,3,10,10,9,6,4,11,9,9,10,16,4,11,11,7,7,9,8,7,11,10,9,5,6,6,7,12,12,7,6,12,6,11,18,3,9,7,8,8,4,10,5,13,10,8,12,8,13,14,3,7,5,7,8,8,4,4,8,11,3,9,9,8,3,7,4,3,7,5,5,9,8,6,7,6,3,9,9,6,10,5,7,15,5,9,11,10,14,2,7,8,10,12,7,19,8,9,9,5,7,5,5,8,11,9,10,8,5,4,11,7,8,13,10,15,12,5,11,15,9,12,8,9,3,11,9,11,9,7,15,11,8,5,8,6,8,8,1,7,5,10,9,7,7,7,9,11,7,13,10,7,9,9,4,6,7,9,7,7,5,10,13,8,6,4,11,5,9,9,7,2,9,1,8,5,8,10,12,10,12,4,8,4,8,7,9,7,3,4,4,3,9,3,8,12,5,6,12,15,8,16,14,9,9,12,5,15,7,8,6,7,11,9,12,3,6,6,7,6,7,3,2,7,10,9,16,2,14,21,11,15,12,10,6,13,1,12,7,15,5,7,11,15,6,2,10,9,10,15,13,14,12,17,12,11,13,8,4,14,10,7,3,4,6,3,5,9,8,10,10,3,7,8,6,8,6,8,12,7,4,9,7,9,5,8,6,10,5,3,6,6,5,3,8,12,5,8,7,12,4,14,13,12,9,14,16,13,14,11,7,8,4,3,8,9,3,5,5,12,7,8,6,11,10,12,13,7,14,17,16,13,15,17,9,14,7,6,9,9,9,11,8,8,7,8,2,12,7,10,6,7,5,5,7,11,7,11,7,3,5,4,9,8,7,9,5,7,8,7,9,5,9,16,15,23,20,14,11,10,5,4,13,14,13,10,16,15,20,16,21,16,17,20,15,16,21,19,13,12,9,11,10,7,12,12,4,9,7,4,14,9,14,15,16,14,9,7,3,6,8,9,6,12,7,12,5,12,10,13,6,10,8,10,6,5,4,8,6,13,6,10,9,6,8,7,7,12,6,5,6,10,12,15,14,9,12,13,17,13,4,3,4,6,8,6,8,9,8,13,3,2,3,6,6,10,6,3,10,4,8,13,5,2,4,2,12,5,6,5,9,8,7,5,7,4,4,5,9,12,14,15,9,8,2,4,6,6,2,6,4,6,1,6,5,4,3,3,6,7,1,3,2,6,1,5,9,5,3,1,4,4,5,6,11,5,4,4,4,4,4,6,4,3,6,5,6,4,4,2,7,6,2,7,4,7,6,4,3,3,4,8,5,0,3,5,4,1,5,1,2,2,5,1,8,3,2,6,1,2,3,5,3,2,4,6,5,3,2,3,2,1,3,2,5,4,3,1,3,2,5,4,2,5,1,2,3,3,2,0,3,0,1,3,5,3,1,4,1,6,3,4,2,2,4,2,2,4,2,3,2,5,4,3,2,2,1,6,10,9,10,7,4,1,2,6,8,18,14,15,16,15,19,15,17,13,18,14,8,3,1,1,0,5,1,3,1,0,0,1,0,3,2,0,1,1,6,6,3,3,3,1,2,2,2,4,2,0,1,4,2,0,1,1,0,1,1,2,1,4,1,4,1,6,3,1,1,2,3,1,0,1,2,0,0,1,0,4,2,1,2,1,1,1,1,3,2,0,1,4,0,0,5,0,1,1,2,1,1,3,2,2,3,3,3,2,0,3,2,1,3,0,2,4,3,3,5,5,3,1,0,2,1,2,0,0,1,4,2,0,1,2,2,2,1,1,1,1,2,2,4,2,3,1,5,1,2,3,3,1,0,1,1,1,0,0,2,2,0,1,3,3,4,2,0,0,2,1,2,2,0,1,1,0,0,0,0,1,1,1,2,0,2,0,2,0,1,0,1,3,1,2,1,0,0,2,1,0,1,1,2,0,1,1,1,0,0,0,1,2,1,2,2,0,0,1,1,1,4,0,1,0,1,5,2,0,2,0,1,1,2,3,1,1,1,2,2,2,3,2,0,2,1,1,0,0,2,4,2,3,0,0,0,0,2,8,2,1,1,1,1,1,1,2,0,2,0,1,1,0,0,0,1,2,1,3,2,3,1,1,3,4,5,0,3,1,2,0,3,2,4,1,3,3,3,2,4,1,5,5,6,1,3,1,2,2,0,4,13,5,8,3,2,2,1,4,2,3,2,4,0,4,5,2,7,5,0,6,5,6,4,3,8,7,4,2,5,3,12,11,12,7,5,11,17,15,12,11,7,6,4,3,8,3,3,4,2,2,2,5,8,4,7,5,4,1,5,3,2,6,2,5,7,16,3,5,4,11,6,12,7,8,6,8,3,7,2,3,6,8,2,5,11,6,6,7,7,4,7,7,11,4,5,3,7,7,0,3,8,5,9,11,4,7,4,15,5,4,2,6,5,7,4,5,10,3,5,10,6,10,6,8,5,8,6,10,9,12,5,7,12,6,8,5,7,9,8,6,6,15,12,17,13,8,14,5,5,9,7,4,8,9,7,10,6,8,7,10,9,10,6,8,10,4,3,9,4,11,4,5,6,9,9,10,8,6,8,5,8,4,4,5,11,3,2,5,3,4,3,5,7,4,8,1,5,6,7,6,14,11,9,11,7,6,8,12,4,10,6,9,13,12,11,2,7,11,7,10,7,8,9,8,9,8,12,18,12,9,14,13,14,16,12,13,8,8,14,14,3,12,5,15,9,12,9,6,8,15,11,9,5,8,11,12,8,8,6,4,10,6,7,17,12,8,9,11,11,8,10,11,6,13,9,11,15,11,12,6,8,9,11,14,7,10,7,10,6,7,13,5,10,14,16,15,20,9,7,7,10,9,13,10,6,10,10,15,16,18,16,12,6,7,10,9,6,7,15,10,6,13,14,25,10,10,8,9,10,8,7,12,13,6,5,6,8,7,12,8,12,11,14,6,10,10,13,11,14,14,9,16,14,10,5,6,8,6,12,8,8,7,8,9,7,7,14,7,7,8,8,4,7,5,6,8,9,24,10,12,14,10,12,9,6,10,14,6,13,7,16,8,9,13,10,15,11,14,14,11,10,16,9,12,9,9,11,6,10,13,4,13,11,10,6,9,11,10,6,7,8,9,7,7,6,7,7,8,10,12,6,12,10,10,6,7,9,11,5,3,6,6,6,10,9,8,9,7,7,14,5,7,7,3,4,11,8,5,10,3,6,12,4,6,4,9,7,7,4,13,6,7,10,7,9,5,8,2,9,8,9,7,6,6,10,10,13,15,8,7,7,11,4,6,9,4,17,12,17,15,12,13,5,13,12,6,18,10,12,8,8,3,6,12,7,4,9,6,8,5,5,18,9,13,15,8,6,7,6,6,8,9,9,11,6,7,3,8,5,4,2,2,9,7,4,6,8,6,12,3,7,11,7,4,10,11,8,4,9,12,15,10,8,8,5,6,7,5,3,7,6,3,3,10,8,3,7,12,9,6,4]}}} diff --git a/cloud/dev/scripts/relay-reserve-cell-refusal.mjs b/cloud/dev/scripts/relay-reserve-cell-refusal.mjs new file mode 100644 index 000000000000..b91dcb5203d0 --- /dev/null +++ b/cloud/dev/scripts/relay-reserve-cell-refusal.mjs @@ -0,0 +1,16 @@ +// Step 5: the director refuses per-cell admin reads for a reserve cell with exactly this body; +// any other 404 stays a failure. +export const RESERVE_CELL_REFUSAL_ERROR = 'cell_in_reserve_mode' + +export function isReserveCellRefusal(status, body) { + return status === 404 && body?.error === RESERVE_CELL_REFUSAL_ERROR +} + +// Step-5 design default 4: a reserve cell is flipped back to db before a roll. +export function reserveCellRollRefusal(cellId) { + return new Error( + `${cellId ?? 'a cell this step reads'} is in reserve mode ` + + `(${RESERVE_CELL_REFUSAL_ERROR}): flip it to admitMode=db with the cell-flags ` + + 'workflow and wait for it to record db before rolling it, then flip it back to reserve' + ) +} diff --git a/cloud/dev/scripts/relay-reserve-watch-rules.mjs b/cloud/dev/scripts/relay-reserve-watch-rules.mjs index 7fd9aae89f25..9da9d1ce40b5 100644 --- a/cloud/dev/scripts/relay-reserve-watch-rules.mjs +++ b/cloud/dev/scripts/relay-reserve-watch-rules.mjs @@ -6,9 +6,9 @@ import { backgroundOf, SHADOW_GATE_THRESHOLDS } from './relay-same-cap-shadow-ga const MINUTE_MS = 60_000 export const RESERVE_WATCH_RULES = { - // The roll ladder's warn line: max(1.5x, background + 20), so a zero background is not tripped - // by one 503. Three complete minutes in a row, not two. The read spans more minutes than that so - // a late publishing front still leaves three published minutes to judge. + // The roll ladder's warn line on non-drain 503s: max(1.5x, background + 20), so a zero background + // is not tripped by one 503. Three complete minutes in a row, not two. The read spans more minutes + // than that so a late publishing front still leaves three published minutes to judge. director503: { multiple: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMultiple, marginPerMinute: SHADOW_GATE_THRESHOLDS.nonDrain503Budget.warnMarginPerMinute, @@ -33,6 +33,80 @@ export const RESERVE_WATCH_RULES = { // sample line's `rules` keys follow this order too. export const RULE_NAMES = ['deadMan', 'overrun', 'director503', 'duplicateSeat', 'hostDrop', 'ledgerDrift'] +// Step 6 design §7: judged only while the cell's switch file says phonePath=memory, against the +// cell's own runtime metrics for the 10 minutes before phonePath went memory. A trip turns the +// phone path off (phonePath=db) and the watch goes on with the reserve rules. +export const PHONE_WATCH_RULES = { + // Each rule trips on three complete minutes in a row over its line. + consecutiveMinutes: 3, + readMinutes: 10, + // Judged as a share of the minute's phone attempts (the signal plus completed accepts; for + // confirm failures, the accepts), not a count: a count line false-tripped 33/121 5-hour watches + // on busy c28 (10-18/19), where bursts of refusals arrive with the evening rise in accepts. A + // minute with fewer attempts is not judged, so low-volume c7 (~2 accepts/min) is mostly unjudged. + minAttemptsPerMinute: 4, + // Over half of the minute's attempts, and 0.25 above the share before the flip, capped so two + // failures per accept always trips. Exactly half is not enough: c28 (04:08Z) and c7 (09:13Z) + // both show 3 minutes at 50% of host-and-phone auth failures. + shareFloor: 0.5, + shareOverBackground: 0.25, + shareCeiling: 0.6, + // Page only: confirms still work, only their audit rows or renewals are lost. + pageAfterMs: 5 * MINUTE_MS +} +export const PHONE_RULE_NAMES = [ + 'phoneCredentialRefusals', + 'phoneConfirmFailures', + 'phoneAttachDeadlines', + 'phoneAcceptDatabaseErrors' +] +const PHONE_RULE_SIGNALS = { + phoneCredentialRefusals: 'credentialRefusals', + phoneConfirmFailures: 'confirmFailures', + phoneAttachDeadlines: 'attachDeadlines', + phoneAcceptDatabaseErrors: 'acceptDatabaseErrors' +} + +const sum = (values) => values.reduce((total, value) => total + value, 0) + +// The share line for a signal whose share of attempts before the flip was `backgroundShare`. +export function phoneShareLine(backgroundShare) { + const { shareFloor, shareOverBackground, shareCeiling } = PHONE_WATCH_RULES + return Math.min(shareCeiling, Math.max(shareFloor, backgroundShare + shareOverBackground)) +} + +// A completed accept makes one confirm, so confirm failures are a share of accepts (capped at +// all of them); every other signal is an attempt that did not complete. +function attemptsAndShare(signal, count, completedAccepts) { + if (signal === 'confirmFailures') { + return { attempts: completedAccepts, share: completedAccepts === 0 ? 0 : Math.min(count, completedAccepts) / completedAccepts } + } + const attempts = count + completedAccepts + return { attempts, share: attempts === 0 ? 0 : count / attempts } +} + +// Over the line: enough attempts that minute, and strictly more than the line refused or failed. +export function phoneMinuteOver(signal, count, completedAccepts, line) { + const { attempts, share } = attemptsAndShare(signal, count, completedAccepts) + return attempts >= PHONE_WATCH_RULES.minAttemptsPerMinute && share > line +} + +// The 10 minutes before phonePath went memory, from `readCellPhoneMinutes`: each signal's share of +// attempts. Null until every minute is metered. +export function measuredPhoneBackground(read) { + if (!read.ok || read.unmetered.length > 0 || read.minutes.length === 0) return null + const accepted = sum(read.series.completedAccepts) + return { + share: Object.fromEntries( + Object.values(PHONE_RULE_SIGNALS).map((signal) => { + const count = sum(read.series[signal]) + return [signal, attemptsAndShare(signal, count, accepted).share] + }) + ), + minutes: read.minutes.length + } +} + export function initialWatchState({ cellId, startedAt, durationMinutes, background503PerMinute, flippedAt }) { return { v: 1, @@ -50,16 +124,34 @@ export function initialWatchState({ cellId, startedAt, durationMinutes, backgrou hosts: [], reserveSeenAt: null, ledgerAboveSince: null, - seatBehindRowFirstSeen: {} + seatBehindRowFirstSeen: {}, + // Step 6: what a trip writes back, and the phone rules' state. `phoneSince` is when the switch + // file went phonePath=memory (flippedAt if it already said so); `phoneOffAt` when the watch + // turned it off; `phoneTrip` a phone verdict the segment has yet to write. + flip: null, + lastSampleAt: null, + phoneSince: null, + phoneBackground: null, + phoneOffAt: null, + phoneTrip: null, + phoneOffFailures: 0, + phoneUnverifiableStreak: 0, + auditDropsLast: null, + auditDropsSince: null, + renewalFailuresSince: null, + paged: [] } } -// The background from the per-minute 503 counts before the flip. `publishedThrough` is the newest -// minute any response code has a point for: a 503 minute at or before it with no point had none, -// and the background waits (null) until it covers the last minute. -export function measuredBackground(perMinute, minutes, publishedThrough) { +// The background from the per-minute non-drain 503s before the flip, the live samples' measure. +// `publishedThrough` is the newest minute any response code has a point for: a 503 minute at or +// before it with no point had none, and the background waits (null) until it covers the last +// minute. Unmetered minutes are left out; under half the minutes metered is no background. +export function measuredBackground({ perMinute, minutes, publishedThrough, unmetered }) { if (minutes.length === 0 || publishedThrough === null || publishedThrough < minutes.at(-1)) return null - const background = backgroundOf({ minutes, series: minutes.map((minute) => perMinute[minute] ?? 0), unverified: false }) + const metered = minutes.filter((minute) => !unmetered.includes(minute)) + if (metered.length * 2 < minutes.length) return null + const background = backgroundOf({ minutes: metered, series: metered.map((minute) => perMinute[minute] ?? 0), unverified: false }) return { medianPerMinute: background.medianPerMinute, minutes: background.minutes, source: 'measured' } } @@ -68,6 +160,8 @@ const trip = (reason, detail = {}) => ({ status: 'trip', reason, ...detail }) const unverifiable = (reason) => ({ status: 'unverifiable', reason }) // Not readable yet by design, so it does not count toward the unverifiable streak. const pending = (reason) => ({ status: 'pending', reason }) +const failedPhoneRead = { ok: false, error: 'no cell phone metrics read' } +const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS function judgeDeadMan(cell) { if (!cell.ok) return unverifiable(cell.error) @@ -96,8 +190,8 @@ function judgeOverrun(cell) { return pass({ enforcedUnits: cell.enforcedUnits, hardCap: cell.hardCap }) } -// Fleet-wide director 503s: no existing signal attributes a 503 to the cell a host was bound for, -// so a problem local to one cell can stay under the fleet's line. This proxy can miss. +// Fleet-wide non-drain director 503s: no existing signal attributes a 503 to the cell a host was +// bound for, so a problem local to one cell can stay under the fleet's line. This proxy can miss. function judgeDirector503(read, state, at) { const { multiple, marginPerMinute, consecutiveMinutes, publishDelayMs } = RESERVE_WATCH_RULES.director503 const { background } = state @@ -116,6 +210,8 @@ function judgeDirector503(read, state, at) { if (recent.length < consecutiveMinutes) { return unverifiable(`director requests are published only through ${read.publishedThrough}`) } + const unmetered = recent.filter((minute) => read.unmetered.includes(minute)) + if (unmetered.length > 0) return unverifiable(`no director runtime metrics for ${unmetered.join(', ')}`) const counts = recent.map((minute) => read.perMinute[minute] ?? 0) const detail = { backgroundPerMinute: perMinute, above, recent: Object.fromEntries(recent.map((m, i) => [m, counts[i]])) } if (counts.every((count) => count > above)) { @@ -181,9 +277,78 @@ function judgeLedgerDrift(cell, ledger, aboveSince, at) { return { verdict: pass({ ...detail, aboveForMs: at - since }), aboveSince: since } } +// A counter still rising (or failing) every sample for 5 minutes pages once; a single drop does not. +function judgeSustained(rising, since, at) { + const next = rising ? (since ?? at) : null + return { since: next, sustained: next !== null && at - next >= PHONE_WATCH_RULES.pageAfterMs } +} + +export function phoneSinceOf(cell, state) { + if (state.phoneSince !== null || !cell.ok || cell.phonePathConfigured !== 'memory') return state.phoneSince + // Memory from the first sample: the operator's flipped-at. Later: the sample before this one. + return state.lastSampleAt === null ? state.flippedAt : state.lastSampleAt +} + +// The phone rules for one sample, or none while the switch file does not say phonePath=memory. +function judgePhoneRules(cell, phone, state, at) { + if (!cell.ok || cell.phonePathConfigured !== 'memory' || state.phoneOffAt !== null) return {} + const background = state.phoneBackground + if (!background) { + const verdict = + state.phoneSince === null + ? unverifiable('the phone rules need --flipped-at: their background is the 10 minutes before it') + : at < minuteFloor(state.phoneSince) + RESERVE_WATCH_RULES.director503.publishDelayMs + ? pending('the phone background before phonePath went memory is not published yet') + : unverifiable('no phone background yet') + return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, verdict])) + } + if (!phone.ok) return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, unverifiable(phone.error)])) + const { consecutiveMinutes } = PHONE_WATCH_RULES + // Only the judged minutes need metrics; a gap earlier in the read changes nothing. + const judged = phone.minutes.slice(-consecutiveMinutes) + const gaps = phone.unmetered.filter((minute) => judged.includes(minute)) + if (gaps.length > 0) { + const gap = unverifiable(`no cell runtime metrics for ${gaps.join(', ')}`) + return Object.fromEntries(PHONE_RULE_NAMES.map((name) => [name, gap])) + } + const accepts = phone.series.completedAccepts.slice(-consecutiveMinutes) + return Object.fromEntries( + PHONE_RULE_NAMES.map((name) => { + const signal = PHONE_RULE_SIGNALS[name] + const line = phoneShareLine(background.share[signal]) + const recent = phone.series[signal].slice(-consecutiveMinutes) + const detail = { backgroundShare: background.share[signal], line, recent, accepts } + const over = + recent.length === consecutiveMinutes && recent.every((count, index) => phoneMinuteOver(signal, count, accepts[index], line)) + return [ + name, + over ? trip(`${signal} over ${Math.round(line * 100)}% of attempts for ${consecutiveMinutes} min (${recent.join(', ')} against ${accepts.join(', ')} accepts)`, detail) : pass(detail) + ] + }) + ) +} + +// Page-only signals: audit rows dropped since the previous sample, and renewal writes failing in +// every one of the last 5 published minutes. +function judgePhonePages(cell, phone, state, at) { + const dropped = cell.ok && Number.isInteger(cell.resumeAuditDropped) ? cell.resumeAuditDropped : null + const audit = judgeSustained( + dropped !== null && state.auditDropsLast !== null && dropped > state.auditDropsLast, + state.auditDropsSince, + at + ) + const renewals = phone?.ok ? (phone.series.renewalFailures ?? []).slice(-PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS) : [] + const renewal = judgeSustained(renewals.length > 0 && renewals.every((count) => count > 0), state.renewalFailuresSince, at) + const pages = [] + if (audit.sustained && !state.paged.includes('audit-drops')) pages.push('audit-drops') + if (renewal.sustained && !state.paged.includes('renewal-failures')) pages.push('renewal-failures') + return { auditDropsLast: dropped ?? state.auditDropsLast, auditDropsSince: audit.since, renewalFailuresSince: renewal.since, pages } +} + /** * One sample. `sample.cell` is the cell's runtime-status, `director503` the complete minutes of - * director 503s with the request series' publishing front (`publishedThrough`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this + * non-drain director 503s with the request series' publishing front (`publishedThrough`) and the + * minutes no runtime metrics cover (`unmetered`), `ledger` the placing directors' newest ledger summaries, `seatBehindRow` this * cell's reconcile reports in the lookback. Each is `{ ok: false, error }` when it could not be read. */ export function evaluateWatchSample(state, sample, { unverifiableLimit }) { @@ -195,14 +360,14 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) { if (!seen && at < state.startedAt + RESERVE_WATCH_RULES.reserveApplyWaitMs) { // Not judged, and the streak holds: the reserve write may not have applied yet. const rules = { reserve: pending(`switch file says ${cell.admitModeConfigured}; waiting for reserve to apply`) } - return { state: next, rules, line: sampleLine(next, sample, rules) } + return { state: next, rules, line: sampleLine(next, sample, rules), pages: [] } } // Someone else flipped it back (or it never became reserve): nothing left to watch. next.state = seen ? 'stopped' : 'refused' next.verdict = seen ? `${state.cellId} is not in reserve mode (switch file says ${cell.admitModeConfigured})` : `${state.cellId} did not report reserve mode within ${RESERVE_WATCH_RULES.reserveApplyWaitMs / MINUTE_MS} min (switch file says ${cell.admitModeConfigured})` - return { state: next, rules: {}, line: sampleLine(next, sample, {}) } + return { state: next, rules: {}, line: sampleLine(next, sample, {}), pages: [] } } if (cell.ok) { next.reserveSeenAt = state.reserveSeenAt ?? at @@ -214,34 +379,63 @@ export function evaluateWatchSample(state, sample, { unverifiableLimit }) { next.seatBehindRowFirstSeen = duplicate.firstSeen const ledger = judgeLedgerDrift(cell, sample.ledger, state.ledgerAboveSince, at) next.ledgerAboveSince = ledger.aboveSince + next.lastSampleAt = at + next.phoneSince = phoneSinceOf(cell, state) + const phoneState = { ...state, phoneSince: next.phoneSince } + const pageCheck = judgePhonePages(cell, sample.phone, state, at) + next.auditDropsLast = pageCheck.auditDropsLast + next.auditDropsSince = pageCheck.auditDropsSince + next.renewalFailuresSince = pageCheck.renewalFailuresSince + next.paged = [...state.paged, ...pageCheck.pages] + const pages = pageCheck.pages const rules = { deadMan: judgeDeadMan(cell), overrun: judgeOverrun(cell), director503: judgeDirector503(sample.director503, state, at), duplicateSeat: duplicate.verdict, hostDrop: cell.ok ? judgeHostDrop(next.hosts) : unverifiable(cell.error), - ledgerDrift: ledger.verdict + ledgerDrift: ledger.verdict, + ...judgePhoneRules(cell, sample.phone ?? failedPhoneRead, phoneState, at) } const tripped = RULE_NAMES.find((name) => rules[name].status === 'trip') if (tripped) { next.state = 'tripped' + next.flip = 'admitMode' next.verdict = `flip back ${state.cellId}: ${tripped}: ${rules[tripped].reason}` - return { state: next, rules, line: sampleLine(next, sample, rules) } + return { state: next, rules, line: sampleLine(next, sample, rules), pages } + } + // A phone trip turns off only the phone path; the segment writes it and the watch goes on. + const phoneNames = PHONE_RULE_NAMES.filter((name) => name in rules) + const phoneTripped = phoneNames.find((name) => rules[name].status === 'trip') + const phoneMissing = phoneNames.filter((name) => rules[name].status === 'unverifiable') + next.phoneUnverifiableStreak = phoneMissing.length > 0 ? state.phoneUnverifiableStreak + 1 : 0 + if (phoneTripped) { + next.phoneTrip = `turn off the phone path on ${state.cellId}: ${phoneTripped}: ${rules[phoneTripped].reason}` + } else if (next.phoneUnverifiableStreak >= unverifiableLimit) { + next.phoneTrip = + `turn off the phone path on ${state.cellId}: unverifiable: ${next.phoneUnverifiableStreak} samples ` + + `in a row could not read ${phoneMissing.join(', ')}` } const missing = RULE_NAMES.filter((name) => rules[name].status === 'unverifiable') - const waiting = RULE_NAMES.some((name) => rules[name].status === 'pending') + const waiting = [...RULE_NAMES, ...phoneNames].some((name) => rules[name].status === 'pending') // A pending rule holds the streak where it was: neither evidence of health nor a gap. next.unverifiableStreak = missing.length > 0 ? state.unverifiableStreak + 1 : waiting ? state.unverifiableStreak : 0 if (next.unverifiableStreak >= unverifiableLimit) { next.state = 'tripped' + next.flip = 'admitMode' next.verdict = `flip back ${state.cellId}: unverifiable: ${next.unverifiableStreak} samples in a row could not ` + `read ${missing.join(', ')}` + } else if (at >= state.deadlineAt && next.phoneTrip) { + // Still unwritten at the end: the workflow's flip back writes it, and pages if that fails too. + next.state = 'tripped' + next.flip = 'phonePath' + next.verdict = `${next.phoneTrip}; phonePath=db did not land before the watch ended` } else if (at >= state.deadlineAt) { next.state = 'completed' next.verdict = `${state.cellId} stayed healthy in reserve mode for the whole watch` } - return { state: next, rules, line: sampleLine(next, sample, rules) } + return { state: next, rules, line: sampleLine(next, sample, rules), pages } } function sampleLine(state, sample, rules) { @@ -253,6 +447,7 @@ function sampleLine(state, sample, rules) { state: state.state, unverifiableStreak: state.unverifiableStreak, rules, + ...(state.phoneTrip ? { phoneTrip: state.phoneTrip } : {}), ...(state.verdict ? { verdict: state.verdict } : {}) }) } diff --git a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs index 2914b2820953..3815a6b965ff 100644 --- a/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs +++ b/cloud/dev/scripts/relay-same-cap-shadow-gate.mjs @@ -232,9 +232,9 @@ const DIRECTOR_DRAIN_FIELDS = [ ] // Five instances at one sample per 30 s is ~100 per 10-min sub-window; this many is truncation. -const DIRECTOR_METRICS_LIMIT = 1000 +export const DIRECTOR_METRICS_LIMIT = 1000 -async function readDirectorDrainReturn(reader, { config, window }) { +export async function readDirectorDrainReturn(reader, { config, window }) { const projection = `json(timestamp,${DIRECTOR_DRAIN_FIELDS .map((field) => `jsonPayload.${field}`) .join(',')})` diff --git a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs index f5e183b47486..f1313bcafecb 100644 --- a/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs +++ b/cloud/dev/scripts/relay-staging-c4-refresh-workflow.test.mjs @@ -42,9 +42,9 @@ const asiaCells = ['production-gce-c27', 'production-gce-c28', 'production-gce-c const c30Digest = '4158d8a2e18e9caec439d257f0c1e45d92ffea8c0262f057b2f08c76a134bcf0' // C31 launches on the digest the director served when it was declared. const c31Digest = 'f30b5cb1ec52b6b6145efecfa1b8be9e3d309403beffd8abcc64197a2087e269' -// Staging C4 is declared on image G (main 2de73812, E-pre with the t_preauth metric); the staging +// Staging C4 is declared on image J (main 30f5bef9, step-6 phonePath staging test); the staging // director deploy binds its digest to this line. -const stagingC4Digest = '07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d' +const stagingC4Digest = '21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4' function cellBlock(tfvars, cellId) { const start = tfvars.indexOf(`"${cellId}"`) diff --git a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs index 1f4d946557c0..66168db1309a 100644 --- a/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs +++ b/cloud/dev/scripts/relay-staging-deploy-identity.test.mjs @@ -150,6 +150,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () => 'google_cloud_run_v2_service_iam_member.github_staging_relay_deploy_director_developer', 'google_iam_workload_identity_pool_provider.github_staging_relay_deploy', 'google_project_iam_member.github_staging_relay_deploy_compute_viewer', + 'google_project_iam_member.github_staging_relay_deploy_logging_viewer', 'google_service_account.github_staging_relay_deploy', 'google_service_account_iam_member.github_staging_relay_deploy_auth_runtime_user', 'google_service_account_iam_member.github_staging_relay_deploy_workload_identity_user', @@ -189,7 +190,7 @@ test('the staging deploy identity declares exactly its enumerated grants', () => const projectRoles = declaredFamilies() .filter((family) => family.startsWith('google_project_iam_member.')) .map((family) => block('google_project_iam_member', family.split('.')[1]).match(/role\s*=\s*"([^"]+)"/)[1]) - assert.deepEqual(projectRoles, ['roles/compute.viewer']) + assert.deepEqual(projectRoles, ['roles/compute.viewer', 'roles/logging.viewer']) assert.doesNotMatch(identity, /roles\/artifactregistry\.writer/) assert.doesNotMatch(identity, /"roles\/(?:owner|editor|viewer)"/) }) diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.mjs index 6f5cc38e279f..4e685ae38a26 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.mjs @@ -21,14 +21,28 @@ import { readRuntime, WRITE_CONFIRMATION } from './operate-relay-cell-flags.mjs' -import { readDirectorRequestsByCodePerMinute, readLogEntries } from './relay-same-cap-shadow-gate.mjs' -import { minutesOf } from './relay-same-cap-shadow-gate-verdict.mjs' +import { + DIRECTOR_METRICS_LIMIT, + readDirectorDrainReturn, + readDirectorRequestsByCodePerMinute, + readLogEntries +} from './relay-same-cap-shadow-gate.mjs' +import { + DIRECTOR_METRICS_INTERVAL_MS, + drainReturnByMinute, + minuteKey, + minutesOf, + withoutDrainDeferrals +} from './relay-same-cap-shadow-gate-verdict.mjs' import { PRODUCTION_LEASE } from './cloud-sql-rollout-lock-census.mjs' import { RELAY_REPOSITORY_ROOT } from './relay-repository.mjs' import { evaluateWatchSample, initialWatchState, measuredBackground, + measuredPhoneBackground, + phoneSinceOf, + PHONE_WATCH_RULES, RESERVE_WATCH_RULES } from './relay-reserve-watch-rules.mjs' @@ -58,8 +72,11 @@ function integerIn(name, raw, min, max) { } // The flip back's request. Its generation is a placeholder: the flip back re-reads the current one. +// A phone-rule trip turns off only the phone path; anything else flips the whole cell back. Both +// remove the phonePath key, so a cell rolled back to an image without the switch takes the write. export function dbFlipRequest(values) { - return parseCellFlagsRequest({ ...values, set: 'admitMode=db', 'expected-generation': '0' }) + const set = values.flip === 'phonePath' ? 'phonePath=default' : 'admitMode=db,phonePath=default' + return parseCellFlagsRequest({ ...values, set, 'expected-generation': '0' }) } // `now` is the watch's start: a later segment of a long watch must not age flipped-at out. @@ -93,6 +110,8 @@ export function parseWatchArguments(values, now = Date.now()) { if (!values['state-file']) throw new Error('--state-file is required') return { request, + // The flag tool's own inputs, for turning the phone path off mid-watch. + flipValues: values, directorService: values['director-service'], durationMinutes: integerIn('duration-minutes', values['duration-minutes'], 5, MAX_WATCH_MINUTES), segmentMinutes: integerIn('segment-minutes', values['segment-minutes'], 1, MAX_SEGMENT_MINUTES), @@ -119,6 +138,8 @@ export async function readCellSample(fetchImpl, request, idToken, wait) { ok: true, admitModeConfigured: runtime.flagsApplied.flags?.admitMode ?? 'db', admitModeEffective: runtime.admitModeEffective, + phonePathConfigured: runtime.flagsApplied.flags?.phonePath ?? 'db', + resumeAuditDropped: runtime.runtime?.resumeAuditDropped, draining: runtime.draining === true, controls, enforcedUnits: runtime.runtime?.enforcedConnectionUnits, @@ -130,31 +151,97 @@ export async function readCellSample(fetchImpl, request, idToken, wait) { } const minuteFloor = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS -const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null }) +const failed503Read = (error) => ({ ok: false, error, perMinute: {}, minutes: [], publishedThrough: null, unmetered: [] }) -// The `minutes` whole minutes that end at the minute boundary `endedAt`: the 503 counts, and the -// newest minute any response code has a point for. The director always serves something, so that -// front is how far the 503 series is published; no front at all is no evidence. A code publishing -// late only delays a trip: a later read sees its points. +// The `minutes` whole minutes that end at the minute boundary `endedAt`: the director's non-drain +// 503s, the roll ladder's measure its line comes from. Cloud Run's 503 count less what the +// director's runtime metrics attribute to drain-return deferrals and to a host re-dialling inside +// its own retry interval or while its last dial is in flight (10-18: ~98% of 503s), since those +// grow with traffic, not trouble. `publishedThrough` is the newest minute any response code has a +// point for (the director always serves something, so no front at all is no evidence); a minute +// with no metrics line from any instance is `unmetered`, since subtracting nothing there is a guess. +// Coverage is any instance, not each: one instance's missing lines under-subtract, biased to a trip. export async function readDirector503Minutes(reader, { projectId, directorService, endedAt, minutes }) { const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) } + const config = { projectId, directorService } let read try { - read = await readDirectorRequestsByCodePerMinute(reader, { config: { projectId, directorService }, span }) + read = await readDirectorRequestsByCodePerMinute(reader, { config, span }) } catch (error) { // A point with an unparseable end time has no minute. return failed503Read(`director 503 metric unreadable: ${error instanceof Error ? error.message : error}`) } if (read.failed) return failed503Read('director 503 metric read failed') + // A line covers the 30 s before it, so the last minute's tail is in the next minute's lines. + const metrics = await readDirectorDrainReturn(reader, { + config, + window: { startedAt: span.startedAt, endedAt: new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS) } + }) + const lineTimes = metrics.flatMap((part) => part.samples.map((sample) => Date.parse(sample.timestamp))) + if (lineTimes.some((at) => !Number.isFinite(at))) return failed503Read('director runtime metrics line with an unparseable timestamp') + const scheduled = drainReturnByMinute(metrics, DIRECTOR_METRICS_LIMIT) + if (scheduled.truncated) return failed503Read('director runtime metrics read failed, truncated, or short') + const keys = minutesOf(span) + const metered = new Set(lineTimes.flatMap((at) => [minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS), minuteKey(at - 1)])) + const nonDrain = withoutDrainDeferrals({ perMinute: read.byCode['503'] ?? {} }, scheduled, keys) const published = Object.values(read.byCode).flatMap((perMinute) => Object.keys(perMinute)).sort() return { ok: true, - perMinute: read.byCode['503'] ?? {}, - minutes: minutesOf(span), - publishedThrough: published.at(-1) ?? null + perMinute: Object.fromEntries(keys.map((minute, index) => [minute, nonDrain.series[index]])), + minutes: keys, + publishedThrough: published.at(-1) ?? null, + unmetered: keys.filter((minute) => !metered.has(minute)) } } +const CELL_METRICS_EVENT = 'orca_relay_runtime_metrics' +const PHONE_SIGNALS = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors', 'renewalFailures', 'completedAccepts'] +const sumOf = (counts) => Object.values(counts ?? {}).reduce((sum, count) => sum + Number(count), 0) + +// The cell's own runtime metrics lines (one per 30 s) for the `minutes` whole minutes ending at +// `endedAt`, per minute: what the phone rules judge. A line counts toward the minute it ends in; +// a minute no line covers is `unmetered`, never zero. +export async function readCellPhoneMinutes(reader, { cellId, endedAt, minutes }) { + const span = { startedAt: new Date(endedAt - minutes * MINUTE_MS), endedAt: new Date(endedAt) } + const read = await readLogEntries(reader, { + filter: [ + 'resource.type="gce_instance"', + `jsonPayload.event="${CELL_METRICS_EVENT}"`, + `jsonPayload.cellId="${cellId}"`, + `timestamp>="${span.startedAt.toISOString()}"`, + `timestamp<"${new Date(endedAt + DIRECTOR_METRICS_INTERVAL_MS).toISOString()}"` + ].join(' AND '), + projection: + 'json(timestamp,jsonPayload.phoneRejectionsByCodeDelta,jsonPayload.resumeConfirmFailuresByCodeDelta,' + + 'jsonPayload.phoneAttachDeadlinesDelta,jsonPayload.phoneAcceptDatabaseErrorsDelta,jsonPayload.resumeRenewalFailuresDelta,' + + 'jsonPayload.clientAcceptCompletedDelta)', + limit: LOG_ENTRY_LIMIT + }) + if (read.failed) return failedRead(read.error) + if (read.entries.length >= LOG_ENTRY_LIMIT) return failedRead('cell runtime metrics read truncated') + const keys = minutesOf(span) + const series = Object.fromEntries(PHONE_SIGNALS.map((signal) => [signal, keys.map(() => 0)])) + const metered = new Set() + for (const entry of read.entries) { + const at = Date.parse(entry.timestamp) + if (!Number.isFinite(at)) return failedRead('cell runtime metrics line with an unparseable timestamp') + const payload = entry.jsonPayload ?? {} + if (payload.phoneAttachDeadlinesDelta === undefined) { + return { ...failedRead('cell runtime metrics carry no phone counters (an image before step 6)'), permanent: true } + } + metered.add(minuteKey(at - DIRECTOR_METRICS_INTERVAL_MS)).add(minuteKey(at - 1)) + const index = keys.indexOf(minuteKey(at - 1)) + if (index < 0) continue + series.credentialRefusals[index] += Number(payload.phoneRejectionsByCodeDelta?.['4401'] ?? 0) + series.confirmFailures[index] += sumOf(payload.resumeConfirmFailuresByCodeDelta) + series.attachDeadlines[index] += Number(payload.phoneAttachDeadlinesDelta) + series.acceptDatabaseErrors[index] += Number(payload.phoneAcceptDatabaseErrorsDelta ?? 0) + series.renewalFailures[index] += Number(payload.resumeRenewalFailuresDelta ?? 0) + series.completedAccepts[index] += Number(payload.clientAcceptCompletedDelta ?? 0) + } + return { ok: true, minutes: keys, series, unmetered: keys.filter((minute) => !metered.has(minute)) } +} + const LEDGER_SUMMARY_EVENT = 'orca_relay_reserve_ledger_summary' const SEAT_BEHIND_ROW_EVENT = 'orca_relay_reserve_ledger_seat_behind_row' @@ -342,13 +429,48 @@ export async function watchSegment(config, dependencies) { endedAt: baselineEndsAt, minutes: BACKGROUND_MINUTES }) - state = { ...state, background: read.ok ? measuredBackground(read.perMinute, read.minutes, read.publishedThrough) : null } + state = { ...state, background: read.ok ? measuredBackground(read) : null } } const reader = createReader(deps, request.projectId) // Serialised: concurrent Logging reads are what earn a 429. + const cell = await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep) + const phoneWatched = cell.ok && cell.phonePathConfigured === 'memory' && state.phoneOffAt === null + const phoneSince = phoneSinceOf(cell, state) + const phoneBaselineEndsAt = phoneSince === null ? null : minuteFloor(phoneSince) + if (phoneWatched && !state.phoneBackground && phoneBaselineEndsAt !== null && at >= phoneBaselineEndsAt + METRIC_DELAY_MS) { + const read = await readCellPhoneMinutes(reader, { + cellId: request.cellId, + endedAt: phoneBaselineEndsAt, + minutes: BACKGROUND_MINUTES + }) + // A restart or roll in those minutes leaves no background to judge against: no phone + // rule could ever pass, so refuse rather than turn the phone path off for certain. + const incomplete = read.ok ? read.unmetered.length > 0 && `no metrics for ${read.unmetered.join(', ')}` : read.permanent && read.error + if (incomplete) { + state = { + ...state, + state: 'refused', + verdict: + `${request.cellId}: the phone background, the 10 minutes before ${new Date(phoneBaselineEndsAt).toISOString()}, ` + + `is incomplete (${incomplete}), as after a cell restart or roll; start a new watch with a later --flipped-at` + } + await writeFile(config.stateFile, JSON.stringify(state)) + break + } + state = { ...state, phoneBackground: measuredPhoneBackground(read) } + } const sample = { at, - cell: await readCellSample(deps.fetchImpl, request, dependencies.idToken, sleep), + cell, + ...(phoneWatched + ? { + phone: await readCellPhoneMinutes(reader, { + cellId: request.cellId, + endedAt: minuteFloor(at - METRIC_DELAY_MS), + minutes: PHONE_WATCH_RULES.readMinutes + }) + } + : {}), director503: await readDirector503Minutes(reader, { projectId: request.projectId, directorService, @@ -362,6 +484,13 @@ export async function watchSegment(config, dependencies) { const result = evaluateWatchSample(state, sample, { unverifiableLimit: config.unverifiableLimit }) state = result.state log(result.line) + for (const kind of result.pages) { + await raiseAlarm( + { mode: request.mode, cellId: request.cellId, projectId: request.projectId, kind, reason: `sustained for ${PHONE_WATCH_RULES.pageAfterMs / MINUTE_MS} min` }, + { runGcloud: dependencies.runGcloud, log } + ) + } + if (state.state === 'watching' && state.phoneTrip) state = await turnPhonePathOff(config, state, at, { ...deps, log }) await writeFile(config.stateFile, JSON.stringify(state)) if (state.state !== 'watching') break await sleep(Math.max(0, at + SAMPLE_INTERVAL_MS - now())) @@ -370,8 +499,41 @@ export async function watchSegment(config, dependencies) { return state } -function writeOutcome(githubOutput, { state, verdict }) { - return appendFile(githubOutput, `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\n`) +// Writes phonePath=db (the flag tool's CAS write, a plan in dry-run) and keeps watching the +// reserve rules. A write that fails keeps the trip, so the next sample retries it, and pages once +// after PHONE_OFF_PAGE_AFTER_FAILURES in a row: the cell stays watched either way. +export const PHONE_OFF_PAGE_AFTER_FAILURES = 3 +async function turnPhonePathOff(config, state, at, dependencies) { + try { + const result = await flipBack({ ...config.flipValues, flip: 'phonePath' }, dependencies) + dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off', cellId: state.cellId, verdict: state.phoneTrip, written: result.written })) + return { ...state, phoneOffAt: at, phoneTrip: null, phoneOffFailures: 0 } + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + const failures = (state.phoneOffFailures ?? 0) + 1 + dependencies.log(JSON.stringify({ event: 'orca_relay_reserve_watch_phone_path_off_failed', cellId: state.cellId, failures, reason })) + if (failures === PHONE_OFF_PAGE_AFTER_FAILURES) { + await raiseAlarm( + { + mode: config.request.mode, + cellId: state.cellId, + projectId: config.request.projectId, + kind: 'flip-back-failed', + flip: 'phonePath', + reason: `${state.phoneTrip}; phonePath=db failed ${failures} times: ${reason}` + }, + { runGcloud: dependencies.runGcloud, log: dependencies.log } + ) + } + return { ...state, phoneOffFailures: failures } + } +} + +function writeOutcome(githubOutput, { state, verdict, flip }) { + return appendFile( + githubOutput, + `state=${state}\nverdict=${(verdict ?? '').replace(/[\r\n]+/g, ' ')}\nflip=${flip ?? 'admitMode'}\n` + ) } // The job's outcome. A watch that ended mid-segment (a crash, a timeout) records failed, which the @@ -431,7 +593,11 @@ export async function flipBack(values, dependencies) { ) for (let attempt = 1; ; attempt += 1) { const current = await readCurrentObject(fetchImpl, base, dependencies.accessToken) - if ((current.object?.flags?.admitMode ?? 'db') !== 'reserve') { + const stillOn = + values.flip === 'phonePath' + ? (current.object?.flags?.phonePath ?? 'db') === 'memory' + : (current.object?.flags?.admitMode ?? 'db') === 'reserve' + if (!stillOn) { const result = { written: false, alreadyDb: true, generation: current.generation } record(result) return result @@ -464,7 +630,7 @@ export async function flipBack(values, dependencies) { export const ALARM_EVENT = 'orca_relay_reserve_watch_alarm' export const ALARM_LOG_NAME = 'orca-relay-reserve-watch' -export const ALARM_KINDS = ['refused', 'flip-back-failed'] +export const ALARM_KINDS = ['refused', 'flip-back-failed', 'audit-drops', 'renewal-failures'] // GitHub's workflow-command escaping, so a verdict cannot end the annotation or start another. export const escapeWorkflowCommand = (text) => text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A') @@ -474,12 +640,18 @@ const escapeWorkflowProperty = (text) => escapeWorkflowCommand(text).replace(/:/ * Both kinds can leave the cell in reserve with nothing watching it. Only write mode pages: a * dry-run watch never flips back, so its refusal or failure changes nothing for the cell. */ -export async function raiseAlarm({ mode, cellId, projectId, kind, reason }, { runGcloud, log = console.log }) { +export async function raiseAlarm({ mode, cellId, projectId, kind, reason, flip }, { runGcloud, log = console.log }) { if (!ALARM_KINDS.includes(kind)) throw new Error(`--kind must be one of ${ALARM_KINDS.join(', ')}`) const message = kind === 'refused' ? `${cellId} may be in reserve mode and unwatched: the watch was refused (${reason}). Rerun it, or flip the cell to db with Operate Relay Production Cell Flags.` - : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.` + : kind === 'audit-drops' + ? `${cellId} is losing resume audit rows (${reason}); phones still connect. Check its database writes, or set phonePath=db.` + : kind === 'renewal-failures' + ? `${cellId} cannot renew phone credentials (${reason}); phones still connect but age toward re-pairing. Check its database writes, or set phonePath=db.` + : flip === 'phonePath' + ? `${cellId} may still have phonePath=memory with no watch: turning it off did not land (${reason}). Set phonePath=default by hand with Operate Relay Production Cell Flags.` + : `${cellId} may still be in reserve mode with no watch: the flip back did not land (${reason}). Flip it to db by hand with Operate Relay Production Cell Flags (set=admitMode=db), or with the break-glass inputs logged above if the cell does not answer.` log(`::error title=${escapeWorkflowProperty(`Reserve watch ${kind} for ${cellId}`)}::${escapeWorkflowCommand(message)}`) if (mode !== 'write') return { paged: false } const entry = JSON.stringify({ event: ALARM_EVENT, cellId, kind, reason }) @@ -518,7 +690,8 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { 'state-file', 'github-output', 'verdict', - 'kind' + 'kind', + 'flip' ].map((name) => [name, { type: 'string' }]) ) }) @@ -528,7 +701,7 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { try { if (command === 'alarm') { await raiseAlarm( - { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '' }, + { mode: values.mode, cellId: values['cell-id'] ?? '', projectId: values['project-id'], kind: values.kind, reason: values.verdict ?? '', flip: values.flip }, { runGcloud } ) // The alarm step is red, whatever the page did. @@ -542,6 +715,13 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { const config = await parseWatchSegment(values) const state = await watchSegment(config, { idToken, + // For turning the phone path off mid-watch; without it that write fails and the workflow's flip back retries. + accessToken: process.env.ORCA_RELAY_CONTROL_ACCESS_TOKEN, + audit: { + writtenBy: `reserve-watch/${process.env.GITHUB_TRIGGERING_ACTOR ?? process.env.GITHUB_ACTOR ?? 'unknown'}`, + runId: process.env.GITHUB_RUN_ID ?? 'local', + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? '0' + }, readRolloutLease: readProductionRolloutLease, runGcloud }) diff --git a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs index 28d34ab47235..0f5b1467c0cd 100644 --- a/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs +++ b/cloud/dev/scripts/watch-relay-reserve-cell.test.mjs @@ -7,7 +7,12 @@ import { test } from 'node:test' import { evaluateWatchSample, initialWatchState, - measuredBackground + measuredBackground, + measuredPhoneBackground, + phoneMinuteOver, + phoneShareLine, + PHONE_RULE_NAMES, + PHONE_WATCH_RULES } from './relay-reserve-watch-rules.mjs' import { ALARM_EVENT, @@ -18,6 +23,7 @@ import { parseWatchArguments, parseWatchSegment, raiseAlarm, + readCellPhoneMinutes, readDirector503Minutes, readSeatBehindRowSample, recordOutcome, @@ -47,7 +53,7 @@ const minutesBefore = (at, count) => ) const quiet503 = (at, perMinute = 10) => { const minutes = minutesBefore(at, 3) - return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1) } + return { ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute) => [minute, perMinute])), publishedThrough: minutes.at(-1), unmetered: [] } } const sampleAt = (at, overrides = {}) => ({ at, @@ -93,7 +99,8 @@ test('503s trip only after three complete minutes over max(1.5x background, back ok: true, minutes, perMinute: Object.fromEntries(minutes.map((minute, index) => [minute, counts[index]])), - publishedThrough: minutes.at(-1) + publishedThrough: minutes.at(-1), + unmetered: [] } } const twoOver = evaluateWatchSample(state, sampleAt(START, { director503: series([151, 140, 200]) }), LIMIT) @@ -115,12 +122,12 @@ test('503s trip only after three complete minutes over max(1.5x background, back test('a quiet fleet with no 503 series at all reads zero, not unverifiable', () => { const minutes = minutesBefore(START, 10) - const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1) } + const quiet = { ok: true, minutes, perMinute: {}, publishedThrough: minutes.at(-1), unmetered: [] } const [result] = run([sampleAt(START, { director503: quiet })], freshState({ background: { medianPerMinute: 0, source: 'measured' } })) assert.equal(result.rules.director503.status, 'pass') assert.deepEqual(Object.values(result.rules.director503.recent), [0, 0, 0]) assert.equal(result.state.unverifiableStreak, 0) - assert.deepEqual(measuredBackground({}, minutes, minutes.at(-1)), { medianPerMinute: 0, minutes: 10, source: 'measured' }) + assert.deepEqual(measuredBackground({ perMinute: {}, minutes, publishedThrough: minutes.at(-1), unmetered: [] }), { medianPerMinute: 0, minutes: 10, source: 'measured' }) }) test('503 minutes past the publishing front are left out, and only no front at all is unverifiable', () => { @@ -129,20 +136,20 @@ test('503 minutes past the publishing front are left out, and only no front at a const judge = (director503) => evaluateWatchSample(state, sampleAt(START, { director503 }), LIMIT) // The front is two minutes behind: the three minutes before it are judged, the two after are not. const over = Object.fromEntries(minutes.map((minute) => [minute, 500])) - const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7] }) + const behind = judge({ ok: true, minutes, perMinute: over, publishedThrough: minutes[7], unmetered: [] }) assert.equal(behind.rules.director503.status, 'trip') assert.deepEqual(Object.keys(behind.rules.director503.recent), minutes.slice(5, 8)) // A sparse 503 series: minutes after its last point but within the front are zero, not late. - const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1) }) + const sparse = judge({ ok: true, minutes, perMinute: { [minutes[2]]: 500 }, publishedThrough: minutes.at(-1), unmetered: [] }) assert.equal(sparse.rules.director503.status, 'pass') assert.equal(sparse.state.unverifiableStreak, 0) // No request series at all. - const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null }) + const empty = judge({ ok: true, minutes, perMinute: {}, publishedThrough: null, unmetered: [] }) assert.equal(empty.rules.director503.status, 'unverifiable') assert.match(empty.rules.director503.reason, /request series is empty/) assert.equal(empty.state.unverifiableStreak, 1) // A front too far behind to leave three published minutes in the read. - const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1] }) + const stale = judge({ ok: true, minutes, perMinute: {}, publishedThrough: minutes[1], unmetered: [] }) assert.equal(stale.rules.director503.status, 'unverifiable') assert.match(stale.rules.director503.reason, new RegExp(`published only through ${minutes[1]}`)) }) @@ -215,16 +222,21 @@ test('the duplicate read carries a positive control: no fresh ledger summary in assert.equal(gap.state.unverifiableStreak, 1) }) -test('the measured background is the median of the minutes before the flip, published gaps as zero', () => { +test('the measured background is the median of the metered minutes before the flip, published gaps as zero', () => { const minutes = ['2026-10-14T11:47', '2026-10-14T11:48', '2026-10-14T11:49'] const front = minutes[2] - assert.equal(measuredBackground({ '2026-10-14T11:48': 30 }, minutes, front).medianPerMinute, 0) - assert.equal(measuredBackground({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, minutes, front).medianPerMinute, 20) - assert.equal(measuredBackground({}, minutes, '2026-10-14T11:52').medianPerMinute, 0) + const background = (perMinute, publishedThrough, readMinutes = minutes, unmetered = []) => + measuredBackground({ perMinute, minutes: readMinutes, publishedThrough, unmetered }) + assert.equal(background({ '2026-10-14T11:48': 30 }, front).medianPerMinute, 0) + assert.equal(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front).medianPerMinute, 20) + assert.equal(background({}, '2026-10-14T11:52').medianPerMinute, 0) // Not published through the last minute yet, or no request series: no background yet. - assert.equal(measuredBackground({ [minutes[0]]: 10 }, minutes, minutes[1]), null) - assert.equal(measuredBackground({}, minutes, null), null) - assert.equal(measuredBackground({}, [], front), null) + assert.equal(background({ [minutes[0]]: 10 }, minutes[1]), null) + assert.equal(background({}, null), null) + assert.equal(background({}, front, []), null) + // An unmetered minute is left out, not read as zero; under half metered is no background. + assert.deepEqual(background({ [minutes[0]]: 10, [minutes[1]]: 30, [minutes[2]]: 20 }, front, minutes, [minutes[0]]), { medianPerMinute: 25, minutes: 2, source: 'measured' }) + assert.equal(background({}, front, minutes, minutes.slice(1)), null) }) test('any overrun trips at once', () => { @@ -391,6 +403,17 @@ test('the 503 background needs flipped-at or an explicit value; flipped-at must }) // A cell, Cloud Monitoring and Cloud Logging, answering from `script` per sample. +const METRICS_EVENT = 'orca_relay_runtime_metrics' +// Each instance's line every 30 s inside the read's timestamp bounds, newest first. +function metricsLines(filter, metrics) { + const [, from, to] = filter.match(/timestamp>="([^"]+)" AND timestamp<"([^"]+)"/) + const lines = [] + for (let at = Math.ceil(Date.parse(from) / 30_000) * 30_000; at < Date.parse(to); at += 30_000) { + for (const deltas of metrics(at)) lines.push({ timestamp: new Date(at).toISOString(), jsonPayload: deltas }) + } + return lines.reverse() +} + function fakeSources({ cell = () => ({}), // Logged at each sample's minute, so it stays fresh for the positive control. @@ -399,7 +422,9 @@ function fakeSources({ // Requests of every other code per minute; zero is a minute with no traffic at all. perOther = () => 1_000, seatControl = true, - rolloutLease = null + rolloutLease = null, + // Each director instance's runtime-metrics deltas for the 30 s ending at `endedAt`. + metrics = () => [{}] } = {}) { let samples = 0 const monitoringSpans = [] @@ -447,6 +472,7 @@ function fakeSources({ const summaries = ledger(samples).map((entry) => ({ ...entry, jsonPayload: { ...entry.jsonPayload, event: SUMMARY_EVENT } })) if (filter.includes(SEAT_EVENT)) return { stdout: JSON.stringify(seatControl ? summaries : []) } if (filter.includes(SUMMARY_EVENT)) return { stdout: JSON.stringify(ledger(samples)) } + if (filter.includes(METRICS_EVENT)) return { stdout: JSON.stringify(metricsLines(filter, metrics)) } return { stdout: '[]' } } return { fetchImpl, runGcloud, monitoringSpans, readRolloutLease: async () => rolloutLease } @@ -473,7 +499,7 @@ test('a segment samples once a minute, carries state to the next segment, and st assert.equal(first.state, 'watching') assert.equal(lines.length, 3) assert.equal(first.background.source, 'measured') - assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\n') + assert.equal(await readFile(join(directory, 'out'), 'utf8'), 'state=watching\nverdict=\nflip=admitMode\n') const second = await watchSegment(config, dependencies) assert.equal(second.state, 'tripped') assert.equal(second.samples, 5) @@ -517,7 +543,7 @@ test('a watch that ended without a verdict records failed, which the workflow fl assert.equal((await recordOutcome(join(directory, 'watching.json'), output)).state, 'failed') await writeFile(join(directory, 'tripped.json'), JSON.stringify(freshState({ state: 'tripped', verdict: 'flip back x:\ny' }))) assert.equal((await recordOutcome(join(directory, 'tripped.json'), output)).state, 'tripped') - assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\n$/) + assert.match(await readFile(output, 'utf8'), /state=tripped\nverdict=flip back x: y\nflip=admitMode\n$/) } finally { await rm(directory, { recursive: true, force: true }) } @@ -525,10 +551,20 @@ test('a watch that ended without a verdict records failed, which the workflow fl // Google Storage and the director, as far as the flag tool's flip back reaches them. `onUpload` // returns 'conflict' to play another writer landing first, optionally with its own admitMode. -function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = true } = {}) { +function fakeFlipBack({ + generation = '7', + onUpload = () => 'ok', + cellAnswers = true, + flags = { admitMode: 'reserve', readinessLocal: true }, + supportedFlags = { + admitMode: { type: 'enum', values: ['db', 'reserve'] }, + readinessLocal: { type: 'boolean' }, + phonePath: { type: 'enum', values: ['db', 'memory'] } + } +} = {}) { const calls = [] let uploads = 0 - let object = { generation: Number(generation), flags: { admitMode: 'reserve', readinessLocal: true } } + let object = { generation: Number(generation), flags } let applied = object const fetchImpl = async (url, init = {}) => { const text = String(url) @@ -557,10 +593,7 @@ function fakeFlipBack({ generation = '7', onUpload = () => 'ok', cellAnswers = t role: 'cell', cellId: CELL, flagsApplied: applied, - supportedFlags: { - admitMode: { type: 'enum', values: ['db', 'reserve'] }, - readinessLocal: { type: 'boolean' } - }, + supportedFlags, admitModeEffective: applied.flags.admitMode, runtime: { controls: 10 } }) @@ -755,6 +788,111 @@ test('a quiet fleet with zero director 503s watches on and never trips', async ( } }) +// 10-20 c29: a hand flip back to db mid-watch crashed the segment ("result.pages is not iterable"). +test('a segment that waits for reserve, then sees a hand flip back to db, stops without crashing', async () => { + const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) + try { + let clock = START + const config = parseAt( + watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': '10', 'flipped-at': new Date(START - 2 * MINUTE).toISOString() }) + ) + const db = { flagsApplied: { generation: 5, flags: { admitMode: 'db', readinessLocal: true } }, admitModeEffective: 'db' } + const lines = [] + const state = await watchSegment(config, { + ...fakeSources({ cell: (sample) => (sample === 1 || sample >= 4 ? db : {}) }), + idToken: 'id', + retryDelayMs: 0, + now: () => clock, + sleep: async (ms) => { clock += ms }, + log: (line) => lines.push(JSON.parse(line)) + }) + assert.equal(lines[0].rules.reserve.status, 'pending') + assert.equal(state.state, 'stopped') + assert.match(state.verdict, /is not in reserve mode \(switch file says db\)/) + assert.equal(lines.length, 4) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +// 2026-10-18's false trips: 503s tripled with traffic, nearly all a host re-dialling early. +const rateLimited = (perLine) => ({ placementRejectionsByReasonDelta: { 'host-rate-limited': perLine }, assign503sByCauseDelta: { 'placement-lane': perLine } }) + +async function watchFor(minutes, sources, flippedAt = START - 2 * MINUTE) { + const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) + try { + let clock = START + const config = parseAt( + watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes), 'flipped-at': new Date(flippedAt).toISOString() }) + ) + const lines = [] + const state = await watchSegment(config, { + ...sources, + idToken: 'id', + retryDelayMs: 0, + now: () => clock, + sleep: async (ms) => { clock += ms }, + log: (line) => lines.push(JSON.parse(line)) + }) + return { state, lines } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +test('a host re-dialling inside its own interval never trips the 503 rule, however much it grows', async () => { + // Background 20/min, all re-dials; then 50/min, 48 of them re-dials. All 503s would trip at 40. + const flippedAt = START - 2 * MINUTE + const { state, lines } = await watchFor(20, fakeSources({ + per503: (minute) => (minute < flippedAt ? 20 : 50), + metrics: (at) => [rateLimited(at <= flippedAt ? 10 : 24)] + }), flippedAt) + assert.equal(state.state, 'watching') + assert.deepEqual(state.background, { medianPerMinute: 0, minutes: 10, source: 'measured' }) + assert.ok(lines.slice(1).every((line) => line.rules.director503.status === 'pass')) + assert.ok(lines.slice(1).every((line) => Object.values(line.rules.director503.recent).every((count) => count <= 2))) +}) + +test('503s the director could not serve trip after three minutes over the line, beside the re-dials', async () => { + const flippedAt = START - 2 * MINUTE + const { state } = await watchFor(10, fakeSources({ + per503: (minute) => (minute < flippedAt ? 20 : 60), + // 10 re-dials a minute throughout; the 40 more after the flip are queue waits that timed out. + metrics: (at) => [{ + ...rateLimited(5), + stickyRejectionsByReasonDelta: at <= flippedAt ? {} : { 'wait-timeout': 20 } + }] + }), flippedAt) + assert.equal(state.state, 'tripped') + assert.deepEqual(state.background, { medianPerMinute: 10, minutes: 10, source: 'measured' }) + assert.match(state.verdict, /director503: director 503s above 30\/min for 3 min \(50, 50, 50\)/) +}) + +test('a minute no director instance logged runtime metrics for is unverifiable, not zero', async () => { + // Two instances, both silent for three minutes in the middle of the read. + const endedAt = START + 2 * MINUTE + const silent = (at) => at > endedAt - 5 * MINUTE && at <= endedAt - 2 * MINUTE + const reader = fakeSources({ per503: () => 50, metrics: (at) => (silent(at) ? [] : [rateLimited(12), rateLimited(12)]) }) + const read = await readDirector503Minutes( + { ...fakeReader(), fetch: reader.fetchImpl, runGcloud: reader.runGcloud }, + { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 } + ) + assert.equal(read.ok, true) + assert.deepEqual(read.unmetered, read.minutes.slice(5, 8)) + assert.equal(read.perMinute[read.minutes[0]], 2) + const judged = evaluateWatchSample(freshState({ background: { medianPerMinute: 0, source: 'measured' } }), sampleAt(START, { director503: read }), LIMIT) + assert.equal(judged.rules.director503.status, 'unverifiable') + assert.match(judged.rules.director503.reason, /no director runtime metrics for/) + // No line at all in the read is short, not a quiet fleet. + const unlogged = fakeSources({ per503: () => 50, metrics: () => [] }) + const none = await readDirector503Minutes( + { ...fakeReader(), fetch: unlogged.fetchImpl, runGcloud: unlogged.runGcloud }, + { projectId: 'onorca-cloud', directorService: 'orca-cloud-relay', endedAt, minutes: 10 } + ) + assert.equal(none.ok, false) + assert.match(none.error, /runtime metrics read failed, truncated, or short/) +}) + test('no request series at all is unverifiable and trips at the limit', async () => { const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) try { @@ -956,3 +1094,346 @@ test('an alarm is an escaped error annotation everywhere, and in write mode a pa assert.match(lines.at(-2), /^::error title=Reserve watch refused for /) assert.match(lines.at(-1), /nothing pages: --project-id is not a GCP project id/) }) + +// Step 6 phone rules. A quiet window, minute by minute: ordinary 4401s from phones whose pairing +// was removed, an occasional failed confirm or half-open control. Synthetic: replaying a recorded +// production window needs read access this suite does not have. +const QUIET_PHONE_MINUTE = (index) => ({ + credentialRefusals: [0, 1, 0, 2, 1][index % 5], + confirmFailures: [0, 0, 1, 0][index % 4], + attachDeadlines: [1, 0, 2, 0, 1, 3][index % 6], + acceptDatabaseErrors: 0, + renewalFailures: 0, + completedAccepts: [10, 12, 8][index % 3] +}) +const phoneRead = (minuteCount, perMinute) => { + const minutes = Array.from({ length: minuteCount }, (_, index) => `2026-10-19T12:${String(index).padStart(2, '0')}`) + const rows = minutes.map((_, index) => perMinute(index)) + const series = Object.fromEntries(Object.keys(rows[0]).map((signal) => [signal, rows.map((row) => row[signal])])) + return { ok: true, minutes, series, unmetered: [] } +} +const memoryCell = (overrides = {}) => healthyCell({ phonePathConfigured: 'memory', resumeAuditDropped: 0, ...overrides }) +const phoneState = (overrides = {}) => + freshState({ + flippedAt: START - 10 * MINUTE, + phoneSince: START - 10 * MINUTE, + phoneBackground: measuredPhoneBackground(phoneRead(10, QUIET_PHONE_MINUTE)), + ...overrides + }) +// The last 10 minutes as of watch minute `offset`, the newest minutes shaped by `spike`. +const phoneSampleAt = (offset, spike = () => ({}), cell = memoryCell()) => + sampleAt(START + offset * MINUTE, { + cell, + phone: phoneRead(10, (index) => ({ ...QUIET_PHONE_MINUTE(offset + index), ...spike(index) })) + }) + +test('phone rules: a quiet hour replays without a trip', () => { + const results = run(Array.from({ length: 60 }, (_, offset) => phoneSampleAt(offset)), phoneState({ deadlineAt: START + 120 * MINUTE })) + assert.equal(results.length, 60) + assert.ok(results.every((result) => result.state.state === 'watching' && result.state.phoneTrip === null)) + assert.deepEqual(Object.keys(results[0].rules).filter((name) => name.startsWith('phone')), PHONE_RULE_NAMES) +}) + +test('phone rules: judged as a share of attempts, over half and over the background', () => { + assert.equal(phoneShareLine(0), 0.5) + assert.equal(phoneShareLine(0.3), 0.55) + assert.equal(phoneShareLine(0.5), 0.6) + // Exactly half is not over; nor is a minute with under 4 attempts. + assert.equal(phoneMinuteOver('credentialRefusals', 9, 9, 0.5), false) + assert.equal(phoneMinuteOver('credentialRefusals', 11, 10, 0.5), true) + assert.equal(phoneMinuteOver('credentialRefusals', 3, 0, 0.5), false) + assert.equal(phoneMinuteOver('credentialRefusals', 4, 0, 0.5), true) + // Each accept makes one confirm: every confirm failing is all of them, not half the attempts. + assert.equal(phoneMinuteOver('confirmFailures', 4, 4, 0.6), true) + assert.equal(phoneMinuteOver('confirmFailures', 2, 4, 0.5), false) + assert.equal(phoneMinuteOver('confirmFailures', 9, 3, 0.5), false) + const background = measuredPhoneBackground(phoneRead(10, () => ({ ...QUIET_PHONE_MINUTE(0), credentialRefusals: 1, completedAccepts: 9 }))) + assert.equal(background.share.credentialRefusals, 0.1) + assert.equal(measuredPhoneBackground({ ...phoneRead(10, QUIET_PHONE_MINUTE), unmetered: ['2026-10-19T12:03'] }), null) +}) + +test('phone rules: each fires on its own spike, and the watch goes on with the reserve rules', () => { + const spikes = { + phoneCredentialRefusals: { credentialRefusals: 25 }, + phoneConfirmFailures: { confirmFailures: 25 }, + phoneAttachDeadlines: { attachDeadlines: 25 }, + phoneAcceptDatabaseErrors: { acceptDatabaseErrors: 25 } + } + for (const [rule, counts] of Object.entries(spikes)) { + const [result] = run([phoneSampleAt(0, (index) => (index >= 7 ? counts : {}))], phoneState()) + assert.equal(result.state.state, 'watching', rule) + assert.match(result.state.phoneTrip, new RegExp(`turn off the phone path on ${CELL}: ${rule}: `), rule) + } + // Two minutes over the line is not three. + const [twoMinutes] = run([phoneSampleAt(0, (index) => (index >= 8 ? { credentialRefusals: 25 } : {}))], phoneState()) + assert.equal(twoMinutes.state.phoneTrip, null) + // Once the watch turned the phone path off, the phone rules stand down. + const [off] = run([phoneSampleAt(0, () => ({ credentialRefusals: 50 }))], phoneState({ phoneOffAt: START })) + assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), []) +}) + +test('phone rules: judged only while the switch file says memory; reserve rules still flip the cell back', () => { + const [off] = run([sampleAt(START, { cell: healthyCell({ phonePathConfigured: 'db' }) })], freshState()) + assert.deepEqual(Object.keys(off.rules).filter((name) => name.startsWith('phone')), []) + const [overrun] = run([phoneSampleAt(0, () => ({}), memoryCell({ enforcedUnits: 2_001 }))], phoneState()) + assert.equal(overrun.state.state, 'tripped') + assert.equal(overrun.state.flip, 'admitMode') +}) + +test('phone rules: missing phone data turns off only the phone path, and needs a flipped-at background', () => { + const gap = (offset) => ({ ...phoneSampleAt(offset), phone: { ...phoneSampleAt(offset).phone, unmetered: ['2026-10-19T12:09'] } }) + const results = run([gap(0), gap(1), gap(2)], phoneState()) + assert.equal(results.at(-1).state.state, 'watching') + assert.equal(results.at(-1).state.unverifiableStreak, 0) + assert.match(results.at(-1).state.phoneTrip, /unverifiable: 3 samples/) + const [noBackground] = run([phoneSampleAt(0)], freshState({ flippedAt: null })) + assert.match(noBackground.rules.phoneCredentialRefusals.reason, /--flipped-at/) + // A gap before the judged 3 minutes changes nothing. + const early = { ...phoneSampleAt(0), phone: { ...phoneSampleAt(0).phone, unmetered: ['2026-10-19T12:02'] } } + const [earlyGap] = run([early], phoneState()) + assert.equal(earlyGap.rules.phoneCredentialRefusals.status, 'pass') +}) + +test('phone rules: memory set partway through a watch measures from then, not from the reserve flip', () => { + const state = freshState({ flippedAt: START - 60 * MINUTE }) + const [before, after] = run([sampleAt(START), phoneSampleAt(1)], state) + assert.equal(before.state.phoneSince, null) + assert.equal(after.state.phoneSince, START) + assert.equal(after.rules.phoneCredentialRefusals.status, 'pending') +}) + +test('phone rules: audit drops and failing renewals page once when sustained, and flip nothing', () => { + const dropping = (offset, dropped) => phoneSampleAt(offset, () => ({}), memoryCell({ resumeAuditDropped: dropped })) + // One drop, then none: no page. + const single = run([dropping(0, 0), dropping(1, 1), ...Array.from({ length: 8 }, (_, index) => dropping(index + 2, 1))], phoneState()) + assert.ok(single.every((result) => result.pages.length === 0)) + const rising = run(Array.from({ length: 9 }, (_, offset) => dropping(offset, offset * 3)), phoneState()) + assert.deepEqual(rising.map((result) => result.pages.join()), ['', '', '', '', '', '', 'audit-drops', '', '']) + const renewing = run(Array.from({ length: 7 }, (_, offset) => phoneSampleAt(offset, () => ({ renewalFailures: 1 }))), phoneState()) + assert.deepEqual(renewing.map((result) => result.pages.join()).filter(Boolean), ['renewal-failures']) + assert.ok([...rising, ...renewing].every((result) => result.state.state === 'watching' && result.state.phoneTrip === null)) +}) + +test('the cell phone read sums each minute from the cell runtime metrics lines, and refuses an old image', async () => { + const endedAt = Date.parse('2026-10-19T12:03:00Z') + const line = (at, payload) => ({ + timestamp: new Date(at).toISOString(), + jsonPayload: { + phoneRejectionsByCodeDelta: {}, + resumeConfirmFailuresByCodeDelta: {}, + phoneAttachDeadlinesDelta: 0, + phoneAcceptDatabaseErrorsDelta: 0, + resumeRenewalFailuresDelta: 0, + clientAcceptCompletedDelta: 0, + ...payload + } + }) + let filter + const read = await readCellPhoneMinutes( + fakeReader({ + runGcloud: async (args) => { + filter = args[2] + return { + stdout: JSON.stringify([ + line(endedAt - 150_000, { phoneRejectionsByCodeDelta: { 4401: 2, 4404: 9 } }), + line(endedAt - 120_000, { resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1, error: 2 } }), + line(endedAt - 90_000, { phoneAttachDeadlinesDelta: 3, phoneAcceptDatabaseErrorsDelta: 1, resumeRenewalFailuresDelta: 4, clientAcceptCompletedDelta: 7 }) + ]) + } + } + }), + { cellId: CELL, endedAt, minutes: 3 } + ) + assert.match(filter, /resource\.type="gce_instance"/) + assert.match(filter, new RegExp(`jsonPayload\\.cellId="${CELL}"`)) + assert.deepEqual(read.series, { + credentialRefusals: [2, 0, 0], + confirmFailures: [3, 0, 0], + attachDeadlines: [0, 3, 0], + acceptDatabaseErrors: [0, 1, 0], + renewalFailures: [0, 4, 0], + completedAccepts: [0, 7, 0] + }) + assert.deepEqual(read.unmetered, ['2026-10-19T12:02']) + const old = await readCellPhoneMinutes( + fakeReader({ runGcloud: async () => ({ stdout: JSON.stringify([{ timestamp: new Date(endedAt - 1).toISOString(), jsonPayload: {} }]) }) }), + { cellId: CELL, endedAt, minutes: 3 } + ) + assert.match(old.error, /no phone counters/) + assert.equal(old.permanent, true) +}) + +// The watch over a memory cell whose phone metrics come from `phoneMetrics(at)`. +async function phoneWatch(phoneMetrics, minutes) { + const directory = await mkdtemp(join(tmpdir(), 'relay-reserve-watch-')) + try { + let clock = START + const lines = [] + const config = parseAt(watchValues({ 'state-file': join(directory, 'state.json'), 'segment-minutes': String(minutes) })) + const sources = fakeSources({ + cell: () => ({ flagsApplied: { generation: 4, flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } } }) + }) + const runGcloud = async (args) => + args[2]?.includes('gce_instance') ? { stdout: JSON.stringify(metricsLines(args[2], (at) => [phoneMetrics(at)])) } : sources.runGcloud(args) + const state = await watchSegment(config, { + ...sources, + runGcloud, + idToken: 'id', + accessToken: 'access', + audit: {}, + retryDelayMs: 0, + now: () => clock, + sleep: async (ms) => { clock += ms }, + log: (line) => lines.push(line) + }) + return { state, lines } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} +const phoneCounters = (overrides = {}) => ({ + phoneRejectionsByCodeDelta: {}, + resumeConfirmFailuresByCodeDelta: {}, + phoneAttachDeadlinesDelta: 0, + phoneAcceptDatabaseErrorsDelta: 0, + resumeRenewalFailuresDelta: 0, + clientAcceptCompletedDelta: 5, + ...overrides +}) + +test('a phone trip whose phonePath=db write fails keeps watching, retries each sample, and pages after 3 failures', async () => { + // No bucket here, so every write fails. + const { state, lines } = await phoneWatch((at) => phoneCounters(at > START ? { phoneAttachDeadlinesDelta: 20 } : {}), 15) + assert.equal(state.state, 'watching') + assert.equal(state.phoneOffAt, null) + assert.match(state.phoneTrip, /phoneAttachDeadlines: /) + const failures = lines.filter((line) => line.includes('orca_relay_reserve_watch_phone_path_off_failed')) + assert.ok(failures.length > 3, `${failures.length} failed writes`) + assert.equal(state.phoneOffFailures, failures.length) + const alarms = lines.filter((line) => line.includes('may still have phonePath=memory')) + assert.equal(alarms.length, 1) +}) + +test('a phone trip still unwritten when the watch ends leaves it to the flip back', () => { + const [last] = run([phoneSampleAt(0)], phoneState({ deadlineAt: START, phoneTrip: 'turn off the phone path on x: phoneAttachDeadlines: y' })) + assert.equal(last.state.state, 'tripped') + assert.equal(last.state.flip, 'phonePath') + assert.match(last.state.verdict, /phoneAttachDeadlines: y; phonePath=db did not land before the watch ended/) +}) + +test('a phone background broken by a restart or roll refuses the watch instead of turning phones off for certain', async () => { + const { state } = await phoneWatch((at) => (at < START - 8 * MINUTE ? {} : phoneCounters()), 5) + assert.equal(state.state, 'refused') + assert.match(state.verdict, /phone background.*incomplete.*later --flipped-at/) +}) + +test('a phone-rule flip back removes phonePath, and does nothing once the file no longer says memory', async () => { + const { fetchImpl, calls } = fakeFlipBack({ flags: { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } }) + const result = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fetchImpl, [])) + assert.equal(result.written, true) + const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) + assert.match(upload.body, /"flags":\{"admitMode":"reserve","readinessLocal":true\}/) + assert.ok(!calls.some((call) => call.url.endsWith('/v1/admin/cell-admit-mode') && JSON.parse(call.body).admitMode)) + const done = await flipBack(flipValues({ flip: 'phonePath' }), flipDependencies(fakeFlipBack().fetchImpl, [])) + assert.equal(done.alreadyDb, true) +}) + +test('both flip backs land on a cell rolled back to an image without the phonePath switch', async () => { + const flags = { admitMode: 'reserve', readinessLocal: true, phonePath: 'memory' } + for (const flip of ['admitMode', 'phonePath']) { + const { fetchImpl, calls } = fakeFlipBack({ flags, supportedFlags: { admitMode: { type: 'enum', values: ['db', 'reserve'] }, readinessLocal: { type: 'boolean' } } }) + const result = await flipBack(flipValues({ flip }), flipDependencies(fetchImpl, [])) + assert.equal(result.written, true, flip) + const upload = calls.find((call) => call.url.includes('/upload/storage/v1/')) + assert.doesNotMatch(upload.body, /phonePath/, flip) + } +}) + +test('workflow: the flip back writes what the watch says tripped, and the whole cell when it says nothing', () => { + const workflow = watchWorkflow + assert.match(workflow, /flip: \$\{\{ steps\.outcome\.outputs\.flip \}\}/) + assert.match(workflow, /FLIP: \$\{\{ needs\.watch\.outputs\.flip \|\| 'admitMode' \}\}/) + assert.match(workflow, /--flip "\$\{FLIP\}"/) + // A failed phone flip back pages with the phone wording. + assert.match(workflow, /--verdict "\$\{VERDICT\}" --flip "\$\{FLIP\}"\n/) + assert.match(workflow, /ORCA_RELAY_CONTROL_ACCESS_TOKEN="\$\(gcloud auth print-access-token\)"\n\s+echo "::add-mask::/) +}) + + +test('a failed phone flip back and the phone pages say what is still on', async () => { + const lines = [] + const runGcloud = async () => ({ stdout: '' }) + await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'flip-back-failed', reason: 'x', flip: 'phonePath' }, { runGcloud, log: (line) => lines.push(line) }) + await raiseAlarm({ mode: 'dry-run', cellId: CELL, projectId: 'onorca-cloud', kind: 'renewal-failures', reason: 'y' }, { runGcloud, log: (line) => lines.push(line) }) + assert.match(lines[0], /may still have phonePath=memory/) + assert.doesNotMatch(lines[0], /reserve mode/) + assert.match(lines[1], /cannot renew phone credentials/) +}) + +// Recorded, not synthetic: 26 h of c7, c28 and c29 runtime metrics (read-only, 10-18/19). The phone +// counters are not on those images yet, so auth failures (host and phone 4401s, an over-count) +// stand in for the refused or failed attempts, against completed accepts. They cannot stand in for +// confirm failures: c28's 04:08Z minutes (9/9, 9/9, 11/10) read as confirms are a total outage. +const PHONE_RULE_SIGNAL_NAMES = ['credentialRefusals', 'confirmFailures', 'attachDeadlines', 'acceptDatabaseErrors'] +const recordedPhoneMinutes = JSON.parse(readFileSync(new URL('./relay-phone-watch-recorded-quiet-minutes.json', import.meta.url), 'utf8')).cells +// The minute index a recorded watch of `signal` from `start` trips at, or null; `minute(i)` may rewrite a minute. +function recordedTrip({ authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts }, signal, start, watchMinutes, minute = (i) => [failures[i], accepts[i]]) { + const { readMinutes, consecutiveMinutes } = PHONE_WATCH_RULES + const refused = failures.slice(start - readMinutes, start) + const background = measuredPhoneBackground({ + ok: true, + minutes: refused, + unmetered: [], + series: { ...Object.fromEntries(PHONE_RULE_SIGNAL_NAMES.map((name) => [name, refused])), completedAccepts: accepts.slice(start - readMinutes, start) } + }) + const line = phoneShareLine(background.share[signal]) + let run = 0 + for (let index = start; index < start + watchMinutes; index += 1) { + run = phoneMinuteOver(signal, ...minute(index), line) ? run + 1 : 0 + if (run >= consecutiveMinutes) return index + } + return null +} + +test('phone rules: 26 recorded hours of c7, c28 and c29 trip no attempt rule in any 60- or 300-minute watch', () => { + assert.deepEqual(Object.keys(recordedPhoneMinutes), ['production-gce-c7', 'production-gce-c28', 'production-gce-c29']) + for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) { + for (const watchMinutes of [60, 300]) { + for (let start = 10; start + watchMinutes <= minutes.authFailuresPerMinute.length; start += 5) { + assert.equal(recordedTrip(minutes, 'credentialRefusals', start, watchMinutes), null, `${cell}: a ${watchMinutes}-min watch from minute ${start} trips`) + } + } + } +}) + +// From minute 30 of each recorded hour, a broken path; it must trip by its third minute (sooner +// when the minute before was already over) wherever those minutes carry 4 attempts each. +const BROKEN_PATHS = { + 'two refusals per accept': ['credentialRefusals', (failed, accepted) => [Math.max(2 * accepted, 4), accepted]], + 'every accept errors on the database, phones give up after one try': ['acceptDatabaseErrors', (failed, accepted) => [accepted, 0]], + // Only trips are tested: the auth-failure stand-in cannot measure confirm false trips. + 'every confirm fails': ['confirmFailures', (failed, accepted) => [accepted, accepted]] +} +test('phone rules: the same recorded hours trip on each broken path wherever its minutes are judged', () => { + for (const [name, [signal, breakMinute]] of Object.entries(BROKEN_PATHS)) { + for (const [cell, minutes] of Object.entries(recordedPhoneMinutes)) { + const { authFailuresPerMinute: failures, completedAcceptsPerMinute: accepts } = minutes + let judged = 0 + for (let start = 10; start + 60 <= failures.length; start += 5) { + const broken = (index) => (index >= start + 30 ? breakMinute(failures[index], accepts[index]) : [failures[index], accepts[index]]) + const judgedMinutes = [0, 1, 2].every((offset) => { + const [count, accepted] = broken(start + 30 + offset) + return (signal === 'confirmFailures' ? accepted : count + accepted) >= PHONE_WATCH_RULES.minAttemptsPerMinute + }) + if (!judgedMinutes) continue + judged += 1 + const at = recordedTrip(minutes, signal, start, 60, broken) + // Confirms may trip earlier: the stand-in reads as failing confirms on c28 (see above). + const earliest = signal === 'confirmFailures' ? start : start + 30 + assert.ok(at !== null && at >= earliest && at <= start + 32, `${cell} ${name}: from minute ${start + 30} trips at ${at}`) + } + // Busy cells are judged most of the day; low-volume c7 rarely reaches 4 attempts a minute. + if (cell !== 'production-gce-c7') assert.ok(judged > 150, `${cell} ${name}: ${judged} judged`) + } + } +}) diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md index 5788ca6d3598..2e23041dc36f 100644 --- a/cloud/docs/orca-relay-operations.md +++ b/cloud/docs/orca-relay-operations.md @@ -241,7 +241,19 @@ Every later change must use the selector CAS. Deploy the selector-compatible director before cutover. Blue/green deployment creates a separate `selector-rollback` revision at minimum instances zero, promotes a second compatible revision, and retains older revisions through the -stabilization window. Cutover removes every older revision and refuses to +stabilization window. With a floor above one, promotion first moves 20% of +traffic and waits (up to six minutes) until every floor instance has served a +request, because Cloud Run routes new floor instances only ~100 s after they +start; a timeout returns all traffic to the previous serving revision. + +If the deploy job is cancelled or loses its runner during that wait, the 20/80 +split stays in place, and both the next deploy and the rehome workflow refuse a +director without one 100% revision. Restore it with +`gcloud run services update-traffic orca-cloud-relay --project onorca-cloud --region us-central1 --to-revisions==100` +(the revision that held 100% before the deploy). A timed-out deploy restores +that itself but removes every tag, including `selector-rollback`, so the rehome +workflow's rollback-digest check refuses (`rollback tag missing`) until the next +successful director deploy. Cutover removes every older revision and refuses to proceed unless only the compatible active and rollback revisions remain. First inspect the current generation and exact membership: diff --git a/cloud/docs/relay-workflows.md b/cloud/docs/relay-workflows.md index edab96a42771..72b749b79de0 100644 --- a/cloud/docs/relay-workflows.md +++ b/cloud/docs/relay-workflows.md @@ -826,7 +826,7 @@ this order: |---|---|---| | Dead-man tripped | runtime-status `admitModeEffective=db` under a reserve switch file | Exact | | Overrun | runtime-status `enforcedConnectionUnits` over `connectionCapacity.hardCap` | Invariant check: the ledger already refuses past its cap, so this trips only if that refusal broke. It counts seats, bookings, in-flight and phone units | -| 503s | Director `request_count` 503s above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line | +| 503s | Director non-drain 503s (the roll ladder's measure: `request_count` 503s less the drain-return deferrals and the `host-rate-limited` and `host-in-flight` re-dials the director's `orca_relay_runtime_metrics` count) above max(1.5x background, background + 20) for 3 complete minutes | Proxy that can miss: fleet-wide, since no signal attributes a 503 to a cell, so a problem on one cell can stay under the fleet's line | | Duplicate past grace | The ledger reconcile's `orca_relay_reserve_ledger_seat_behind_row` for the cell, reported again more than 90 s (60 s demotion close + 30 s map lag) after it was first seen, keyed by director instance, cell, epoch and row epoch | Proxy that can only over-count: the line names no host, so two hosts sharing that key read as one seat | | Host drop | runtime-status `controls` fell more than 5% (and at least 10 hosts) within 5 minutes with no `draining` sample | Exact | | Ledger drift | Queued plus dropped rows in each director's newest `orca_relay_reserve_ledger_summary`, over the cell's controls, above 0.5% for 2 minutes | Proxy that can only over-count: the summaries are fleet-wide, so exact only while this is the only reserve cell | @@ -837,7 +837,15 @@ newest minute any code has a point for is how far the series is published: a 503 before it with no point had no 503s, so a quiet fleet reads zero. Minutes past it are left out, not counted; fewer than 3 published minutes in the read, no request series at all, or an unparseable timestamp is unverifiable. This assumes the codes publish together; one code lagging the others -delays a trip by its lag but cannot hide one, because a later read sees its points. Until the +delays a trip by its lag but cannot hide one, because a later read sees its points. A host's own +early or duplicate re-dial says nothing about a cell and grows with traffic: on 2026-10-18 it was +~98% of 503s and tripped c7 back twice with no fault. The subtraction is only trusted where the +director logged: a judged minute no runtime-metrics line covers is unverifiable, a 10-minute read +with fewer lines than a single instance logs (20) or at its entry limit is unverifiable, and the +measured background leaves such minutes out (and is not measured with fewer than half of its 10 +metered). Coverage is any instance, not every one: an instance whose lines are missing while others +log under-subtracts its re-dials, which can only push toward a trip, never hide one. The same measure is the +background and `background-503-per-minute`. Until the minutes before `flipped-at` are published (3 minutes past it) the 503 rule is pending, which does not add to the streak. The duplicate read also takes `orca_relay_reserve_ledger_summary`, which every placing director logs each minute, in the same query and window: with no summary from the diff --git a/cloud/infra/terraform/environments/staging.tfvars b/cloud/infra/terraform/environments/staging.tfvars index d74abd85b75c..bebb51a9126a 100644 --- a/cloud/infra/terraform/environments/staging.tfvars +++ b/cloud/infra/terraform/environments/staging.tfvars @@ -48,8 +48,8 @@ relay_gce_cells = { boot_disk_gb = 30 boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-7" capacity_requests = 4000 - # Step-5 image G (main 2de73812, reports preAuthLatencyMs). Never 2d0f6e6: it cannot report capped telemetry. - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d" + # Step-6 image J (main 30f5bef9, phonePath). Never 2d0f6e6: it cannot report capped telemetry. + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4" # Step-5 staging e2e: c1 is the second US reserve cell, so it needs a cap and rehome trust. connection_hard_cap = 600 connection_unobserved_bound = 60 @@ -86,7 +86,7 @@ relay_gce_cells = { boot_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-121-18867-528-21" capacity_requests = 6000 database_pool_max = 10 - image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:07b5bf67a7706b2a61ef5e9a6d8ce85ee37c2a08609496a94eb5254881b5b87d" + image = "us-central1-docker.pkg.dev/onorca-cloud-staging/orca-cloud/relay@sha256:21f1fead4d3b9900d3b1179fd72569dcba548dc0ee29c76c8389fec94f74a7b4" initially_enabled = false connection_hard_cap = 3000 connection_unobserved_bound = 60 diff --git a/cloud/infra/terraform/relay-staging-deploy-iam.tf b/cloud/infra/terraform/relay-staging-deploy-iam.tf index a45652f2f682..74ee413f109b 100644 --- a/cloud/infra/terraform/relay-staging-deploy-iam.tf +++ b/cloud/infra/terraform/relay-staging-deploy-iam.tf @@ -170,3 +170,13 @@ resource "google_project_iam_member" "github_staging_relay_deploy_compute_viewer role = "roles/compute.viewer" member = google_service_account.github_staging_relay_deploy[0].member } + +# Deploy Relay Staging step "Deploy director blue/green" with a director floor above one reads +# the candidate's Cloud Run request log to confirm every floor instance serves before the full move. +resource "google_project_iam_member" "github_staging_relay_deploy_logging_viewer" { + count = local.create_staging_relay_deploy_identity ? 1 : 0 + + project = var.project_id + role = "roles/logging.viewer" + member = google_service_account.github_staging_relay_deploy[0].member +} diff --git a/cloud/package.json b/cloud/package.json index 6701f3859fd1..de5daabc60dc 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -17,7 +17,7 @@ "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-control-flag-alerts.test.mjs dev/scripts/relay-db-fault-alerts.test.mjs dev/scripts/relay-lock-contention-alerts.test.mjs dev/scripts/relay-reconnect-herd-alerts.test.mjs dev/scripts/relay-reserve-dead-man-alert.test.mjs dev/scripts/relay-region-hint-metrics.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs && pnpm test:runner", - "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "test": "pnpm -r --filter \"!@orca-cloud/relay\" test && pnpm --filter @orca-cloud/relay test && node --test dev/scripts/check-relay-same-cap-headroom.test.mjs dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/drive-relay-director-deploy.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/measure-relay-drain-disconnect-gap.test.mjs dev/scripts/operate-relay-cell-flags.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/push-gateway-workflow.test.mjs dev/scripts/push-gateway-recovery.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-clock-skew-monitor-workflow.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-duplicate-assign.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-resume-phone.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-load-splice-retry.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-same-cap-job-mode-conditions.test.mjs dev/scripts/relay-same-cap-script-census.test.mjs dev/scripts/relay-same-cap-shadow-gate.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/watch-relay-reserve-cell.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck", "test:runner": "node --import tsx --conditions=orca-source --test dev/scripts/run-relay-tests.test.mjs" }, @@ -30,7 +30,7 @@ }, "engines": { "node": ">=24 <27", - "pnpm": ">=10" + "pnpm": ">=12" }, - "packageManager": "pnpm@10.34.6" + "packageManager": "pnpm@12.14.0+sha512.c18ba6d87993d3d164cb5e8948f32b2c06ee854487e06cd8d86781342fb3e5bb4634367034711e86802baa9c37bea906ce1929ad08d70884c97d801df13389a6" } diff --git a/cloud/packages/relay-contract/src/close-codes.ts b/cloud/packages/relay-contract/src/close-codes.ts index cb2c6307b906..12c125e3fcaa 100644 --- a/cloud/packages/relay-contract/src/close-codes.ts +++ b/cloud/packages/relay-contract/src/close-codes.ts @@ -8,3 +8,15 @@ export const RELAY_CLOSE_CODE = { } as const export type RelayCloseCode = (typeof RELAY_CLOSE_CODE)[keyof typeof RELAY_CLOSE_CODE] + +// Mirror of MOBILE_RELAY_PHONE_CLOSE_REASON in src/shared/mobile-relay-close-codes.ts +// in the Orca app repo half. Close reasons on a phone socket that refine its close +// code; not RELAY_HOST_CLOSE_REASON, which names host control-socket closes. +export const RELAY_PHONE_CLOSE_REASON = { + // A HOST_OFFLINE sent while the host's control looked alive but its data link + // missed hostAttachDeadlineMs; the phone redials the same cell once, quickly. + HOST_ATTACH_TIMEOUT: 'attach-timeout' +} as const + +export type RelayPhoneCloseReason = + (typeof RELAY_PHONE_CLOSE_REASON)[keyof typeof RELAY_PHONE_CLOSE_REASON] diff --git a/cloud/packages/relay-contract/src/contract.test.ts b/cloud/packages/relay-contract/src/contract.test.ts index 391a0877c651..2a1a134ed6ae 100644 --- a/cloud/packages/relay-contract/src/contract.test.ts +++ b/cloud/packages/relay-contract/src/contract.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { RELAY_CLOSE_CODE } from './close-codes.js' +import { RELAY_CLOSE_CODE, RELAY_PHONE_CLOSE_REASON } from './close-codes.js' import { AuthRefreshSchema, DrainSchema, @@ -52,6 +52,8 @@ describe('relay protocol contract', () => { LIMIT_EXCEEDED: 4429, DRAINING: 4503 }) + // Shipped phones match this exact text; it is a wire value, not copy. + expect(RELAY_PHONE_CLOSE_REASON).toEqual({ HOST_ATTACH_TIMEOUT: 'attach-timeout' }) expect(RELAY_PROTOCOL_LIMITS).toMatchObject({ firstFrameDeadlineMs: 2_000, maxHttpBodyBytes: 4_096, diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml index d57e00bd7312..ba788c076b27 100644 --- a/cloud/pnpm-lock.yaml +++ b/cloud/pnpm-lock.yaml @@ -1,3 +1,161 @@ +--- +lockfileVersion: '9.0' + +importers: + + .: + configDependencies: {} + packageManagerDependencies: + pnpm: + specifier: 12.14.0 + version: 12.14.0 + +packages: + + '@pnpm/exe.android-arm64@12.14.0': + resolution: {integrity: sha512-+iSh7x21C3A2hqc+Z0ZmbKTcn6YNlzrXM142PsnQH/dLj0DYxFzoBvc6XAnL/yfQ5nnH2ebBiRj/zVxr61fZbQ==} + cpu: [arm64] + os: [android] + + '@pnpm/exe.android-x64@12.14.0': + resolution: {integrity: sha512-qT+0LqEr2wBVaxcG2T8cX/R8SSRz2Rvpp8nRR4fh8tnGUZRch87gFmsHUAoSQh7w6j1h0qTjrA384j4xi8frvw==} + cpu: [x64] + os: [android] + + '@pnpm/exe.darwin-arm64@12.14.0': + resolution: {integrity: sha512-tnrdUB8ecJoehuKKqi1YHCBVl61xtJ9XVGcx5EviC6fWO4DeGI8GtXrjvJxDBM3nuVjJeWistgsUp/N142x4sw==} + cpu: [arm64] + os: [darwin] + + '@pnpm/exe.darwin-x64@12.14.0': + resolution: {integrity: sha512-vlBfGkruqQRsEHTW43cPwpYd7QlbIJ//cvpAdb3bVXkI8SgTPoXhCnc05m+ZNiPrrEkBsX/t7KHw9aXpkzPm3Q==} + cpu: [x64] + os: [darwin] + + '@pnpm/exe.freebsd-x64@12.14.0': + resolution: {integrity: sha512-Qg5rxBrGYl6Tfso+EyygPf84fY1Fzy55E/0or+iuYKtnf0vc0MlHJfVIlT7pBukMRntQHRik7hUi0Z4s7AlJsw==} + cpu: [x64] + os: [freebsd] + + '@pnpm/exe.linux-arm64-musl@12.14.0': + resolution: {integrity: sha512-DRIe4yQkXBGA/PhVJdzlbnllxBdAtnMMvTVLLcGpsK5FvJz9kxD2Hyxtn68ovmNkVX06f6p45cQwfm3wx2jn1A==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@pnpm/exe.linux-arm64@12.14.0': + resolution: {integrity: sha512-7joGShZ44Nk3WpNU+60JoOPjCXlgxhJAq11S2UiVtwCUDgTer1yTPPfxGW8pxaD6046actYLIYDbZNlubHWs0A==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@pnpm/exe.linux-ppc64@12.14.0': + resolution: {integrity: sha512-gSoryn7ySdXd//gjgf3MvvA8ZerJayaSbuMZMtAWq4vb2/wlVEgpl0Kvqt54y58TRj533Qjq2txKgnxd4JyoAQ==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@pnpm/exe.linux-riscv64@12.14.0': + resolution: {integrity: sha512-EMjew1H9WqQ7pL+tYqCPwiuB+0PeCiJRV7fyWeuFw0wwYUajFBjcaPO3IGAVu9sdmJDU+e+HYv/Zey6aKae0Vw==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@pnpm/exe.linux-s390x@12.14.0': + resolution: {integrity: sha512-shYOj/SFoXD35fHe0pWr2JofwSXulC0BnEIXLj2QokDi0EF//AfKsui35yKIkTEl+s2VJybc3PO4iJNhKIj5Qw==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@pnpm/exe.linux-x64-musl@12.14.0': + resolution: {integrity: sha512-btZiZ2Ot0llKK1ndSfA5i3IRX7/S0VrmzseNH/BJ4c+Cady+S81H6vvU1zByggHMyqQhC8OgxO5PiIY7D9bH0Q==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@pnpm/exe.linux-x64@12.14.0': + resolution: {integrity: sha512-F6K8e/jWxiuEYJaQtRRUG9ynBSTEHa+t7Lf86MKuOO5UA7WVVMbi+oB8nY6iKW++ncWS2yG6T9faXZGxqNsfoQ==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@pnpm/exe.win32-arm64@12.14.0': + resolution: {integrity: sha512-DOjlkVB6TtTKeAAe0hNZXn5S7YqESmo1Z6SGbR4BZNQ4isRg31XVHjX2RFTbY8gnZq68IwzyLmPqlEjMmYxofg==} + cpu: [arm64] + os: [win32] + + '@pnpm/exe.win32-x64@12.14.0': + resolution: {integrity: sha512-/LSX6mew/sLf08XqcjemuklNKf1K/Ftpgj4//hK/HKzI2dKbmxDB27a/2OaQ5a0O3/3OFzlssXDziZsi+yVBwQ==} + cpu: [x64] + os: [win32] + + pnpm@12.14.0: + resolution: {integrity: sha512-wYum2HmT09Fky16JSPMrLAbuhUSH4GzY2GeBNC+z5btGNDZwNHEehoArqpw3vqkGzhkprQjXCITJfYAd8TOJpg==} + engines: {node: '>=18.*'} + hasBin: true + +snapshots: + + '@pnpm/exe.android-arm64@12.14.0': + optional: true + + '@pnpm/exe.android-x64@12.14.0': + optional: true + + '@pnpm/exe.darwin-arm64@12.14.0': + optional: true + + '@pnpm/exe.darwin-x64@12.14.0': + optional: true + + '@pnpm/exe.freebsd-x64@12.14.0': + optional: true + + '@pnpm/exe.linux-arm64-musl@12.14.0': + optional: true + + '@pnpm/exe.linux-arm64@12.14.0': + optional: true + + '@pnpm/exe.linux-ppc64@12.14.0': + optional: true + + '@pnpm/exe.linux-riscv64@12.14.0': + optional: true + + '@pnpm/exe.linux-s390x@12.14.0': + optional: true + + '@pnpm/exe.linux-x64-musl@12.14.0': + optional: true + + '@pnpm/exe.linux-x64@12.14.0': + optional: true + + '@pnpm/exe.win32-arm64@12.14.0': + optional: true + + '@pnpm/exe.win32-x64@12.14.0': + optional: true + + pnpm@12.14.0: + optionalDependencies: + '@pnpm/exe.android-arm64': 12.14.0 + '@pnpm/exe.android-x64': 12.14.0 + '@pnpm/exe.darwin-arm64': 12.14.0 + '@pnpm/exe.darwin-x64': 12.14.0 + '@pnpm/exe.freebsd-x64': 12.14.0 + '@pnpm/exe.linux-arm64': 12.14.0 + '@pnpm/exe.linux-arm64-musl': 12.14.0 + '@pnpm/exe.linux-ppc64': 12.14.0 + '@pnpm/exe.linux-riscv64': 12.14.0 + '@pnpm/exe.linux-s390x': 12.14.0 + '@pnpm/exe.linux-x64': 12.14.0 + '@pnpm/exe.linux-x64-musl': 12.14.0 + '@pnpm/exe.win32-arm64': 12.14.0 + '@pnpm/exe.win32-x64': 12.14.0 + +--- lockfileVersion: '9.0' settings: diff --git a/cloud/pnpm-workspace.yaml b/cloud/pnpm-workspace.yaml index 97bf7cd8c2ef..af76ff32b474 100644 --- a/cloud/pnpm-workspace.yaml +++ b/cloud/pnpm-workspace.yaml @@ -3,3 +3,8 @@ packages: - packages/* minimumReleaseAge: 4320 +engineStrict: false + +# pnpm 10 skipped this unlisted build script; keep the binary-only install it produced. +allowBuilds: + esbuild: false diff --git a/config/audit-tools/package.json b/config/audit-tools/package.json index 2d366a5484b0..52aae3d426fd 100644 --- a/config/audit-tools/package.json +++ b/config/audit-tools/package.json @@ -9,5 +9,5 @@ "engines": { "node": "24" }, - "packageManager": "pnpm@12.8.1+sha512.f64ba907507f5ceafe06c8d38e6052d0179444580ec1279ddd5bfc11cb48aa8a2644b66598e07e761da84872a9fc57d5f902b87fa49d024198d558612aabbe45" + "packageManager": "pnpm@12.14.0+sha512.c18ba6d87993d3d164cb5e8948f32b2c06ee854487e06cd8d86781342fb3e5bb4634367034711e86802baa9c37bea906ce1929ad08d70884c97d801df13389a6" } diff --git a/config/audit-tools/pnpm-lock.yaml b/config/audit-tools/pnpm-lock.yaml index c9f420dc8035..8ad8aa84360e 100644 --- a/config/audit-tools/pnpm-lock.yaml +++ b/config/audit-tools/pnpm-lock.yaml @@ -7,153 +7,153 @@ importers: configDependencies: {} packageManagerDependencies: pnpm: - specifier: 12.8.1 - version: 12.8.1 + specifier: 12.14.0 + version: 12.14.0 packages: - '@pnpm/exe.android-arm64@12.8.1': - resolution: {integrity: sha512-siDxcidfz5eM6L2tJfik+i2bbUEw8UC7MZ3Yl2HYdRmqbdhHGAOQACtOOu6QVW4Jul4/w5wVKm+wM8ACtgh1Pg==} + '@pnpm/exe.android-arm64@12.14.0': + resolution: {integrity: sha512-+iSh7x21C3A2hqc+Z0ZmbKTcn6YNlzrXM142PsnQH/dLj0DYxFzoBvc6XAnL/yfQ5nnH2ebBiRj/zVxr61fZbQ==} cpu: [arm64] os: [android] - '@pnpm/exe.android-x64@12.8.1': - resolution: {integrity: sha512-0F/uqPRc3aN4vLHsa0f+6fTtmsigFQ9/yxEU43fionF//pNHEd5e97b3w/nKIgqYMCUQCJphmBRMA9Q8/wlo9Q==} + '@pnpm/exe.android-x64@12.14.0': + resolution: {integrity: sha512-qT+0LqEr2wBVaxcG2T8cX/R8SSRz2Rvpp8nRR4fh8tnGUZRch87gFmsHUAoSQh7w6j1h0qTjrA384j4xi8frvw==} cpu: [x64] os: [android] - '@pnpm/exe.darwin-arm64@12.8.1': - resolution: {integrity: sha512-/rwavvMQJsl2RIxOHBnDF+4Gk+fhQFAY4M3PQZoKskJOEnaJcygG9p1xby6wcQcbS4d9dIubv09CQhufTgfmpw==} + '@pnpm/exe.darwin-arm64@12.14.0': + resolution: {integrity: sha512-tnrdUB8ecJoehuKKqi1YHCBVl61xtJ9XVGcx5EviC6fWO4DeGI8GtXrjvJxDBM3nuVjJeWistgsUp/N142x4sw==} cpu: [arm64] os: [darwin] - '@pnpm/exe.darwin-x64@12.8.1': - resolution: {integrity: sha512-htYt2gN7zqJKLhC99Ft6EUiO7myjZAZXfmGCTJUL1v+o7vE9hZdE1+fAkk4Oj1ZHNRti6b2woOxEkRkAusi8OQ==} + '@pnpm/exe.darwin-x64@12.14.0': + resolution: {integrity: sha512-vlBfGkruqQRsEHTW43cPwpYd7QlbIJ//cvpAdb3bVXkI8SgTPoXhCnc05m+ZNiPrrEkBsX/t7KHw9aXpkzPm3Q==} cpu: [x64] os: [darwin] - '@pnpm/exe.freebsd-x64@12.8.1': - resolution: {integrity: sha512-PB5BULNgytxvExoOXUUeseS/DcpPENs+cg3iA6xHHeaLT4ctEGKSGXEbuj7USeqvo8JBpAx039hivnbzGqDwnA==} + '@pnpm/exe.freebsd-x64@12.14.0': + resolution: {integrity: sha512-Qg5rxBrGYl6Tfso+EyygPf84fY1Fzy55E/0or+iuYKtnf0vc0MlHJfVIlT7pBukMRntQHRik7hUi0Z4s7AlJsw==} cpu: [x64] os: [freebsd] - '@pnpm/exe.linux-arm64-musl@12.8.1': - resolution: {integrity: sha512-kKeyC/ArjrgdciQRBrOL0j9HtOI4gDRgoyP+eJ1CQKyadpKlf/DhDDyspXe07R3B9KagTVLUF2nlt6Dr5quFIg==} + '@pnpm/exe.linux-arm64-musl@12.14.0': + resolution: {integrity: sha512-DRIe4yQkXBGA/PhVJdzlbnllxBdAtnMMvTVLLcGpsK5FvJz9kxD2Hyxtn68ovmNkVX06f6p45cQwfm3wx2jn1A==} cpu: [arm64] os: [linux] libc: [musl] - '@pnpm/exe.linux-arm64@12.8.1': - resolution: {integrity: sha512-q4s9a9X2O3N9aeUFooW24orNrxvu20+jyVUFR5RanPCqOKKPBrgs2iywMkBBx1aXkkzdWT54/mfz8Be8sJlWEw==} + '@pnpm/exe.linux-arm64@12.14.0': + resolution: {integrity: sha512-7joGShZ44Nk3WpNU+60JoOPjCXlgxhJAq11S2UiVtwCUDgTer1yTPPfxGW8pxaD6046actYLIYDbZNlubHWs0A==} cpu: [arm64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-ppc64@12.8.1': - resolution: {integrity: sha512-F7XSjKJthwCh/L6abZiMpAfo2Cm61SZIT3e4dfgiIjESndlWaXCnfLKngSgZ/SEMsdmHT6dud3+kWIB3tw0/xQ==} + '@pnpm/exe.linux-ppc64@12.14.0': + resolution: {integrity: sha512-gSoryn7ySdXd//gjgf3MvvA8ZerJayaSbuMZMtAWq4vb2/wlVEgpl0Kvqt54y58TRj533Qjq2txKgnxd4JyoAQ==} cpu: [ppc64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-riscv64@12.8.1': - resolution: {integrity: sha512-z1ecpIK/ZY08Unk69KWWE9867oaKUytGUjrHnBoGFAAhsK+q+0cfvBH+f5fWobJ7FX97leMlLO+gkJolddM45Q==} + '@pnpm/exe.linux-riscv64@12.14.0': + resolution: {integrity: sha512-EMjew1H9WqQ7pL+tYqCPwiuB+0PeCiJRV7fyWeuFw0wwYUajFBjcaPO3IGAVu9sdmJDU+e+HYv/Zey6aKae0Vw==} cpu: [riscv64] os: [linux] libc: [glibc] - '@pnpm/exe.linux-s390x@12.8.1': - resolution: {integrity: sha512-KU6tnrBfu4uKXTq6N5qJwSm8FSrs9Wc8b5AUEsHK9bRXwNdsZ1/lRxG1++X29SKRW9OSNuDSjxASv5mAvGo1gA==} + '@pnpm/exe.linux-s390x@12.14.0': + resolution: {integrity: sha512-shYOj/SFoXD35fHe0pWr2JofwSXulC0BnEIXLj2QokDi0EF//AfKsui35yKIkTEl+s2VJybc3PO4iJNhKIj5Qw==} cpu: [s390x] os: [linux] libc: [glibc] - '@pnpm/exe.linux-x64-musl@12.8.1': - resolution: {integrity: sha512-eUE8BWJkDr2tJb/yrk0yVnszlmNqFarVxUlqiZ9BAkct28Nq/5g2H0tvMDmPA7fAQW3STNj+omJPPRVT+4s5sQ==} + '@pnpm/exe.linux-x64-musl@12.14.0': + resolution: {integrity: sha512-btZiZ2Ot0llKK1ndSfA5i3IRX7/S0VrmzseNH/BJ4c+Cady+S81H6vvU1zByggHMyqQhC8OgxO5PiIY7D9bH0Q==} cpu: [x64] os: [linux] libc: [musl] - '@pnpm/exe.linux-x64@12.8.1': - resolution: {integrity: sha512-8bDZ0lZlCdi2rvnFul7EYgcC7zKeWSe76y8JV2oXobaS8p9i9d6PSf6LgLtTM0fI7R9Oh4eLCbveXyNDMmvZ+g==} + '@pnpm/exe.linux-x64@12.14.0': + resolution: {integrity: sha512-F6K8e/jWxiuEYJaQtRRUG9ynBSTEHa+t7Lf86MKuOO5UA7WVVMbi+oB8nY6iKW++ncWS2yG6T9faXZGxqNsfoQ==} cpu: [x64] os: [linux] libc: [glibc] - '@pnpm/exe.win32-arm64@12.8.1': - resolution: {integrity: sha512-MsT0dlrRxnCRCHCaosNbhWpEiAFnFuMEu3KuDJDU41BXgTwiINxU8goloUSUDejWZUqKaUqNC/qUTihl9nFtqg==} + '@pnpm/exe.win32-arm64@12.14.0': + resolution: {integrity: sha512-DOjlkVB6TtTKeAAe0hNZXn5S7YqESmo1Z6SGbR4BZNQ4isRg31XVHjX2RFTbY8gnZq68IwzyLmPqlEjMmYxofg==} cpu: [arm64] os: [win32] - '@pnpm/exe.win32-x64@12.8.1': - resolution: {integrity: sha512-SWtPe0PsbTTk//gJND10tMdOfCYjrkq5+qV49hzVhY7xz2iJ339Rc+xwASlcvbwTfrvH39YNqpKYibD+CRGLwA==} + '@pnpm/exe.win32-x64@12.14.0': + resolution: {integrity: sha512-/LSX6mew/sLf08XqcjemuklNKf1K/Ftpgj4//hK/HKzI2dKbmxDB27a/2OaQ5a0O3/3OFzlssXDziZsi+yVBwQ==} cpu: [x64] os: [win32] - pnpm@12.8.1: - resolution: {integrity: sha512-9kupB1B/XOr+BsjTjmBS0BeURFgOwSed3Vv8EctIqoomRLZlmOB+dh2oSHKp/FfV+QK4f6SdAkGY1VhhKqu+RQ==} + pnpm@12.14.0: + resolution: {integrity: sha512-wYum2HmT09Fky16JSPMrLAbuhUSH4GzY2GeBNC+z5btGNDZwNHEehoArqpw3vqkGzhkprQjXCITJfYAd8TOJpg==} engines: {node: '>=18.*'} hasBin: true snapshots: - '@pnpm/exe.android-arm64@12.8.1': + '@pnpm/exe.android-arm64@12.14.0': optional: true - '@pnpm/exe.android-x64@12.8.1': + '@pnpm/exe.android-x64@12.14.0': optional: true - '@pnpm/exe.darwin-arm64@12.8.1': + '@pnpm/exe.darwin-arm64@12.14.0': optional: true - '@pnpm/exe.darwin-x64@12.8.1': + '@pnpm/exe.darwin-x64@12.14.0': optional: true - '@pnpm/exe.freebsd-x64@12.8.1': + '@pnpm/exe.freebsd-x64@12.14.0': optional: true - '@pnpm/exe.linux-arm64-musl@12.8.1': + '@pnpm/exe.linux-arm64-musl@12.14.0': optional: true - '@pnpm/exe.linux-arm64@12.8.1': + '@pnpm/exe.linux-arm64@12.14.0': optional: true - '@pnpm/exe.linux-ppc64@12.8.1': + '@pnpm/exe.linux-ppc64@12.14.0': optional: true - '@pnpm/exe.linux-riscv64@12.8.1': + '@pnpm/exe.linux-riscv64@12.14.0': optional: true - '@pnpm/exe.linux-s390x@12.8.1': + '@pnpm/exe.linux-s390x@12.14.0': optional: true - '@pnpm/exe.linux-x64-musl@12.8.1': + '@pnpm/exe.linux-x64-musl@12.14.0': optional: true - '@pnpm/exe.linux-x64@12.8.1': + '@pnpm/exe.linux-x64@12.14.0': optional: true - '@pnpm/exe.win32-arm64@12.8.1': + '@pnpm/exe.win32-arm64@12.14.0': optional: true - '@pnpm/exe.win32-x64@12.8.1': + '@pnpm/exe.win32-x64@12.14.0': optional: true - pnpm@12.8.1: + pnpm@12.14.0: optionalDependencies: - '@pnpm/exe.android-arm64': 12.8.1 - '@pnpm/exe.android-x64': 12.8.1 - '@pnpm/exe.darwin-arm64': 12.8.1 - '@pnpm/exe.darwin-x64': 12.8.1 - '@pnpm/exe.freebsd-x64': 12.8.1 - '@pnpm/exe.linux-arm64': 12.8.1 - '@pnpm/exe.linux-arm64-musl': 12.8.1 - '@pnpm/exe.linux-ppc64': 12.8.1 - '@pnpm/exe.linux-riscv64': 12.8.1 - '@pnpm/exe.linux-s390x': 12.8.1 - '@pnpm/exe.linux-x64': 12.8.1 - '@pnpm/exe.linux-x64-musl': 12.8.1 - '@pnpm/exe.win32-arm64': 12.8.1 - '@pnpm/exe.win32-x64': 12.8.1 + '@pnpm/exe.android-arm64': 12.14.0 + '@pnpm/exe.android-x64': 12.14.0 + '@pnpm/exe.darwin-arm64': 12.14.0 + '@pnpm/exe.darwin-x64': 12.14.0 + '@pnpm/exe.freebsd-x64': 12.14.0 + '@pnpm/exe.linux-arm64': 12.14.0 + '@pnpm/exe.linux-arm64-musl': 12.14.0 + '@pnpm/exe.linux-ppc64': 12.14.0 + '@pnpm/exe.linux-riscv64': 12.14.0 + '@pnpm/exe.linux-s390x': 12.14.0 + '@pnpm/exe.linux-x64': 12.14.0 + '@pnpm/exe.linux-x64-musl': 12.14.0 + '@pnpm/exe.win32-arm64': 12.14.0 + '@pnpm/exe.win32-x64': 12.14.0 --- lockfileVersion: '9.0' diff --git a/config/build-plugins/markdown-parser-exports.ts b/config/build-plugins/markdown-parser-exports.ts index 2b468c0b02fe..40b545da1e4f 100644 --- a/config/build-plugins/markdown-parser-exports.ts +++ b/config/build-plugins/markdown-parser-exports.ts @@ -1,11 +1,16 @@ import { createRequire } from 'node:module' const require = createRequire(import.meta.url) -const parserRequire = createRequire(require.resolve('remark-parse')) +// Why: walk the declared dependency chain; isolated installs don't hoist decode-named-character-reference next to remark-parse. +const fromMarkdownRequire = createRequire( + createRequire(require.resolve('remark-parse')).resolve('mdast-util-from-markdown') +) const mathRequire = createRequire(require.resolve('rehype-katex')) // Both preview paths use DOM-free parsing so their dependencies also run in workers. export const markdownParserAliases = { - 'decode-named-character-reference': parserRequire.resolve('decode-named-character-reference'), + 'decode-named-character-reference': fromMarkdownRequire.resolve( + 'decode-named-character-reference' + ), 'hast-util-from-html-isomorphic': mathRequire.resolve('hast-util-from-html-isomorphic') } diff --git a/config/dark-build-switches.cjs b/config/dark-build-switches.cjs index 5b5cb5c02838..b46f40cb6864 100644 --- a/config/dark-build-switches.cjs +++ b/config/dark-build-switches.cjs @@ -2,8 +2,9 @@ const { existsSync } = require('node:fs') const { join } = require('node:path') // Dark build switches: a build compiles a switch's path in only when its variable is 'main'. Such a -// main build leaves the switch's marker in out/main, and packaging refuses it; packaging and the -// orcad template also refuse the variable itself. +// main build leaves the switch's marker in out/main, the phone page bundle (out/mobile-web), out/orcad +// and a --dark-test-template orcad template; packaging refuses any of them, and packaging and a +// release orcad template also refuse the variable itself. const DARK_BUILD_SWITCHES = [ // src/shared/layout-owner-switch.ts { variable: 'ORCA_LAYOUT_OWNER', off: 'window' }, @@ -37,7 +38,12 @@ function turnedOnDarkBuildDefines(env = process.env) { ) } -/** The marker each switched-on build leaves in out/main. */ +/** Every marker a switched-on build can leave, so a bundle check accounts for them. */ +function darkBuildMarkerFileNames() { + return DARK_BUILD_SWITCHES.map(markerFileName) +} + +/** The marker each switched-on build leaves in its output. */ function darkBuildMarkers(env = process.env) { return turnedOn(env).map((darkSwitch) => ({ fileName: markerFileName(darkSwitch), @@ -45,14 +51,8 @@ function darkBuildMarkers(env = process.env) { })) } -function assertNotDarkBuild(outputDirs, env = process.env) { - const [asked] = turnedOn(env) - if (asked) { - throw new Error( - `${asked.variable}=main is for dark test builds; unset it to package a release.` - ) - } - for (const darkSwitch of DARK_BUILD_SWITCHES) { +function assertUnmarked(outputDirs, switches) { + for (const darkSwitch of switches) { const marked = outputDirs.find((dir) => existsSync(join(dir, markerFileName(darkSwitch)))) if (marked) { throw new Error( @@ -62,9 +62,30 @@ function assertNotDarkBuild(outputDirs, env = process.env) { } } +function assertNotDarkBuild(outputDirs, env = process.env) { + const [asked] = turnedOn(env) + if (asked) { + throw new Error( + `${asked.variable}=main is for dark test builds; unset it to package a release.` + ) + } + assertUnmarked(outputDirs, DARK_BUILD_SWITCHES) +} + +/** For a build that may itself be dark: refuses only outputs marked by a switch this build has off. */ +function assertNoDarkOutputOfOffSwitches(outputDirs, env = process.env) { + const on = turnedOn(env) + assertUnmarked( + outputDirs, + DARK_BUILD_SWITCHES.filter((darkSwitch) => !on.includes(darkSwitch)) + ) +} + module.exports = { + assertNoDarkOutputOfOffSwitches, assertNotDarkBuild, darkBuildDefines, + darkBuildMarkerFileNames, darkBuildMarkers, turnedOnDarkBuildDefines } diff --git a/config/deprecated-runtime-owner-importers-baseline.txt b/config/deprecated-runtime-owner-importers-baseline.txt deleted file mode 100644 index a8b94e02abf3..000000000000 --- a/config/deprecated-runtime-owner-importers-baseline.txt +++ /dev/null @@ -1,61 +0,0 @@ -# Renderer files that import getRuntimeEnvironmentIdForWorktree / resolveRuntimeOwnerForWorktree. -# This is a RATCHET: the list may only SHRINK. Both resolvers read an ambiguous (two-host) -# worktree as "not a runtime", which callers treat as local. -# Prune after migrating a file: pnpm check:deprecated-runtime-owner-ratchet --prune -src/renderer/src/components/browser-pane/assemble-chrome/browser-workspace-pane.tsx -src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-lifecycle.ts -src/renderer/src/components/browser-pane/workspace-doc/doc-preview-document-actions.ts -src/renderer/src/components/browser-pane/workspace-doc/workspace-doc-page-pane.tsx -src/renderer/src/components/native-chat/native-chat-file-link.ts -src/renderer/src/components/native-chat/native-chat-http-link-source-owner.ts -src/renderer/src/components/native-chat/native-chat-runtime-owner.ts -src/renderer/src/components/right-sidebar/FileExplorer.tsx -src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-controller-state.tsx -src/renderer/src/components/right-sidebar/file-explorer-operation-owner.ts -src/renderer/src/components/right-sidebar/file-explorer-row-context-menu.tsx -src/renderer/src/components/right-sidebar/git-status-file-watch-refresh.ts -src/renderer/src/components/right-sidebar/source-control/listing/entry-context-menu.tsx -src/renderer/src/components/right-sidebar/use-file-explorer-tree-pane-state.ts -src/renderer/src/components/right-sidebar/useFileSearchRunner.ts -src/renderer/src/components/right-sidebar/useFileSearchScope.ts -src/renderer/src/components/tab-bar/use-tab-bar-runtime-model.ts -src/renderer/src/components/tab-bar/web-runtime-tab-move-mirror.ts -src/renderer/src/components/tab-group/useTabGroupActivationCommands.ts -src/renderer/src/components/tab-group/useTabGroupCreationCommands.ts -src/renderer/src/components/terminal-pane/terminal-input-host-platform.ts -src/renderer/src/components/terminal-pane/terminal-pane-menu-paste.ts -src/renderer/src/components/terminal-pane/terminal-pane-paste-execution.ts -src/renderer/src/components/terminal-pane/terminal-pane-paste-listeners.ts -src/renderer/src/components/terminal-workspace-model.ts -src/renderer/src/hooks/editor-external-watch-targets.ts -src/renderer/src/hooks/ipc-events/browser-state-ipc-bridge.ts -src/renderer/src/lib/activate-ai-vault-structured-session.ts -src/renderer/src/lib/active-agent-note-send.ts -src/renderer/src/lib/active-agent-note-target.ts -src/renderer/src/lib/agent-hibernation-coordinator.ts -src/renderer/src/lib/client-creation-action-policy.ts -src/renderer/src/lib/doc-preview-grants.ts -src/renderer/src/lib/execute-open-editor-path-move.ts -src/renderer/src/lib/execution-host-facts.ts -src/renderer/src/lib/file-preview.ts -src/renderer/src/lib/launch-agent-background-session.ts -src/renderer/src/lib/launch-agent-in-new-tab.ts -src/renderer/src/lib/launch-ai-vault-session.ts -src/renderer/src/lib/structured-agent-session-tab-activation.ts -src/renderer/src/lib/tab-number-shortcuts.ts -src/renderer/src/lib/terminal-tab-owner-activation.ts -src/renderer/src/lib/terminal-worktree-route.ts -src/renderer/src/lib/web-runtime-worktree-terminal-after-wake.ts -src/renderer/src/lib/workspace-tab-palette-activation.ts -src/renderer/src/lib/worktree-activation-pty-inventory.ts -src/renderer/src/lib/worktree-activation.ts -src/renderer/src/lib/worktree-creation-agent-seeds.ts -src/renderer/src/lib/worktree-row-open-in-owner.ts -src/renderer/src/lib/worktree-sort-order-host-split.ts -src/renderer/src/store/slices/browser/browser-page-conversion-actions.ts -src/renderer/src/store/slices/browser/browser-tab-actions.ts -src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts -src/renderer/src/store/slices/terminal-tab-retirement.ts -src/renderer/src/store/terminals/terminal-active-workspace-creation.ts -src/renderer/src/store/terminals/terminal-pane-hibernation.ts -src/renderer/src/store/terminals/terminal-shutdown.ts diff --git a/config/docker/headless-pairing/Dockerfile.build b/config/docker/headless-pairing/Dockerfile.build index 0300d8a82d49..01ca56f235d0 100644 --- a/config/docker/headless-pairing/Dockerfile.build +++ b/config/docker/headless-pairing/Dockerfile.build @@ -26,7 +26,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN corepack enable \ - && corepack prepare pnpm@12.8.1 --activate \ + && corepack prepare pnpm@12.14.0 --activate \ && pnpm --version WORKDIR /workspace diff --git a/config/docker/test-sandbox/Dockerfile b/config/docker/test-sandbox/Dockerfile index 7e31d4d4c123..8fe1109b0520 100644 --- a/config/docker/test-sandbox/Dockerfile +++ b/config/docker/test-sandbox/Dockerfile @@ -86,7 +86,11 @@ RUN useradd --create-home --shell /bin/bash runner \ USER runner WORKDIR /work +# Why: the image is a throwaway per-shard install, so it keeps a project-local node_modules as CI does. +ENV ORCA_PNPM_GLOBAL_VIRTUAL_STORE=0 + COPY --chown=runner:runner package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY --chown=runner:runner config/pnpmfile.cjs ./config/pnpmfile.cjs COPY --chown=runner:runner config/patches ./config/patches # Why: pnpm-workspace.yaml lists these packages, so install fails without their manifests. COPY --chown=runner:runner src/packages ./src/packages diff --git a/config/e2e-typecheck-baseline.txt b/config/e2e-typecheck-baseline.txt index c96721068268..e22e466ed325 100644 --- a/config/e2e-typecheck-baseline.txt +++ b/config/e2e-typecheck-baseline.txt @@ -27,8 +27,6 @@ 1 tests/e2e/cross-version-wire/structured-agent-session-host-fixture.ts 1 tests/e2e/cross-version-wire/submission-positions-downgrade.unit.test.ts 1 tests/e2e/cross-version-wire/versioned-agent-session-wire.ts -1 tests/e2e/daemon-generation-legacy-close-safety.spec.ts -1 tests/e2e/daemon-generation-reconnect-safety.spec.ts 1 tests/e2e/editable-context-paste-ownership.spec.ts 1 tests/e2e/file-explorer-linked-watch-refresh.spec.ts 5 tests/e2e/fixtures/daemon-generation-legacy-close-client.ts @@ -38,7 +36,6 @@ 1 tests/e2e/github-created-issue-start-prefill.spec.ts 1 tests/e2e/headless-paired-remote-terminal-reveal-grid.spec.ts 1 tests/e2e/headless-serve-desktop-activation.spec.ts -1 tests/e2e/helpers/daemon-generation-safety-fixtures.ts 5 tests/e2e/helpers/headless-paired-runtime-host.ts 1 tests/e2e/helpers/native-chat-view.ts 2 tests/e2e/helpers/nested-runtime-ssh-client-route.ts @@ -121,8 +118,6 @@ 2 tests/e2e/terminal-tui-wheel-reports.spec.ts 1 tests/e2e/terminal-warm-reattach-split-live-output.spec.ts 1 tests/e2e/workspace-space-git-status.spec.ts -1 tests/e2e/worktree-parent-reorder.spec.ts -1 tests/e2e/worktree-scroll-to-current.spec.ts 1 tests/tools/omp-child-history-rendered/fixture.tsx 1 tests/tools/omp-fresh-launch-windows-smoke.ts 1 tests/tools/omp-remote-sidebar-rendered/fixture.tsx diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs index 5835807ed418..7f37e4f91829 100644 --- a/config/electron-builder.config.cjs +++ b/config/electron-builder.config.cjs @@ -66,6 +66,7 @@ const { verifyAppImageNodeRuntimeServesPty } = require('./scripts/appimage-node-runtime.cjs') const { signWindowsUninstallerViaSignPath } = require('./scripts/windows-uninstaller-signing.cjs') +const { buildMacShipItScheduling } = require('./scripts/macos-shipit-scheduling.cjs') const isMacRelease = process.env.ORCA_MAC_RELEASE === '1' const isLinuxArm64Release = process.env.ORCA_LINUX_ARM64_RELEASE === '1' @@ -226,6 +227,8 @@ module.exports = { '!out/runtimes{,/**/*}', '!out/node-runtime-cache{,/**/*}', '!config{,/**/*}', + // Cloud services deploy separately; their sources are not desktop runtime resources. + '!cloud{,/**/*}', // Release archives and other build staging are not runtime resources. '!.build{,/**/*}', // Worktree-local Vite dependency caches are development inputs. @@ -262,6 +265,8 @@ module.exports = { '!out/**/*.map', // Why: Vite's manifest is only used to project the paired web client. '!out/renderer/.vite{,/**/*}', + // The installed browser serves its allowlisted assets from the unpacked renderer. + '!out/web{,/**/*}', // Why: out/electron-dev caches `pnpm dev`'s per-branch Electron.app copies (~270MB each). // CI never creates it, but packaging on a machine that has run dev would pack them all. '!out/electron-dev{,/**/*}', @@ -316,7 +321,7 @@ module.exports = { // entries reached solely from the Electron main process stay packed, since // asar redirects their app.asar paths. // Why: `orca serve` runs the orcad selection entry under ELECTRON_RUN_AS_NODE too (#27307). - // Why out/web and out/mobile-web: their assets are opened by path, and opening a packed file + // Why out/renderer and out/mobile-web: their assets are opened by path, and opening a packed file // reads a /tmp extract Electron makes once and never remakes, so a /tmp sweep broke them (#27976). asarUnpack: [ 'out/package.json', @@ -346,7 +351,7 @@ module.exports = { 'out/main/parcel-watcher-process-entry.js', 'out/main/orcad/orcad-local-serve-selection-entry.js', 'out/main/chunks/**', - 'out/web/**', + 'out/renderer/**', 'out/mobile-web/**', 'resources/**', 'node_modules/ws/**', @@ -364,9 +369,11 @@ module.exports = { beforePack: ( context, mobileWebBundleDir = MOBILE_WEB_BUNDLE_DIR, - ensureWaylandClipboard = ensureBundledWaylandClipboard + ensureWaylandClipboard = ensureBundledWaylandClipboard, + orcadTemplateDir = join(__dirname, '..', orcadTemplateExtraResource.from) ) => { - assertNotDarkBuild([join(__dirname, '..', 'out', 'main')]) + // The template is the server every SSH host gets, so a dark test template must never ship. + assertNotDarkBuild([join(__dirname, '..', 'out', 'main'), mobileWebBundleDir, orcadTemplateDir]) assertPackagedNativeVariantsInstalled(context.electronPlatformName, context.arch) assertBundledRipgrepInstalled() assertProcessHostOutputBuilt() @@ -399,6 +406,8 @@ module.exports = { context.packager.appInfo.productFilename ) const architecture = macSliceArch(context.arch, 'local-build compatibility') + const schedulingAddon = buildMacShipItScheduling(resourcesDir, architecture) + await signMacStandaloneHelper(schedulingAddon, 'ShipIt scheduling', context.packager) const version = context.packager.appInfo.version let commit = process.env.ORCA_BUILD_COMMIT || process.env.GITHUB_SHA || 'unknown' if (commit === 'unknown') { diff --git a/config/focus-setting-read-baseline.txt b/config/focus-setting-read-baseline.txt index a77bfd21391f..789b4b189019 100644 --- a/config/focus-setting-read-baseline.txt +++ b/config/focus-setting-read-baseline.txt @@ -4,7 +4,7 @@ # the default host, through defaultCreationHost. Everything else routes by the owner. # Prune after removing reads: pnpm check:focus-read-ratchet --prune 2 src/renderer/src/App.tsx -5 src/renderer/src/app-shell/use-app-shell-services.ts +3 src/renderer/src/app-shell/use-app-shell-services.ts 1 src/renderer/src/components/JiraIssueWorkspace.tsx 1 src/renderer/src/components/Landing.tsx 2 src/renderer/src/components/LinearIssueTextEditor.tsx @@ -24,23 +24,16 @@ 1 src/renderer/src/components/automations/use-automations-page-destination-form.ts 2 src/renderer/src/components/automations/use-automations-page-setup-state.ts 1 src/renderer/src/components/browser-pane/ReopenBrowserPageOnServerButton.tsx -1 src/renderer/src/components/browser-pane/assemble-chrome/browser-egress-indicator.tsx -1 src/renderer/src/components/browser-pane/assemble-chrome/browser-page-viewport-overlays.tsx 1 src/renderer/src/components/browser-pane/assemble-chrome/browser-workspace-pane.tsx -1 src/renderer/src/components/browser-pane/assemble-chrome/ssh-routed-browser-page-gate.tsx 1 src/renderer/src/components/browser-pane/browser-reopen-on-server.ts 1 src/renderer/src/components/browser-pane/stream-remote/remote-browser-page-pane.tsx 1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-lifecycle.ts 1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-page-stream.ts 1 src/renderer/src/components/browser-pane/stream-remote/use-remote-browser-stream-activation.ts -2 src/renderer/src/components/browser-pane/use-ssh-workspace-browser-route.ts -1 src/renderer/src/components/browser-pane/workspace-doc/HtmlDocPreview.tsx 1 src/renderer/src/components/browser-pane/workspace-doc/doc-preview-document-actions.ts 1 src/renderer/src/components/browser-pane/workspace-doc/workspace-doc-page-pane.tsx 2 src/renderer/src/components/cmd-j/quick-action-context.ts -2 src/renderer/src/components/dashboard/dashboard-card-terminal-input.ts -1 src/renderer/src/components/dashboard/launch-dashboard-agent.ts -1 src/renderer/src/components/dashboard/use-agent-row-conversation-name.ts +1 src/renderer/src/components/dashboard/dashboard-card-terminal-input.ts 2 src/renderer/src/components/editor/EditorContent.tsx 1 src/renderer/src/components/editor/EditorDiffFileSurface.tsx 1 src/renderer/src/components/editor/EditorEditFileSurface.tsx @@ -76,21 +69,12 @@ 1 src/renderer/src/components/linear-item-drawer-edit-section.tsx 2 src/renderer/src/components/native-chat-resume-modal.test-support.tsx 1 src/renderer/src/components/native-chat-resume-tree.test-support.ts -1 src/renderer/src/components/native-chat/NativeChatAgentMessageSenders.tsx 1 src/renderer/src/components/native-chat/NativeChatResolvedView.tsx 1 src/renderer/src/components/native-chat/NativeChatVisualFrame.tsx -1 src/renderer/src/components/native-chat/StructuredAgentSessionAttentionBridge.tsx -1 src/renderer/src/components/native-chat/StructuredAgentSessionPaneOverlayLayer.tsx -1 src/renderer/src/components/native-chat/StructuredAgentSessionStatusBridge.tsx -5 src/renderer/src/components/native-chat/native-chat-agent-message-sender-label.ts -2 src/renderer/src/components/native-chat/native-chat-attachment-upload.ts 1 src/renderer/src/components/native-chat/native-chat-file-link.ts 3 src/renderer/src/components/native-chat/native-chat-http-link-source-owner.ts 1 src/renderer/src/components/native-chat/native-chat-link-target-actions.ts 2 src/renderer/src/components/native-chat/native-chat-runtime-owner.ts -1 src/renderer/src/components/native-chat/native-chat-skill-discovery-context.ts -1 src/renderer/src/components/native-chat/use-native-chat-file-drops.ts -2 src/renderer/src/components/native-chat/use-native-chat-workspace-file-drop.ts 1 src/renderer/src/components/new-workspace/use-jira-source-connection.ts 4 src/renderer/src/components/new-workspace/use-linear-source-connection.ts 1 src/renderer/src/components/new-workspace/use-smart-workspace-name-field-controller.ts @@ -102,12 +86,9 @@ 2 src/renderer/src/components/right-sidebar/FileExplorer.tsx 1 src/renderer/src/components/right-sidebar/FileExplorerFilesTreePane.tsx 1 src/renderer/src/components/right-sidebar/FolderWorkspacePrChecksRow.tsx -1 src/renderer/src/components/right-sidebar/ai-vault-original-pane-actions.ts -1 src/renderer/src/components/right-sidebar/ai-vault-session-resume-in-chat-workspace.ts 3 src/renderer/src/components/right-sidebar/checks-panel-hosted-review-click-routing.ts 1 src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-controller-state.tsx 1 src/renderer/src/components/right-sidebar/checks-panel/use-checks-panel-create-review.tsx -1 src/renderer/src/components/right-sidebar/file-explorer-operation-owner.ts 1 src/renderer/src/components/right-sidebar/file-explorer-row-context-menu.tsx 1 src/renderer/src/components/right-sidebar/git-status-file-watch-refresh.ts 1 src/renderer/src/components/right-sidebar/source-control/listing/entry-context-menu.tsx @@ -116,9 +97,9 @@ 2 src/renderer/src/components/right-sidebar/use-file-explorer-tree-pane-state.ts 1 src/renderer/src/components/right-sidebar/useCreatePullRequestDialogFields.ts 4 src/renderer/src/components/right-sidebar/useFileSearchPanel.ts -2 src/renderer/src/components/right-sidebar/useFileSearchRunner.ts -2 src/renderer/src/components/right-sidebar/useFileSearchScope.ts -2 src/renderer/src/components/right-sidebar/useGitStatusPolling.ts +1 src/renderer/src/components/right-sidebar/useFileSearchRunner.ts +1 src/renderer/src/components/right-sidebar/useFileSearchScope.ts +1 src/renderer/src/components/right-sidebar/useGitStatusPolling.ts 1 src/renderer/src/components/settings/BaseRefPicker.tsx 2 src/renderer/src/components/settings/BrowserPane.tsx 1 src/renderer/src/components/settings/CliSkillRuntimeSetup.tsx @@ -162,9 +143,8 @@ 1 src/renderer/src/components/sidebar/SidebarTaskNavButton.tsx 1 src/renderer/src/components/sidebar/WorkspaceAttachmentSearch.tsx 1 src/renderer/src/components/sidebar/WorkspaceKanbanDrawer.tsx -1 src/renderer/src/components/sidebar/WorktreeCardPorts.tsx -2 src/renderer/src/components/sidebar/WorktreeList.tsx -1 src/renderer/src/components/sidebar/WorktreeRowOpenInSubMenu.tsx +1 src/renderer/src/components/sidebar/WorktreeList.tsx +1 src/renderer/src/components/sidebar/WorktreeRowOpenInSubMenu.tsx # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down 1 src/renderer/src/components/sidebar/WorktreeVisibilityDialog.tsx 2 src/renderer/src/components/sidebar/delete-worktree-flow.ts 1 src/renderer/src/components/sidebar/linear-agent-skill-runtime.ts @@ -191,15 +171,14 @@ 1 src/renderer/src/components/sidebar/worktree-filter-visibility.ts 1 src/renderer/src/components/sidebar/worktree-list/listing/use-filters.ts 1 src/renderer/src/components/sidebar/worktree-list/listing/use-section-rows.ts +1 src/renderer/src/components/sidebar/worktree-list/listing/use-window-list-model.ts 1 src/renderer/src/components/sidebar/worktree-list/rows/ProjectGroupDialogs.tsx 1 src/renderer/src/components/sidebar/worktree-visibility-host-target.ts 1 src/renderer/src/components/skills/SkillsPage.tsx 2 src/renderer/src/components/status-bar/ClaudeSwitcherMenu.tsx 1 src/renderer/src/components/status-bar/CodexSwitcherMenu.tsx -1 src/renderer/src/components/status-bar/PortsStatusSegment.tsx 1 src/renderer/src/components/status-bar/SshStatusSegment.tsx 1 src/renderer/src/components/status-bar/StatusBarSurface.tsx -1 src/renderer/src/components/status-bar/ports-status-popover-rows.tsx 2 src/renderer/src/components/status-bar/provider-account-sync-key.ts 1 src/renderer/src/components/status-bar/usage-provider-settings-target.ts 2 src/renderer/src/components/status-bar/use-codex-switcher-controller.ts @@ -207,11 +186,9 @@ 1 src/renderer/src/components/status-bar/use-workspace-space-git-refresh-action.ts 1 src/renderer/src/components/status-bar/use-workspace-space-manager-panel.ts 1 src/renderer/src/components/status-bar/workspace-space-decision-details.ts -1 src/renderer/src/components/tab-bar/QuickLaunchButton.tsx 1 src/renderer/src/components/tab-bar/TabBar.tsx -1 src/renderer/src/components/tab-bar/TabBarItemRow.tsx 3 src/renderer/src/components/tab-bar/tab-bar-windows-shell-launch-test-harness.ts -4 src/renderer/src/components/tab-bar/use-tab-bar-runtime-model.ts +3 src/renderer/src/components/tab-bar/use-tab-bar-runtime-model.ts 2 src/renderer/src/components/tab-bar/web-runtime-tab-move-mirror.ts 1 src/renderer/src/components/tab-bar/windows-shell-menu-visibility.ts 2 src/renderer/src/components/tab-group/useTabGroupActivationCommands.ts @@ -221,25 +198,20 @@ 9 src/renderer/src/components/task-page/TaskPage.tsx 2 src/renderer/src/components/task-page/jira/IssueAssigneeField.tsx 1 src/renderer/src/components/task-page/linear/Filters.tsx -1 src/renderer/src/components/terminal-pane/TerminalPaneSurface.tsx 1 src/renderer/src/components/terminal-pane/pending-split-close-test-fixture.ts 1 src/renderer/src/components/terminal-pane/pty-connection/agent-idle-working-handlers.ts -1 src/renderer/src/components/terminal-pane/pty-connection/interrupt-input-intent.ts -1 src/renderer/src/components/terminal-pane/pty-connection/pane-agent-identity.ts -1 src/renderer/src/components/terminal-pane/pty-connection/pane-host-capability-flags.ts 1 src/renderer/src/components/terminal-pane/retire-unbound-runtime-terminal-pane.ts -1 src/renderer/src/components/terminal-pane/terminal-ctrl-arrow-conpty.ts 1 src/renderer/src/components/terminal-pane/terminal-fit-restore.ts -2 src/renderer/src/components/terminal-pane/terminal-input-host-platform.ts +1 src/renderer/src/components/terminal-pane/terminal-input-host-platform.ts 1 src/renderer/src/components/terminal-pane/terminal-keyboard-hook.ts 1 src/renderer/src/components/terminal-pane/terminal-keyboard-runtime.ts 1 src/renderer/src/components/terminal-pane/terminal-link-open-hints.ts 2 src/renderer/src/components/terminal-pane/terminal-pane-close-admission.ts -1 src/renderer/src/components/terminal-pane/terminal-pane-manager-options.ts 2 src/renderer/src/components/terminal-pane/terminal-pane-menu-paste.ts 3 src/renderer/src/components/terminal-pane/terminal-pane-mount-preparation.ts 1 src/renderer/src/components/terminal-pane/terminal-pane-paste-execution.ts 1 src/renderer/src/components/terminal-pane/terminal-pane-paste-listeners.ts +1 src/renderer/src/components/terminal-pane/terminal-paste-runtime.ts # the paste owner's legacy-fallback read for raw PTY ids (round 3 F5); F8 nets it down 2 src/renderer/src/components/terminal-pane/use-terminal-pane-close-actions.ts 2 src/renderer/src/components/terminal-pane/use-terminal-pane-controller.ts 1 src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.ts @@ -285,9 +257,7 @@ 1 src/renderer/src/hooks/ipc-events/tab-lifecycle-ipc-bridge.ts 1 src/renderer/src/hooks/ipc-events/terminal-request-ipc-bridge.ts 2 src/renderer/src/hooks/use-active-skill-discovery-runtime-target.ts -1 src/renderer/src/hooks/use-terminal-quick-command-hosts.ts 1 src/renderer/src/hooks/useActiveProjectSkillRuntime.ts -2 src/renderer/src/hooks/useAgentDetectionTarget.ts 1 src/renderer/src/hooks/useEditorExternalWatch.ts 1 src/renderer/src/hooks/useInstalledAgentSkills.ts 1 src/renderer/src/hooks/useIpcEvents.ts @@ -296,29 +266,18 @@ 7 src/renderer/src/hooks/useSettingsNavigationMetadata.ts 1 src/renderer/src/hooks/useUnreadDockBadge.ts 2 src/renderer/src/hooks/useWindowsTerminalCapabilityOwnerKey.ts -4 src/renderer/src/lib/activate-ai-vault-structured-session.ts +3 src/renderer/src/lib/activate-ai-vault-structured-session.ts 1 src/renderer/src/lib/active-agent-note-send.ts 1 src/renderer/src/lib/active-agent-note-target.ts -1 src/renderer/src/lib/agent-account-restart-startup.ts 1 src/renderer/src/lib/agent-background-session-launch-host.ts -1 src/renderer/src/lib/agent-detection-target-inventory.ts 1 src/renderer/src/lib/agent-hibernation-coordinator-test-fixture.ts -1 src/renderer/src/lib/agent-hibernation-coordinator.ts -2 src/renderer/src/lib/agent-launch-route-input.ts -1 src/renderer/src/lib/agent-message-send.ts -1 src/renderer/src/lib/ai-vault-resume-platform.ts -1 src/renderer/src/lib/ai-vault-tab-title-requests.ts -2 src/renderer/src/lib/ai-vault-tab-title-sync-inputs.ts 1 src/renderer/src/lib/client-creation-action-error.ts 3 src/renderer/src/lib/client-creation-action-policy.ts 1 src/renderer/src/lib/default-creation-host.ts # the sanctioned reader of the default host -1 src/renderer/src/lib/doc-preview-grants.ts -1 src/renderer/src/lib/empty-workspace-default-agent-chat.ts -3 src/renderer/src/lib/execute-open-editor-path-move.ts -1 src/renderer/src/lib/execution-host-display-label.ts -2 src/renderer/src/lib/execution-host-facts.ts -6 src/renderer/src/lib/file-preview.ts -2 src/renderer/src/lib/folder-workspace-runtime-owner.ts +1 src/renderer/src/lib/doc-preview-grants.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down +1 src/renderer/src/lib/execute-open-editor-path-move.ts +1 src/renderer/src/lib/execution-host-facts.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down +5 src/renderer/src/lib/file-preview.ts 6 src/renderer/src/lib/http-link-destinations.ts 3 src/renderer/src/lib/http-link-routing.ts 1 src/renderer/src/lib/launch-agent-background-session.ts @@ -326,40 +285,32 @@ 2 src/renderer/src/lib/launch-agent-in-new-tab.ts 2 src/renderer/src/lib/launch-agent-web-host-tab.ts 2 src/renderer/src/lib/launch-ai-vault-session.ts -1 src/renderer/src/lib/launch-structured-agent-session.ts 1 src/renderer/src/lib/linear-issue-url-lookup.ts 1 src/renderer/src/lib/local-preflight-context.ts -3 src/renderer/src/lib/open-agent-message-sender.ts 2 src/renderer/src/lib/provider-runtime-context.ts -2 src/renderer/src/lib/remap-open-editor-tabs-for-path-change.ts 1 src/renderer/src/lib/remote-workspace-drop-test-fixtures.ts 2 src/renderer/src/lib/repo-runtime-owner.ts 1 src/renderer/src/lib/runtime-session-mirror-owners.ts 1 src/renderer/src/lib/runtime-session-mirror-targets.ts -1 src/renderer/src/lib/runtime-workspace-file-route.ts 1 src/renderer/src/lib/single-runtime-legacy-owner.ts -1 src/renderer/src/lib/sleeping-agent-session-launch.ts -1 src/renderer/src/lib/structured-agent-session-launch.ts -1 src/renderer/src/lib/structured-agent-session-provisional-tab.ts -4 src/renderer/src/lib/structured-agent-session-tab-activation.ts +1 src/renderer/src/lib/structured-agent-session-tab-activation.ts 1 src/renderer/src/lib/tab-number-shortcuts.ts 1 src/renderer/src/lib/terminal-close-route.ts 1 src/renderer/src/lib/terminal-tab-owner-activation.ts -5 src/renderer/src/lib/terminal-worktree-route.ts +6 src/renderer/src/lib/terminal-worktree-route.ts # resolveTerminalPtyOwner: teardown, close and PTY identity read spawn's attribution (round 3 F8), which retired the deprecated resolver's readers 3 src/renderer/src/lib/unread-badge-count-selector.ts 2 src/renderer/src/lib/web-runtime-worktree-terminal-after-wake.ts -5 src/renderer/src/lib/workspace-browser-tab-open.ts -1 src/renderer/src/lib/workspace-file-drag-source.ts +4 src/renderer/src/lib/workspace-browser-tab-open.ts +1 src/renderer/src/lib/workspace-file-browser-open-target.ts # moved out of file-preview.ts (round 3 F7), same read 1 src/renderer/src/lib/workspace-tab-palette-activation.ts -1 src/renderer/src/lib/worktree-activation-pty-inventory.ts -3 src/renderer/src/lib/worktree-activation.ts +1 src/renderer/src/lib/worktree-activation-pty-inventory.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down +2 src/renderer/src/lib/worktree-activation.ts 1 src/renderer/src/lib/worktree-creation-agent-seeds.ts 1 src/renderer/src/lib/worktree-jump-navigation.ts -2 src/renderer/src/lib/worktree-operation-route.ts -1 src/renderer/src/lib/worktree-row-open-in-owner.ts -6 src/renderer/src/lib/worktree-runtime-owner.ts # typed owner resolver plus its null-collapsing wrapper; callers net -3 +1 src/renderer/src/lib/worktree-row-open-in-owner.ts # reads focus through resolveOwnerWithLegacyFallback, newly counted (round 3 F2); F8 nets it down +3 src/renderer/src/lib/worktree-runtime-owner.ts # typed owner resolver and the legacy-fallback adapter the folds moved to (round 3 F0); F8 deleted the deprecated pair 1 src/renderer/src/lib/worktree-sort-order-host-split.ts -1 src/renderer/src/runtime/local-structured-session-tabs-sync/snapshot-apply.ts +1 src/renderer/src/lib/worktree-sort-order-persistence.ts # its split helper calls resolveOwnerWithLegacyFallback directly (round 3 F4); F8 nets it down 17 src/renderer/src/runtime/runtime-jira-client.ts 2 src/renderer/src/runtime/runtime-jira-summary-client.ts 1 src/renderer/src/runtime/runtime-jira-target.ts @@ -367,10 +318,7 @@ 8 src/renderer/src/runtime/runtime-linear-client.ts 6 src/renderer/src/runtime/runtime-linear-issue-mutations.ts 12 src/renderer/src/runtime/runtime-linear-project-client.ts -3 src/renderer/src/runtime/structured-agent-session-owner.ts -1 src/renderer/src/runtime/structured-conversation-name.ts 5 src/renderer/src/runtime/use-runtime-session-mirror-environment-key.ts -1 src/renderer/src/runtime/use-worktree-runtime-target.ts 2 src/renderer/src/runtime/web-runtime-browser-creation.ts 1 src/renderer/src/runtime/web-runtime-initial-terminal-bootstrap-dispatch.ts 1 src/renderer/src/runtime/web-runtime-session-tab-lifecycle.ts @@ -382,33 +330,27 @@ 2 src/renderer/src/runtime/web-session-tabs-sync/global-session-subscription.ts 1 src/renderer/src/runtime/web-session-tabs-sync/tracking-decisions.ts 3 src/renderer/src/runtime/web-session-tabs-sync/use-web-session-tabs-sync.ts -1 src/renderer/src/store/folder-workspaces/folder-workspace-mutations.ts 1 src/renderer/src/store/github/repository-routing.ts 2 src/renderer/src/store/github/visible-hosted-review-refresh-targets.ts 2 src/renderer/src/store/github/work-item-aggregate-actions.ts 3 src/renderer/src/store/github/work-item-fetch-actions.ts 2 src/renderer/src/store/github/work-item-routing.ts 9 src/renderer/src/store/index.ts -3 src/renderer/src/store/project-groups/project-group-mutations.ts +2 src/renderer/src/store/project-groups/project-group-mutations.ts 1 src/renderer/src/store/repos/all-host-repo-catalog-actions.ts -1 src/renderer/src/store/repos/converted-ssh-browser-pages.ts 2 src/renderer/src/store/repos/repo-add-actions.ts 1 src/renderer/src/store/repos/repo-update.ts 1 src/renderer/src/store/repos/runtime-repo-catalog-actions.ts 1 src/renderer/src/store/slices/browser-slice-test-harness.ts -1 src/renderer/src/store/slices/browser/browser-close-actions.ts 16 src/renderer/src/store/slices/browser/browser-cookie-import-actions.ts 1 src/renderer/src/store/slices/browser/browser-host-actions.ts -10 src/renderer/src/store/slices/browser/browser-host-state.ts -2 src/renderer/src/store/slices/browser/browser-page-conversion-actions.ts -1 src/renderer/src/store/slices/browser/browser-page-focus-actions.ts +7 src/renderer/src/store/slices/browser/browser-host-state.ts +1 src/renderer/src/store/slices/browser/browser-page-conversion-actions.ts 14 src/renderer/src/store/slices/browser/browser-profile-import-actions.ts 14 src/renderer/src/store/slices/browser/browser-profile-list-actions.ts -7 src/renderer/src/store/slices/browser/browser-tab-actions.ts -1 src/renderer/src/store/slices/browser/browser-tab-focus-actions.ts +6 src/renderer/src/store/slices/browser/browser-tab-actions.ts 1 src/renderer/src/store/slices/browser/create-browser-slice.ts 1 src/renderer/src/store/slices/browser/paired-browser-tab-creator.ts -3 src/renderer/src/store/slices/diff-comment-persistence.ts 2 src/renderer/src/store/slices/editor-slice-test-harness.ts 1 src/renderer/src/store/slices/editor/actions/markdown-link-action.ts 1 src/renderer/src/store/slices/editor/actions/markdown-preview-actions.ts @@ -441,14 +383,13 @@ 2 src/renderer/src/store/slices/preflight.ts 1 src/renderer/src/store/slices/repo-host-identity.ts 1 src/renderer/src/store/slices/repo-reorder-host-split.ts -4 src/renderer/src/store/slices/repos.ts +3 src/renderer/src/store/slices/repos.ts 1 src/renderer/src/store/slices/runtime-status.ts 2 src/renderer/src/store/slices/settings-owner-hydration-publication.ts 5 src/renderer/src/store/slices/settings.ts 8 src/renderer/src/store/slices/store-test-helpers.ts -1 src/renderer/src/store/slices/tabs/tabs-close-actions.ts 2 src/renderer/src/store/slices/tabs/tabs-host-mirroring.ts -2 src/renderer/src/store/slices/terminal-tab-retirement.ts +1 src/renderer/src/store/slices/terminal-tab-retirement.ts 3 src/renderer/src/store/slices/workspace-attention/workspace-unread-characterization-fixture.ts 1 src/renderer/src/store/slices/worktree-visibility-owner-settings.ts 3 src/renderer/src/store/slices/worktree-visibility-settings-write.ts @@ -462,12 +403,8 @@ 2 src/renderer/src/store/slices/worktrees/metadata/update-worktree-meta.ts 1 src/renderer/src/store/slices/worktrees/metadata/worktree-lineage-actions.ts 3 src/renderer/src/store/slices/worktrees/metadata/worktree-lineage-host-merge.ts -1 src/renderer/src/store/slices/worktrees/metadata/worktree-lineage-refresh.ts 3 src/renderer/src/store/slices/worktrees/session/worktree-terminal-reference.ts -1 src/renderer/src/store/slices/worktrees/teardown/purged-structured-launch-retirement.ts 2 src/renderer/src/store/terminals/terminal-active-workspace-creation.ts -1 src/renderer/src/store/terminals/terminal-pane-hibernation.ts -1 src/renderer/src/store/terminals/terminal-shutdown.ts 1 src/renderer/src/store/terminals/terminal-tab-appearance.ts 1 src/renderer/src/store/terminals/terminal-tab-close-providers.ts 1 src/renderer/src/web/preload-api/web-app-api.ts diff --git a/config/fork-ownership.json b/config/fork-ownership.json index fb20e8387de8..24ee0895da10 100644 --- a/config/fork-ownership.json +++ b/config/fork-ownership.json @@ -585,7 +585,7 @@ "kind": "registration", "lines": [ "import { createHeimdallFleetSlice } from './slices/fork-heimdall/fleet'", - " ...createHeimdallFleetSlice(...a)" + " ...createHeimdallFleetSlice(...a)" ] }, { @@ -641,6 +641,14 @@ " wireHeimdallFleetWindows(runtime, () => state.mainWindow)" ] }, + { + "path": "src/main/startup/main-process-ready-duplicate-pty-repair.test.ts", + "feature": "heimdall", + "kind": "registration", + "lines": [ + " ipcMain: { handle: vi.fn(), removeHandler: vi.fn(), on: vi.fn(), removeListener: vi.fn() }," + ] + }, { "path": "src/main/startup/main-process-state.ts", "feature": "heimdall", @@ -1367,7 +1375,7 @@ ] }, { - "path": "src/renderer/src/components/native-chat/native-chat-picker-items.ts", + "path": "src/shared/native-chat-picker-items.ts", "feature": "skill-plugin-attribution", "kind": "registration", "lines": [" pluginName?: string", " namespacePluginSkills = false"] @@ -1519,8 +1527,8 @@ "feature": "worktree-groups", "kind": "passthrough", "lines": [ - " groupMembershipDrag: ReturnType", - " trackWorktreeGroupMembershipDragFrame: groupMembershipDrag.trackPointerDragFrame," + " groupMembershipDrag?: ReturnType", + " trackWorktreeGroupMembershipDragFrame: groupMembershipDrag?.trackPointerDragFrame," ] }, { @@ -1547,7 +1555,7 @@ "kind": "passthrough", "lines": [ " worktreeGroupMembershipDragPreview: WorktreeGroupMembershipDragPreview", - " 'group relative flex h-6 w-full items-center gap-1.5 pr-2 text-left transition-all',", + " 'group relative flex h-6 w-full items-center gap-1.5 pr-2 text-left transition-all',", " sectionSeparatorClass?: string | false", " args.sectionSeparatorClass," ] @@ -2509,7 +2517,7 @@ "kind": "registration", "lines": [ "import { ForkAiVaultLineageLine } from '@/components/agent-session-continuation/fork-session-handoff/ForkAiVaultLineageLine'", - " " + " " ] }, { @@ -2923,13 +2931,10 @@ "kind": "registration", "lines": [ "import { app, ipcMain } from 'electron'", - "import { forwardAskEventsToRenderer } from '../fork-ask-question-tool/ask-ipc-forward'", + "import { forwardRuntimeAskEventsToRenderer } from '../fork-ask-question-tool/ask-ipc-forward'", "import { startClaudeSuppressionVerdictPublisher } from '../fork-ask-question-tool/claude-suppression-verdict-publisher'", "import { getDashboardPopoutWindow } from '../window/dashboard-popout-window'", - " forwardAskEventsToRenderer(", - " runtime.getAskServices().registry,", - " () => state.mainWindow,", - " getDashboardPopoutWindow", + " forwardRuntimeAskEventsToRenderer(runtime, () => state.mainWindow, getDashboardPopoutWindow)", " startClaudeSuppressionVerdictPublisher({", " store,", " ipcMain,", @@ -2975,7 +2980,7 @@ "kind": "registration", "lines": [ "import { createAsksSlice } from './slices/fork-ask-question-tool/asks'", - " ...createAsksSlice(...a)," + " ...createAsksSlice(...a)," ] }, { @@ -3257,6 +3262,27 @@ "import { WorkspaceActivityHiddenCountRow } from './fork-workspace-activity-window/WorkspaceActivityHiddenCountRow'" ] }, + { + "path": "src/renderer/src/components/sidebar/worktree-list/listing/sidebar-list-model.ts", + "feature": "workspace-activity-window", + "kind": "passthrough", + "lines": [" activityHiddenCount?: number"] + }, + { + "path": "src/renderer/src/components/sidebar/worktree-list/listing/use-main-list-model.ts", + "feature": "workspace-activity-window", + "kind": "passthrough", + "lines": [" activityHiddenFolderCount: 0"] + }, + { + "path": "src/renderer/src/components/sidebar/worktree-list/listing/use-window-list-model.ts", + "feature": "workspace-activity-window", + "kind": "passthrough", + "lines": [ + " const { visibleWorktrees, activityHiddenCount, pairedDeviceIdsByEnvironment } =", + " return { visibleWorktrees, visibleScope, externalWorktreeCards, rowModel, activityHiddenCount }" + ] + }, { "path": "src/renderer/src/components/sidebar/sidebar-filter-actions.ts", "feature": "workspace-activity-window", @@ -3560,7 +3586,7 @@ ] }, { - "path": "src/renderer/src/components/tab-bar/SortableTab.tsx", + "path": "src/renderer/src/components/tab-bar/use-terminal-tab-activity.ts", "feature": "ask-question-tool", "kind": "registration", "lines": [ @@ -4109,21 +4135,13 @@ ] }, { - "path": "src/main/runtime/orca-runtime.ts", + "path": "src/main/runtime/orca-runtime-agent-transitions.ts", "feature": "project-ledger", "kind": "import-swap", "lines": [ "import { OrcaRuntimeWithLedger as OrcaRuntimeWithMigrationCatalog } from './orca-runtime-ledger'" ] }, - { - "path": "src/main/runtime/sidebar-inputs/orca-runtime-workspace-activity-facts.ts", - "feature": "project-ledger", - "kind": "import-swap", - "lines": [ - "import { OrcaRuntimeWithLedger as OrcaRuntimeWithMigrationCatalog } from '../orca-runtime-ledger'" - ] - }, { "path": "src/renderer/src/components/sidebar/worktree-list/viewport/viewport-props.ts", "feature": "project-ledger", @@ -5571,7 +5589,7 @@ "kind": "registration", "lines": [ "import { createLoopsSlice } from '@/fork-loops/loops-slice'", - " ...createLoopsSlice(...a)," + " ...createLoopsSlice(...a)," ] }, { @@ -5888,11 +5906,26 @@ "reason": "Tests for the fork's release-tooling delta above.", "status": "permanent" }, + { + "path": "config/scripts/mac-notarization-builder-contract.test.mjs", + "status": "permanent", + "reason": "Asserts the macOS release channel runs on Blacksmith through the dual-arch notarization-overlap builder; the fork's release channel builds arm64 only on a hosted runner, so its release case asserts that shape instead." + }, + { + "path": "config/scripts/ci-managed-ssh-execution.test.mjs", + "status": "permanent", + "reason": "Contracts for the fork-owned PR job graph: this fork's PR workflow has no e2e caller, so the shard case feeds the lane the selector's output directly and asserts the caller's absence." + }, { "path": "config/scripts/release-cut-token-permissions.test.mjs", "status": "permanent", "reason": "Asserts upstream's release-cut job matrix and github.repository == 'stablyai/orca'; the fork publishes macOS-only arm64 artifacts from zpyoung/orca, so upstream's version can never describe this pipeline." }, + { + "path": "config/scripts/release-windows-signing-gate.test.mjs", + "reason": "Upstream's Windows signing-gate contract, kept against the fork's release-cut.yml: the fork's Windows/Linux build job is disabled, so it asserts if: false where upstream asserts the build-mac success condition.", + "status": "permanent" + }, { "path": "config/scripts/release-rc-history.mjs", "reason": "Fork release-tooling delta for the fork's own RC history.", @@ -6607,6 +6640,11 @@ "reason": "Upstream's paste-remount coverage, run against the fork composer field and actions (textarea, not upstream's rich-text editor).", "status": "permanent" }, + { + "path": "src/renderer/src/components/native-chat/NativeChatComposer.resume-command.test.tsx", + "reason": "Upstream's /resume coverage, run against the fork composer field, keydown and draft hooks (upstream's mocks target components the fork composer does not render).", + "status": "permanent" + }, { "path": "src/renderer/src/components/native-chat/native-chat-composer-prompt-recall.test.tsx", "reason": "Exercises upstream's transcript prompt recall (useNativeChatComposerRecall), which the fork composer replaces with its own per-pane history; the fork's equivalent coverage is in fork-agent-composer.", @@ -6701,6 +6739,10 @@ } ], "residuals": { + "src/main/startup/main-process-ready-duplicate-pty-repair.test.ts": { + "added": 1, + "removed": 0 + }, "tests/e2e/native-chat-ssh-delivery.unit.test.ts": { "added": 10, "removed": 9 @@ -6894,7 +6936,7 @@ "removed": 0 }, "src/main/startup/main-process-runtime-service.ts": { - "added": 21, + "added": 17, "removed": 2 }, "src/main/startup/main-process-state.ts": { @@ -7194,7 +7236,7 @@ "removed": 1 }, "src/renderer/src/components/native-chat/native-chat-composer-state.test.ts": { - "added": 144, + "added": 157, "removed": 1 }, "src/renderer/src/components/native-chat/native-chat-composer-state.ts": { @@ -7215,15 +7257,15 @@ }, "src/renderer/src/components/native-chat/native-chat-composer-types.ts": { "added": 14, - "removed": 40 + "removed": 37 }, "src/renderer/src/components/native-chat/native-chat-pending.ts": { "added": 1, "removed": 1 }, - "src/renderer/src/components/native-chat/native-chat-picker-items.ts": { - "added": 65, - "removed": 17 + "src/shared/native-chat-picker-items.ts": { + "added": 76, + "removed": 21 }, "src/renderer/src/components/native-chat/native-chat-pty-session-options.ts": { "added": 25, @@ -7626,8 +7668,8 @@ "removed": 1 }, "src/shared/native-chat-agent-profiles.ts": { - "added": 16, - "removed": 2 + "added": 11, + "removed": 1 }, "src/shared/native-chat-session-option-state.test.ts": { "added": 65, @@ -7882,8 +7924,8 @@ "removed": 0 }, "src/main/runtime/orca-runtime.ts": { - "added": 6, - "removed": 1 + "added": 5, + "removed": 0 }, "src/main/runtime/rpc/methods/terminal/terminal-multiplex-cleanup.ts": { "added": 3, @@ -7982,9 +8024,21 @@ "removed": 13 }, "src/renderer/src/components/sidebar/WorktreeList.tsx": { - "added": 46, + "added": 49, "removed": 5 }, + "src/renderer/src/components/sidebar/worktree-list/listing/sidebar-list-model.ts": { + "added": 1, + "removed": 0 + }, + "src/renderer/src/components/sidebar/worktree-list/listing/use-main-list-model.ts": { + "added": 2, + "removed": 1 + }, + "src/renderer/src/components/sidebar/worktree-list/listing/use-window-list-model.ts": { + "added": 11, + "removed": 10 + }, "src/renderer/src/components/sidebar/sidebar-filter-actions.ts": { "added": 17, "removed": 3 @@ -8093,7 +8147,7 @@ "added": 3, "removed": 1 }, - "src/renderer/src/components/tab-bar/SortableTab.tsx": { + "src/renderer/src/components/tab-bar/use-terminal-tab-activity.ts": { "added": 2, "removed": 0 }, @@ -8114,16 +8168,16 @@ "removed": 2 }, "src/main/ipc/repos.ts": { - "added": 2, - "removed": 2 + "added": 1, + "removed": 1 }, "src/main/ipc/repos/project-group-handlers.ts": { "added": 23, "removed": 5 }, "src/main/ipc/repos/repo-catalog-handlers.ts": { - "added": 24, - "removed": 4 + "added": 15, + "removed": 2 }, "src/main/ipc/repos/repo-ipc-arg-schemas.ts": { "added": 6, @@ -8174,7 +8228,7 @@ "removed": 0 }, "src/main/runtime/runtime-project-group-controller.ts": { - "added": 64, + "added": 62, "removed": 3 }, "src/main/runtime/runtime-repository-settings-controller.ts": { @@ -8270,8 +8324,8 @@ "removed": 2 }, "src/renderer/src/store/repos/repo-state.ts": { - "added": 15, - "removed": 2 + "added": 11, + "removed": 1 }, "src/renderer/src/store/right-sidebar-route.test.ts": { "added": 21, @@ -8337,7 +8391,7 @@ "added": 20, "removed": 0 }, - "src/main/runtime/sidebar-inputs/orca-runtime-workspace-activity-facts.ts": { + "src/main/runtime/orca-runtime-agent-transitions.ts": { "added": 1, "removed": 1 }, @@ -8626,11 +8680,11 @@ "removed": 0 }, "src/renderer/src/lib/launch-agent-background-session.ts": { - "added": 38, - "removed": 22 + "added": 30, + "removed": 18 }, "src/renderer/src/lib/launch-agent-in-new-tab-web-runtime.test.ts": { - "added": 27, + "added": 28, "removed": 0 }, "src/renderer/src/lib/launch-agent-in-new-tab.ts": { diff --git a/config/id-only-store-write-baseline.txt b/config/id-only-store-write-baseline.txt new file mode 100644 index 000000000000..00835d310956 --- /dev/null +++ b/config/id-only-store-write-baseline.txt @@ -0,0 +1,9 @@ +# Store writes whose first parameter is a bare repo or worktree id (R4-2), as Class.method, +# and persistence repo-row lookups by id alone (review-5 #10), as lookup:file#function. +# This is a RATCHET: entries may only be REMOVED. Take RepoHostTarget or HostWorktreeTarget, +# or match the row by id AND host (getRepoExecutionHostId). +# Prune after a fix: pnpm check:id-only-store-write-ratchet --prune +lookup:src/main/persistence/loading-store/terminal-binding-recovery.ts#getConnectionIdForWorktree +lookup:src/main/persistence/scheduling-automations/automation-definition-operations.ts#createAutomation +lookup:src/main/persistence/scheduling-automations/automation-definition-operations.ts#updateAutomation +lookup:src/main/persistence/tracking-repos/project-host-operations.ts#getRepo diff --git a/config/legacy-relay-compat-tags-baseline.txt b/config/legacy-relay-compat-tags-baseline.txt index 31905b926410..16a3bfc3fee8 100644 --- a/config/legacy-relay-compat-tags-baseline.txt +++ b/config/legacy-relay-compat-tags-baseline.txt @@ -20,7 +20,7 @@ src/main/ssh/orcad-migration-terminal-gate.ts src/main/managed-server/lifecycle/orcad-runtime-conversion.test.ts src/main/ssh/ssh-host-server-on-connect.test.ts src/main/ssh/ssh-host-server-on-connect.ts -src/main/ssh/ssh-target-orcad-preflight.ts +src/main/ssh/targets/ssh-target-orcad-preflight.ts src/preload/api-types.ts src/preload/api/legacy-relay-bridge-api.ts src/renderer/src/components/terminal-pane/legacy-relay-bridge-panes.ts diff --git a/config/main-layout-writer-baseline.txt b/config/main-layout-writer-baseline.txt index c7b9453dfcab..4166afdb39de 100644 --- a/config/main-layout-writer-baseline.txt +++ b/config/main-layout-writer-baseline.txt @@ -71,5 +71,6 @@ 4 src/main/runtime/workspace-session-worktree-id.ts 1 src/main/runtime/workspaces/session-hosts/get-runtime-id.ts 2 src/main/runtime/workspaces/workspace-file-session-targets.ts -1 src/main/ssh/ssh-target-id-migration.ts +1 src/main/ssh/orcad-migration-source-retention.ts +1 src/main/ssh/targets/ssh-target-id-migration.ts 2 src/main/terminal-scrollback-snapshots.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 5e8ac38c8903..10cb9ef36594 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -4,5 +4,5 @@ # Regenerate/prune: pnpm check:max-lines-ratchet --prune (removes stale entries only) inline src/main/ipc/worktree-remote.ts inline src/main/ssh/ssh-channel-multiplexer.ts -inline src/main/ssh/ssh-connection.ts +inline src/main/ssh/connection/ssh-connection.ts inline src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts diff --git a/config/packaged-native-build-entries.cjs b/config/packaged-native-build-entries.cjs new file mode 100644 index 000000000000..f0c8301e35c4 --- /dev/null +++ b/config/packaged-native-build-entries.cjs @@ -0,0 +1,40 @@ +const { existsSync, readdirSync, renameSync, rmSync } = require('node:fs') +const { join } = require('node:path') +const { + NATIVE_BUILD_PACKAGES, + isNativeBuildEntryName, + nativeBuildEntryName +} = require('./scripts/native-build-entry.cjs') + +/** + * Ship a dev install's per-ABI native build as build/Release. + * + * A package dir shared across worktrees (pnpm's global virtual store) holds Orca's source build + * in build/ rather than build/Release; see scripts/native-build-entry.cjs. Packaged + * consumers (daemon relocation, the macOS terminal host, the job-ownership gate) all read + * build/Release, so the packaged copy gets exactly the layout a CI install produces. + */ +function adoptPackagedNativeBuildEntries(resourcesDir, electronPlatformName, architecture) { + for (const moduleName of Object.keys(NATIVE_BUILD_PACKAGES)) { + const buildDir = join(resourcesDir, 'node_modules', ...moduleName.split('/'), 'build') + if (!existsSync(buildDir)) { + continue + } + const entryDir = join( + buildDir, + nativeBuildEntryName(moduleName, { platform: electronPlatformName, arch: architecture }) + ) + if (existsSync(entryDir)) { + rmSync(join(buildDir, 'Release'), { recursive: true, force: true }) + renameSync(entryDir, join(buildDir, 'Release')) + } + // Other arches, plus any staging or quarantine leftovers, are dead weight in this slice. + for (const name of readdirSync(buildDir)) { + if (isNativeBuildEntryName(name)) { + rmSync(join(buildDir, name), { recursive: true, force: true }) + } + } + } +} + +module.exports = { adoptPackagedNativeBuildEntries } diff --git a/config/packaged-runtime-node-modules.cjs b/config/packaged-runtime-node-modules.cjs index 63556ad3b6dc..62ed2ff47f55 100644 --- a/config/packaged-runtime-node-modules.cjs +++ b/config/packaged-runtime-node-modules.cjs @@ -10,6 +10,11 @@ const { const { dirname, join, resolve } = require('node:path') const { builtinModules, createRequire } = require('node:module') const { PE_MACHINE, readPeMachine } = require('./scripts/windows-pe-machine.cjs') +const { adoptPackagedNativeBuildEntries } = require('./packaged-native-build-entries.cjs') +const { + NATIVE_BUILD_PACKAGES, + TRANSIENT_NATIVE_BUILD_ENTRY_FILTER +} = require('./scripts/native-build-entry.cjs') const projectDir = resolve(__dirname, '..') const requireFromProject = createRequire(join(projectDir, 'package.json')) @@ -223,7 +228,9 @@ function createPackagedRuntimeNodeModuleResources(electronPlatformName = process return collectPackagedRuntimePackages(electronPlatformName).map(([packageName, packageDir]) => ({ from: packageDir, to: join('node_modules', ...packageName.split('/')), - ...(packageName === '@orca/process-host' ? { filter: ['package.json', 'dist/**/*'] } : {}) + ...(packageName === '@orca/process-host' ? { filter: ['package.json', 'dist/**/*'] } : {}), + // Why: another worktree can rename these away mid-copy, failing the walk with ENOENT. + ...(NATIVE_BUILD_PACKAGES[packageName] ? { filter: TRANSIENT_NATIVE_BUILD_ENTRY_FILTER } : {}) })) } @@ -658,6 +665,8 @@ function assertPackagedNativeVariantsInstalled(electronPlatformName, electronArc function prunePackagedRuntimeNodeModules(resourcesDir, electronPlatformName, electronArch) { const architecture = normalizeElectronArchitecture(electronArch) + // Before the node-pty prune, which judges build/Release. + adoptPackagedNativeBuildEntries(resourcesDir, electronPlatformName, architecture) prunePackagedNodePty(resourcesDir, electronPlatformName, architecture) prunePackagedParcelWatcher(resourcesDir, electronPlatformName, architecture) // Why before the filename walk: zod/src is deleted wholesale, so walking it first is wasted work. diff --git a/config/patches/@vscode__windows-process-tree@0.8.0.patch b/config/patches/@vscode__windows-process-tree@0.8.0.patch index bc50a8cfa631..ac0defcee9fa 100644 --- a/config/patches/@vscode__windows-process-tree@0.8.0.patch +++ b/config/patches/@vscode__windows-process-tree@0.8.0.patch @@ -32,13 +32,17 @@ index 11a5e71..1b2e74b 100644 "/guard:cf", "/sdl", diff --git a/lib/index.js b/lib/index.js -index e586cd6..0ee62c3 100644 +index 9747a7402600cd252859144d32580ed45c8c93f7..69cf7fe6d14e52b199ac9ab869567be5d9bb708c 100644 --- a/lib/index.js +++ b/lib/index.js -@@ -7,11 +7,14 @@ +@@ -6,12 +6,17 @@ + Object.defineProperty(exports, "__esModule", { value: true }); exports.getAllProcesses = exports.getProcessTree = exports.getProcessCpuUsage = exports.getProcessList = exports.filterProcessList = exports.buildProcessTree = exports.ProcessDataFlag = void 0; const util_1 = require("util"); - const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; +-const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; ++// Orca's per-ABI build, published into package dirs shared across worktrees, wins over build/Release. ++const orcaNativeBuild = `${__dirname}/../build/orca-napi-win32-${process.arch}/windows_process_tree.node`; ++const native = process.platform === 'win32' ? require(require('fs').existsSync(orcaNativeBuild) ? orcaNativeBuild : '../build/Release/windows_process_tree.node') : undefined; +exports.getProcessCreationTime = native === undefined ? undefined : native.getProcessCreationTime; +exports.supportedProcessDataFlags = native === undefined ? undefined : native.supportedProcessDataFlags; var ProcessDataFlag; @@ -50,7 +54,7 @@ index e586cd6..0ee62c3 100644 })(ProcessDataFlag = exports.ProcessDataFlag || (exports.ProcessDataFlag = {})); // requestInProgress is used for any function that uses CreateToolhelp32Snapshot, as multiple calls // to this cannot be done at the same time. -@@ -66,11 +69,12 @@ +@@ -66,11 +71,12 @@ function buildProcessTree(rootPid, processList, maxDepth = MAX_FILTER_DEPTH) { // • the properties are inlined/splatted // • the 'ppid' field is omitted // • the depth of the tree is limited by `maxDepth` @@ -65,13 +69,17 @@ index e586cd6..0ee62c3 100644 }); return buildNode(root, maxDepth); diff --git a/lib/index.ts b/lib/index.ts -index 7b53aad..284dae1 100644 +index 7b53aad05c3682a5c7d814d81a39c3f391ae97e3..364b384e7d55f3682515431839352b5b4a9ffa93 100644 --- a/lib/index.ts +++ b/lib/index.ts -@@ -6,12 +6,16 @@ +@@ -5,13 +5,19 @@ + import { promisify } from 'util'; - const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; +-const native = process.platform === 'win32' ? require('../build/Release/windows_process_tree.node') : undefined; ++// Orca's per-ABI build, published into package dirs shared across worktrees, wins over build/Release. ++const orcaNativeBuild = `${__dirname}/../build/orca-napi-win32-${process.arch}/windows_process_tree.node`; ++const native = process.platform === 'win32' ? require(require('fs').existsSync(orcaNativeBuild) ? orcaNativeBuild : '../build/Release/windows_process_tree.node') : undefined; +export const getProcessCreationTime: ((pid: number) => number | undefined) | undefined = native?.getProcessCreationTime; +/** The flag bits this compiled addon reports; undefined off win32. */ +export const supportedProcessDataFlags: number | undefined = native?.supportedProcessDataFlags; @@ -86,7 +94,7 @@ index 7b53aad..284dae1 100644 } type RequestCallback = (processList: IProcessInfo[]) => void; -@@ -81,11 +85,12 @@ +@@ -81,11 +87,12 @@ export function buildProcessTree(rootPid: number, processList: Iterable