diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81762cb2..b6f8cb58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,12 @@ jobs: release-shards: name: Release / ${{ matrix.shard }} runs-on: ${{ matrix.runner }} + container: + image: ${{ matrix.browser_image }} + options: --ipc=host --init + defaults: + run: + shell: bash permissions: contents: read packages: read @@ -89,14 +95,17 @@ jobs: - shard: ui-1 port: 32004 runner: ubuntu-24.04 + browser_image: mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e - shard: ui-2 port: 32008 runner: ubuntu-24.04 + browser_image: mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e # `release-suites.mjs` mints the demo access password per run, so # nothing here needs a secret. - shard: demo port: 32006 runner: ubuntu-24.04 + browser_image: mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e - shard: proof port: 32005 runner: ubuntu-24.04 @@ -105,10 +114,11 @@ jobs: ci-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard }} CLOCKWORK_RELEASE_ARTIFACT_ROOT: .artifacts/release-ci CLOCKWORK_RELEASE_PORT_BASE: ${{ matrix.port }} - CLOCKWORK_RELEASE_CACHE_ROOT: .artifacts/release-cache/${{ matrix.shard }} + # Separate machines provide isolation. A stable relative path lets + # actions/cache restore the same archive into every machine of its kind. + CLOCKWORK_RELEASE_CACHE_ROOT: .artifacts/release-cache/ci CLOCKWORK_RELEASE_SHARD: ${{ matrix.shard }} - PLAYWRIGHT_BROWSERS_PATH: - .artifacts/release-cache/${{ matrix.shard }}/playwright-browsers + PLAYWRIGHT_BROWSERS_PATH: .artifacts/release-cache/ci/playwright-browsers LOCAL_DB_HOST: 127.0.0.1 LOCAL_DB_PASSWORD: postgres steps: @@ -119,6 +129,13 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 2 + # The container runs as root, while its mounted workspace is created by + # the host runner. Own this disposable checkout before isolating Git's + # configuration; relying on checkout's global safe.directory exemption + # would stop working when that inherited configuration is scrubbed. + - name: Own the container checkout + if: matrix.browser_image != '' + run: chown -R --no-dereference "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" - uses: pnpm/action-setup@v4 with: version: 10.34.5 @@ -142,15 +159,40 @@ jobs: echo "GIT_CONFIG_GLOBAL=/dev/null" echo "GIT_CONFIG_NOSYSTEM=1" } >> "$GITHUB_ENV" - - name: Frozen install in a fresh isolated store + # Reuse only downloaded package content. Installation, tests and builds + # still run; each shard owns and removes its restored store. Main writes + # one host cache and one container cache (their compression tools differ) + # before the release runner cleans up the stores. + - name: Restore package downloads + id: package-cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CLOCKWORK_RELEASE_CACHE_ROOT }}/pnpm-store + key: + packages-v1-${{ matrix.browser_image && 'browser' || 'host' }}-${{ + runner.os }}-${{ runner.arch }}-node24.18.1-pnpm10.34.5-${{ + hashFiles('pnpm-lock.yaml', 'pnpm-workspace.yaml', '.npmrc', + 'patches/**') }} + - name: Frozen install with package integrity verification run: | - pnpm install --frozen-lockfile --store-dir "$CLOCKWORK_RELEASE_CACHE_ROOT/pnpm-store" + pnpm install --frozen-lockfile --prefer-offline --verify-store-integrity --store-dir "$CLOCKWORK_RELEASE_CACHE_ROOT/pnpm-store" echo "CLOCKWORK_RELEASE_FROZEN_INSTALL_VERIFIED=1" >> "$GITHUB_ENV" - - name: Install Chromium and Linux dependencies + - name: Save verified package downloads + if: + github.event_name == 'push' && github.ref == 'refs/heads/main' && + (matrix.shard == 'static' || matrix.shard == 'ui-1') && + steps.package-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ env.CLOCKWORK_RELEASE_CACHE_ROOT }}/pnpm-store + key: ${{ steps.package-cache.outputs.cache-primary-key }} + - name: Prepare and verify Chromium if: startsWith(matrix.shard, 'ui-') || matrix.shard == 'demo' || matrix.shard == 'proof' - run: pnpm playwright:install:ci + env: + CLOCKWORK_PLAYWRIGHT_IMAGE: ${{ matrix.browser_image }} + run: node scripts/prepare-ci-browser.mjs # `semantic-pdf.integration.test.ts` spawns `pdftotext` directly to read # back all 15 rendered document kinds, so Poppler is a hard dependency of # this shard rather than a convenience. The runner image does not carry diff --git a/.github/workflows/deploy-check.yml b/.github/workflows/deploy-check.yml index e5425277..5443f284 100644 --- a/.github/workflows/deploy-check.yml +++ b/.github/workflows/deploy-check.yml @@ -67,9 +67,18 @@ jobs: # pull request and type-checks every package there, so this job checks # that the image builds rather than repeating that work. - name: Build the image - run: - docker buildx build --platform linux/arm64 --build-arg - CLOCKWORK_TYPECHECKED_IN_CI=1 -t clockwork:check --load . + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/arm64 + load: true + tags: clockwork:check + build-args: CLOCKWORK_TYPECHECKED_IN_CI=1 + # PRs may read main's dependency layers; only deployments write the + # reusable image cache. Placeholder configuration is still rebuilt. + cache-from: | + type=gha,scope=clockwork-image-staging-arm64-v1 + type=gha,scope=clockwork-image-production-arm64-v1 - name: Check the migration tooling is in the image run: docker run --rm --entrypoint sh clockwork:check -c 'psql --version && diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 881a4d7a..55e0574b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,6 +29,10 @@ jobs: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_repository.full_name == github.repository) + # workflow_run defaults to a read-only cache token. This job accepts only + # successful main pushes from this repository (or a main-only dispatch), + # so its reusable deployment may publish trusted image layers. + cache-mode: write uses: ./.github/workflows/terraform.yml permissions: id-token: write @@ -42,6 +46,7 @@ jobs: production: needs: staging + cache-mode: write uses: ./.github/workflows/terraform.yml permissions: id-token: write diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index f1a1b0dd..a1a7e3d7 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -143,12 +143,31 @@ jobs: run: make init working-directory: deploy - - name: Build and push the image - run: make docker-push + - name: Prepare the image build working-directory: deploy - env: - CLOCKWORK_TYPECHECKED_IN_CI: - ${{ inputs.typechecked_in_ci && '1' || '0' }} + run: | + make docker-login .env.production.local + echo "IMAGE_REF=$(make --no-print-directory -s image-ref)" >> "$GITHUB_ENV" + + # The Docker context excludes secrets; the rendered environment contains + # public configuration only. Each environment's COPY still participates + # in the cache key, while dependency and OS layers can be reused across + # environments. Cache export failure must not fail a valid deployment. + - name: Build and push the image + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/arm64 + push: true + tags: ${{ env.IMAGE_REF }} + build-args: | + CLOCKWORK_TYPECHECKED_IN_CI=${{ inputs.typechecked_in_ci && '1' || '0' }} + cache-from: | + type=gha,scope=clockwork-image-staging-arm64-v1 + type=gha,scope=clockwork-image-production-arm64-v1 + cache-to: + type=gha,scope=clockwork-image-${{ inputs.environment + }}-arm64-v1,mode=max,ignore-error=true,timeout=2m - name: Migrate the database run: make migrate diff --git a/deploy/Makefile b/deploy/Makefile index 5e480a76..8f6d9728 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -95,6 +95,10 @@ docker-build: .env.production.local docker-push: docker-login .env.production.local cd .. && docker buildx build --platform $(PLATFORM) $(IMAGE_BUILD_ARGS) -t $(IMAGE_REF) --push . +.PHONY: image-ref +image-ref: + @printf '%s\n' '$(IMAGE_REF)' + # Every target that registers a task definition checks the tag first. ECS # retries a missing image for the whole CodeDeploy window (an hour) and only # then rolls back, so an `apply-app` without a `docker-push` for the same diff --git a/docs/operations/release-orchestration.md b/docs/operations/release-orchestration.md index c6a8d704..5d2c4976 100644 --- a/docs/operations/release-orchestration.md +++ b/docs/operations/release-orchestration.md @@ -24,6 +24,43 @@ contract for the join validator. `lint` is the type-aware ESLint pass, split from `static` because it was the longest static step (178 of 449 seconds on main, 2026-10-05). +## Reusable setup, fresh assertions + +CI restores package downloads into each shard's disposable store using an exact +OS, architecture, Node, pnpm, lockfile, workspace configuration and patch key. +Every shard still performs a frozen install with store-integrity verification. +Only the static and first UI shards on a push to main save package caches, +before cleanup. Host and browser-container caches have separate keys because +their archive compression tools differ. Every machine uses the same relative +store path so the cache archive can actually be shared within its runner type. +Installed dependencies, test results, application builds and authentication +state are not restored from that cache. Turbo result caching remains disabled. + +The two UI shards and the demo shard run inside an official Playwright Ubuntu +image pinned by version and digest. Its version must match the application's +Playwright package. Chromium is copied into the shard's disposable browser store +and launched before testing. This removes repeated OS/font downloads, which took +539 seconds in the 2026-10-22 release. The database proof stays on the host for +Docker access; it downloads the pinned browser and checks it launches. It +installs OS packages only if launch reports missing system libraries. Other +setup failures fail immediately, and no product assertions are retried. + +Deployment builds use environment-scoped BuildKit layer caches. Both +environments can reuse dependency and runtime layers; the rendered public +environment remains a hashed build input, so staging configuration cannot +satisfy a production build. Secrets remain outside the Docker context and arrive +at runtime from Secrets Manager. PR image checks read these caches; deployments +write them. The two deployment jobs explicitly grant cache writes only behind +the existing successful same-repository main-push or main-only manual-dispatch +gate: GitHub otherwise gives `workflow_run` read-only cache access. Export +errors do not block a valid deployment. Migrations, staged rollout and smoke +checks still run for each environment. + +Compare complete workflow elapsed time and setup-step durations with recent +clean runs. The first run populates caches and is not a warm-cache measurement. +A clean baseline on 2026-10-12 took 7m41s for main CI and 14m25s for both +deployments. + The UI shards explicitly select the non-production `demo` adapter and drive a development server, because their journeys authenticate by persona header. The demo shard builds the application into its own output directory and serves that diff --git a/scripts/prepare-ci-browser.mjs b/scripts/prepare-ci-browser.mjs new file mode 100644 index 00000000..6b6a6c1c --- /dev/null +++ b/scripts/prepare-ci-browser.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { cp, mkdir, readFile, readdir } from "node:fs/promises"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export function verifyBrowserImage(image, version) { + assert.match( + image, + /^mcr\.microsoft\.com\/playwright:v[\d.]+-noble@sha256:[a-f\d]{64}$/, + "The browser environment must use an immutable official Playwright image.", + ); + assert.equal(image.split(":v")[1].split("-noble")[0], version); +} + +export function missingBrowserLibraries(error) { + return /Host system is missing dependencies|error while loading shared libraries/.test( + String(error), + ); +} + +async function main() { + const image = process.env.CLOCKWORK_PLAYWRIGHT_IMAGE; + const browserRoot = process.env.PLAYWRIGHT_BROWSERS_PATH; + assert.ok( + browserRoot, + "PLAYWRIGHT_BROWSERS_PATH must name the isolated store.", + ); + if (image) { + const manifest = JSON.parse( + await readFile( + new URL("../apps/web/package.json", import.meta.url), + "utf8", + ), + ); + verifyBrowserImage(image, manifest.devDependencies["@playwright/test"]); + const directories = (await readdir("/ms-playwright")).filter((name) => + /^(chromium-|chromium_headless_shell-|ffmpeg-)/.test(name), + ); + assert.ok(directories.length >= 3, "The image is missing Chromium assets."); + await mkdir(browserRoot, { recursive: true }); + for (const name of directories) + await cp( + path.join("/ms-playwright", name), + path.join(browserRoot, name), + { + recursive: true, + }, + ); + } else { + // GitHub's Ubuntu runner already supplies Chromium's runtime libraries. + // Avoid upgrading the OS and downloading fonts before every proof run. + execFileSync("pnpm", ["playwright:install"], { stdio: "inherit" }); + } + + const require = createRequire( + new URL("../apps/web/package.json", import.meta.url), + ); + const { chromium } = require("@playwright/test"); + const launch = async () => { + const browser = await chromium.launch(); + await browser.close(); + }; + try { + await launch(); + } catch (error) { + // A runner-image change may remove a system library. Only that diagnosed + // setup failure installs OS dependencies; product tests are never retried. + if (image || !missingBrowserLibraries(error)) throw error; + execFileSync("pnpm", ["playwright:install:ci"], { stdio: "inherit" }); + await launch(); + } + console.log("Pinned Chromium launches successfully in the isolated store."); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) + await main(); diff --git a/scripts/release-artifacts.test.mjs b/scripts/release-artifacts.test.mjs index 3c38856c..9a5a87b9 100644 --- a/scripts/release-artifacts.test.mjs +++ b/scripts/release-artifacts.test.mjs @@ -3,6 +3,10 @@ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { readFile, readdir } from "node:fs/promises"; import { test } from "node:test"; +import { + missingBrowserLibraries, + verifyBrowserImage, +} from "./prepare-ci-browser.mjs"; import { expectedReleaseCommands, @@ -1149,6 +1153,96 @@ test("runs every CI shard on the Linux runner", () => { assert.equal(runner, "ubuntu-24.04", shard); }); +test("pins all UI container images to the installed Playwright version and a digest", async () => { + const webPackage = JSON.parse( + await readFile( + new URL("../apps/web/package.json", import.meta.url), + "utf8", + ), + ); + const images = [...checkedInWorkflow.matchAll(/browser_image:\s*(\S+)/g)]; + assert.equal(images.length, 3); + for (const [, image] of images) + verifyBrowserImage(image, webPackage.devDependencies["@playwright/test"]); + assert.throws(() => verifyBrowserImage(images[0][1], "0.0.0")); + assert.throws(() => + verifyBrowserImage("mcr.microsoft.com/playwright:latest", "1.62.1"), + ); +}); + +test("only missing system libraries permit a browser setup fallback", () => { + assert.equal( + missingBrowserLibraries( + new Error("Host system is missing dependencies to run browsers."), + ), + true, + ); + assert.equal( + missingBrowserLibraries( + new Error("error while loading shared libraries: libnss3.so"), + ), + true, + ); + for (const message of [ + "page assertion failed", + "browser launch timed out", + "Executable doesn't exist", + "Target page has been closed", + ]) + assert.equal(missingBrowserLibraries(new Error(message)), false); +}); + +test("reused package downloads still require a frozen verified installation", () => { + assert.match( + checkedInWorkflow, + /pnpm install --frozen-lockfile --prefer-offline --verify-store-integrity --store-dir/, + ); + const save = checkedInWorkflow.match( + /- name: Save verified package downloads\n([\s\S]*?)\n {6}- name:/, + )?.[1]; + assert.ok(save); + assert.match(save, /github\.event_name == 'push'/); + assert.match(save, /github\.ref == 'refs\/heads\/main'/); + assert.match(save, /matrix\.shard == 'static'/); + assert.match(save, /matrix\.shard == 'ui-1'/); + assert.match( + checkedInWorkflow, + /CLOCKWORK_RELEASE_CACHE_ROOT: \.artifacts\/release-cache\/ci/, + ); + assert.ok( + checkedInWorkflow.indexOf( + "Frozen install with package integrity verification", + ) < checkedInWorkflow.indexOf("Save verified package downloads"), + ); + assert.ok( + checkedInWorkflow.indexOf("Save verified package downloads") < + checkedInWorkflow.indexOf("Run isolated release shard"), + ); +}); + +test("deployment cache writes retain the trusted main-run gate and PRs only restore", async () => { + const deploy = await readFile( + new URL("../.github/workflows/deploy.yml", import.meta.url), + "utf8", + ); + const pullRequest = await readFile( + new URL("../.github/workflows/deploy-check.yml", import.meta.url), + "utf8", + ); + assert.equal([...deploy.matchAll(/cache-mode: write/g)].length, 2); + assert.match(deploy, /branches: \[main\]/); + assert.match(deploy, /github\.ref ==\s*'refs\/heads\/main'/); + assert.match(deploy, /github\.event\.workflow_run\.conclusion == 'success'/); + assert.match(deploy, /github\.event\.workflow_run\.event == 'push'/); + assert.match( + deploy, + /github\.event\.workflow_run\.head_repository\.full_name == github\.repository/, + ); + assert.match(deploy, /production:\s*needs: staging\s*cache-mode: write/); + assert.match(pullRequest, /cache-from:/); + assert.doesNotMatch(pullRequest, /cache-to:/); +}); + test("runs every browser test in exactly one ui shard", async () => { // The `ui-*` shards name their spec files, so a spec added to // `apps/web/e2e` and to neither list would never run. Discover the specs on diff --git a/scripts/release-suites.mjs b/scripts/release-suites.mjs index 6d681b09..8420ba87 100644 --- a/scripts/release-suites.mjs +++ b/scripts/release-suites.mjs @@ -770,6 +770,11 @@ async function runSuite(name, index, context) { context.workspaces.get(name), context.gitEnvironment, ); + const workspaceStatus = execFileSync("git", ["status", "--porcelain"], { + cwd: context.workspaces.get(name), + encoding: "utf8", + env: context.gitEnvironment, + }).trim(); const result = { ...contract, status, @@ -781,15 +786,16 @@ async function runSuite(name, index, context) { candidateIdentityPreserved: JSON.stringify(finalWorkspaceIdentity) === JSON.stringify(workspaceIdentity), - trackedWorkspaceClean: - execFileSync("git", ["status", "--porcelain"], { - cwd: context.workspaces.get(name), - encoding: "utf8", - env: context.gitEnvironment, - }).trim() === "", + trackedWorkspaceClean: workspaceStatus === "", + workspaceChanges: workspaceStatus ? workspaceStatus.split("\n") : [], }; - if (!result.trackedWorkspaceClean || !result.candidateIdentityPreserved) + if (!result.trackedWorkspaceClean || !result.candidateIdentityPreserved) { result.status = "failed"; + console.error( + `[${name}] Release source changed during verification.`, + workspaceStatus || "The source identity no longer matches the candidate.", + ); + } await writeFile( path.join(suiteDirectory, "result.json"), `${JSON.stringify(result, null, 2)}\n`, @@ -944,6 +950,11 @@ async function main() { ); const allowedCacheRoots = [ path.join(artifactRoot, ".isolated-cache"), + // Fresh CI machines may share a relative archive path without sharing a + // filesystem. Local parallel worktrees keep their unique namespaces. + ...(useSharedWorkspace + ? [path.resolve(".artifacts", "release-cache", "ci")] + : []), path.resolve( ".artifacts", "release-cache",