From eb926d05ddf620c7c5c0297a7667da2eda4a561a Mon Sep 17 00:00:00 2001 From: Dayton Allen Date: Tue, 27 Oct 2026 23:27:10 +0000 Subject: [PATCH] feat(docker): stage per-service runtime trees and lower image size budgets Split the image build so core, browser, and cli each ship only the workspaces and production dependencies that service runs. node-pty loads lazily from the CLI host, so the core image can import the CLI host client without the native module. Fixes oll4com/spaceapp#64 Signed-off-by: Dayton Allen Co-authored-by: dayton536 --- .github/workflows/containers.yml | 11 +- CHANGELOG.md | 4 + Dockerfile | 18 +- PUBLIC_EXPORT_MANIFEST.json | 14 +- apps/api/package.json | 1 - deploy/docker/smoke-container-image.mjs | 253 +++++++++ deploy/docker/stage-container-runtime.mjs | 487 ++++++++++++++++++ package-lock.json | 1 - packages/cli-host/src/node-pty-spawn.ts | 17 +- scripts/container-image-size-budget.mjs | 20 +- .../container-image-size-budget.test.mjs | 6 +- .../tests/public-docker-distribution.test.mjs | 248 ++++++++- scripts/tests/public-workflows.test.mjs | 3 + 13 files changed, 1042 insertions(+), 41 deletions(-) create mode 100644 deploy/docker/smoke-container-image.mjs create mode 100644 deploy/docker/stage-container-runtime.mjs diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 97d278576..0ff45b870 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -9,6 +9,7 @@ on: - "deploy/docker/**" - "starter-memory/**" - "Dockerfile" + - "package.json" - "package-lock.json" push: branches: @@ -20,6 +21,7 @@ on: - "deploy/docker/**" - "starter-memory/**" - "Dockerfile" + - "package.json" - "package-lock.json" workflow_dispatch: @@ -44,7 +46,7 @@ jobs: - amd64 - arm64 runs-on: ubuntu-latest - timeout-minutes: 50 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -71,3 +73,10 @@ jobs: node scripts/container-image-size-budget.mjs --target "${{ matrix.target }}" --image "spaceapp-ci:${{ matrix.target }}-${{ matrix.arch }}" + + - name: Smoke check container image + run: >- + node deploy/docker/smoke-container-image.mjs + --target "${{ matrix.target }}" + --image "spaceapp-ci:${{ matrix.target }}-${{ matrix.arch }}" + --arch "${{ matrix.arch }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index a2a4964db..ad687423f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Semantic Versioning. ## [Unreleased] +### Container images + +- `spaceapp-core`, `spaceapp-browser`, and `spaceapp-cli` each ship only the workspaces and production dependencies that service runs. Image size budgets are 850 MiB (core), 1350 MiB (browser), and 1000 MiB (cli). + ## [1.1.8] - 2026-10-25 Source: sanitized Space app @ 8d5bf21b32 (4948 files). Verifies SpaceApp's image attestations before an diff --git a/Dockerfile b/Dockerfile index 160db70fa..c3596ee7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ # syntax=docker/dockerfile:1.7 FROM node:22.23.0-trixie-slim@sha256:cfd8f2a5bc50526aee08e88970979f92722828e7dcc6d8983607fb8bff4bdb82 AS build +ARG TARGETARCH WORKDIR /app RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential liblzma5 python3 pkg-config \ @@ -9,7 +10,8 @@ COPY . . RUN npm ci --no-audit --no-fund \ && npm run build \ && npm prune --omit=dev \ - && find node_modules -type f \( -name '*.js.map' -o -name '*.mjs.map' -o -name '*.cjs.map' -o -name '*.css.map' \) -delete + && find node_modules -type f \( -name '*.js.map' -o -name '*.mjs.map' -o -name '*.cjs.map' -o -name '*.css.map' \) -delete \ + && node deploy/docker/stage-container-runtime.mjs --source /app --output /app-targets --arch "${TARGETARCH:-}" FROM node:22.23.0-trixie-slim@sha256:cfd8f2a5bc50526aee08e88970979f92722828e7dcc6d8983607fb8bff4bdb82 AS runtime-base ARG TARGETARCH @@ -54,17 +56,8 @@ RUN npm install --global --ignore-scripts --no-audit --no-fund "npm@${NPM_VERSIO && groupadd --gid 10001 spaceapp \ && useradd --uid 10001 --gid 10001 --create-home --home-dir /var/lib/spaceapp-cli --shell /bin/bash spaceapp WORKDIR /app -COPY --from=build --chown=spaceapp:spaceapp /app/package.json /app/package-lock.json ./ -COPY --from=build --chown=spaceapp:spaceapp /app/node_modules ./node_modules -COPY --from=build --chown=spaceapp:spaceapp /app/apps ./apps -COPY --from=build --chown=spaceapp:spaceapp /app/packages ./packages -COPY --from=build --chown=spaceapp:spaceapp /app/agent-skills ./agent-skills COPY --from=build --chown=spaceapp:spaceapp /app/LICENSE /app/NOTICE /app/THIRD_PARTY_NOTICES.md ./ COPY --from=build --chown=spaceapp:spaceapp /app/THIRD_PARTY_LICENSES ./THIRD_PARTY_LICENSES -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/reset-owner-password.mjs ./scripts/reset-owner-password.mjs -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/rotate-owner-setup-token.mjs ./scripts/rotate-owner-setup-token.mjs -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/portable-backup.mjs ./scripts/portable-backup.mjs -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/portable-restore.mjs ./scripts/portable-restore.mjs COPY --chown=spaceapp:spaceapp deploy/docker ./deploy/docker RUN chmod 0755 deploy/docker/core-entrypoint.sh deploy/docker/browser-entrypoint.sh deploy/docker/cli-entrypoint.sh \ && install -m 0755 deploy/docker/public-cli-wrapper.sh /usr/local/bin/spaceapp-cli-wrapper \ @@ -73,6 +66,9 @@ RUN chmod 0755 deploy/docker/core-entrypoint.sh deploy/docker/browser-entrypoint done FROM runtime-base AS core +# Per-target tree: compiled core workspaces, their production node_modules, agent-skills, +# and the owner/backup scripts (rotate-owner-setup-token.mjs, reset-owner-password.mjs, portable-backup.mjs, portable-restore.mjs). +COPY --from=build --chown=spaceapp:spaceapp /app-targets/core/ ./ COPY --chown=spaceapp:spaceapp starter-memory /opt/spaceapp/starter-memory # The Databases tool's SQLite worker (apps/api/src/databases/sqlite-worker.py) needs Python 3.12+ with its sqlite3 module; # python3-minimal has no sqlite3 (1.1.7 review: SQLite could not run in public installations). @@ -91,6 +87,7 @@ HEALTHCHECK --interval=15s --timeout=5s --start-period=45s --retries=5 \ ENTRYPOINT ["/app/deploy/docker/core-entrypoint.sh"] FROM runtime-base AS browser +COPY --from=build --chown=spaceapp:spaceapp /app-targets/browser/ ./ RUN apt-get update \ && apt-get install -y --no-install-recommends chromium fonts-liberation liblzma5 \ && rm -rf /var/lib/apt/lists/* \ @@ -103,6 +100,7 @@ ENV SPACE_BROWSER_CHROME_PATH=/usr/bin/chromium \ ENTRYPOINT ["/app/deploy/docker/browser-entrypoint.sh"] FROM runtime-base AS cli +COPY --from=build --chown=spaceapp:spaceapp /app-targets/cli/ ./ COPY --chown=root:root starter-memory/AGENTS.md /etc/AGENTS.md COPY --chown=root:root starter-memory/installation-guide.md /etc/spaceapp-installation-guide.md RUN chmod 0644 /etc/AGENTS.md diff --git a/PUBLIC_EXPORT_MANIFEST.json b/PUBLIC_EXPORT_MANIFEST.json index a5746cf44..da394ce49 100644 --- a/PUBLIC_EXPORT_MANIFEST.json +++ b/PUBLIC_EXPORT_MANIFEST.json @@ -3,7 +3,7 @@ "schemaVersion": 2, "sourceCommit": "8d5bf21b3228e5ea89557ab56b6a3bee352fc5a5", "fileCount": 4948, - "transformations": 421, + "transformations": 423, "files": [ { "path": ".gitignore", @@ -8570,8 +8570,8 @@ { "path": "apps/api/package.json", "sourceObjectId": "676dcd5165796b77548b01df98301f12c6cd6e1d", - "sha256": "045fd6412a7f53b0a652ecb12508038b030ef0b69fdcc1413502129852bd41fa", - "size": 2078 + "sha256": "f2f1c41903427c10315a65ecc39514dfe777febf3c8899ed9dc3273a7f75db5f", + "size": 2052 }, { "path": "apps/api/src/__tests__/activity-log.test.ts", @@ -23540,8 +23540,8 @@ { "path": "package-lock.json", "sourceObjectId": "bebd5dbb1bc020651af5b6b60e61823bae5fad6b", - "sha256": "2bc219913dd6fe9003169bdc3881b33f25f939510b47f58e7c26c2af85124617", - "size": 419142 + "sha256": "d4d2886596cfa962df4ab63835910b3aafceae39a9962140a504876cafcb1db4", + "size": 419112 }, { "path": "package.json", @@ -23648,8 +23648,8 @@ { "path": "packages/cli-host/src/node-pty-spawn.ts", "sourceObjectId": "47f18bfadd5b4ce8afd1da7c166362e8fcc94640", - "sha256": "9040fe7bd0fc4fd6554f071fdf918b3d3600de59607c123fa5546b3c99fd96f6", - "size": 726 + "sha256": "697222c207b1610c6e391aee9a01b5e96909c4235459f73de12e6d3a9322987f", + "size": 1205 }, { "path": "packages/cli-host/src/server.ts", diff --git a/apps/api/package.json b/apps/api/package.json index 676dcd516..5f4880b4d 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -42,7 +42,6 @@ "mammoth": "^1.12.0", "mysql2": "^3.24.5", "nanoid": "^5.1.6", - "node-pty": "^1.1.0", "pdfjs-dist": "6.3.289", "sanitize-html": "^2.18.1", "zod": "^4.6.7" diff --git a/deploy/docker/smoke-container-image.mjs b/deploy/docker/smoke-container-image.mjs new file mode 100644 index 000000000..bb787a2fb --- /dev/null +++ b/deploy/docker/smoke-container-image.mjs @@ -0,0 +1,253 @@ +#!/usr/bin/env node +// Starts one built image the way its entrypoint runs in production and checks +// the signal that image is supposed to provide. Core and browser need the same +// secret files and pgvector database the Compose stack mounts; the CLI host +// only has to accept a connection on its socket. +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +// Same pgvector image the public Compose file runs, so migrations can create +// the vector extension. The digest is a multi-arch index. +export const SMOKE_POSTGRES_IMAGE = + "pgvector/pgvector:0.8.3-pg17-trixie@sha256:3a3147ccd9a0aee952bb69e1acea1eb3cc7183544185b87a662f872cfa47ae3b"; + +export const CLI_HOST_SOCKET = "/run/spaceapp-cli/host.sock"; +export const BROWSER_HOST_SOCKET = "/run/spaceapp-browser/browser-host.sock"; + +const READY_PROBE = + "fetch('http://127.0.0.1:4911/readyz').then(async r=>{const b=await r.json();if(!r.ok||b.ok!==true)process.exit(1)}).catch(()=>process.exit(1))"; + +function socketProbe(socketPath) { + return `import('node:net').then(({createConnection})=>{const s=createConnection(${JSON.stringify(socketPath)},()=>{s.end();process.exit(0)});s.on('error',()=>process.exit(1))})`; +} + +export const CHROMIUM_PROBE = [ + "/usr/bin/chromium", + "--headless", + "--no-sandbox", + "--disable-gpu", + "--disable-dev-shm-usage", + "--dump-dom", + "about:blank" +]; + +export function parseSmokeArgs(argv) { + let target = ""; + let image = ""; + let arch = "amd64"; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--target") { + target = argv[index + 1] ?? ""; + index += 1; + } else if (arg === "--image") { + image = argv[index + 1] ?? ""; + index += 1; + } else if (arg === "--arch") { + arch = argv[index + 1] ?? "amd64"; + index += 1; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!["core", "browser", "cli"].includes(target)) { + throw new Error(`Unsupported container target "${target}". Expected core, browser, or cli.`); + } + if (!image) throw new Error("Missing required --image ."); + if (!["amd64", "arm64"].includes(arch)) { + throw new Error(`Unsupported architecture "${arch}". Expected amd64 or arm64.`); + } + return { target, image, arch }; +} + +export function buildSmokePlan({ target, image, arch = "amd64", runId = "smoke", secretDir = "/tmp/spaceapp-smoke-secrets" }) { + const platform = `linux/${arch}`; + const networkName = `spaceapp-smoke-${target}-${arch}-${runId}`.replace(/[^a-z0-9_.-]/gi, "-").slice(0, 60); + const pgContainer = `${networkName}-pg`.slice(0, 63); + const targetContainer = `${networkName}-${target}`.slice(0, 63); + const databaseUrl = `postgresql://spaceapp:spaceapp@${pgContainer}:5432/spaceapp`; + const requiresPostgres = target !== "cli"; + const secretMounts = []; + if (target === "core" || target === "browser") { + secretMounts.push(`${secretDir}/database-url:/run/secrets/database-url:ro`); + } + if (target === "core") { + secretMounts.push(`${secretDir}/session-secret:/run/secrets/session-secret:ro`); + secretMounts.push(`${secretDir}/setup-token:/run/secrets/setup-token:ro`); + } + + const containerEnv = {}; + if (target === "core") { + Object.assign(containerEnv, { + SPACE_PUBLIC_DISTRIBUTION: "true", + SPACEAPP_WORKFLOWS_ENABLED: "false", + SPACE_RUNTIME_STORE: "postgres", + SPACE_BROWSER_SESSIONS_ENABLED: "false", + SPACE_DEV_LOGIN: "false", + SPACE_AGENT_TOOLS_WRITER: "disabled" + }); + } else if (target === "browser") { + Object.assign(containerEnv, { + SPACE_PUBLIC_DISTRIBUTION: "true", + SPACE_RUNTIME_STORE: "postgres", + SPACE_DATABASE_URL_FILE: "/run/secrets/database-url", + SPACE_BROWSER_HOST_SOCKET: BROWSER_HOST_SOCKET + }); + } else { + containerEnv.SPACE_CLI_HOST_SOCKET = CLI_HOST_SOCKET; + } + + const envFlags = Object.entries(containerEnv).flatMap(([key, value]) => ["-e", `${key}=${value}`]); + const mountFlags = secretMounts.flatMap((mount) => ["-v", mount]); + const runArgs = [ + "run", "-d", + "--name", targetContainer, + "--platform", platform, + ...(requiresPostgres ? ["--network", networkName] : []), + ...mountFlags, + ...envFlags, + image + ]; + + return { + target, + image, + arch, + platform, + networkName, + pgContainer: requiresPostgres ? pgContainer : null, + targetContainer, + requiresPostgres, + databaseUrl: requiresPostgres ? databaseUrl : null, + postgresImage: SMOKE_POSTGRES_IMAGE, + secretMounts, + containerEnv, + // Core's entrypoint applies packages/db migrations before the supervisor. + // Browser has no migration entrypoint, so the smoke runs migrate.js first. + migrateArgs: target === "browser" + ? ["run", "--rm", "--platform", platform, "--network", networkName, "-e", `SPACE_DATABASE_URL=${databaseUrl}`, "--entrypoint", "node", image, "packages/db/dist/migrate.js"] + : null, + runArgs, + readinessProbe: target === "core" ? READY_PROBE : null, + socketProbe: target === "cli" ? socketProbe(CLI_HOST_SOCKET) : target === "browser" ? socketProbe(BROWSER_HOST_SOCKET) : null, + chromiumProbe: target === "browser" ? CHROMIUM_PROBE : null + }; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function pollUntil(fn, { attempts, delayMs, description }) { + let lastError = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + await fn(); + return; + } catch (error) { + lastError = error; + if (attempt < attempts) await sleep(delayMs); + } + } + const message = lastError instanceof Error ? lastError.message : String(lastError); + throw new Error(`Timed out waiting for ${description}: ${message}`); +} + +async function containerLogs(execImpl, name) { + try { + const { stdout, stderr } = await execImpl("docker", ["logs", "--tail", "120", name], { timeout: 20_000 }); + return `${stdout ?? ""}${stderr ?? ""}`.trim(); + } catch (error) { + return error instanceof Error ? error.message : String(error); + } +} + +export async function runContainerSmoke(options, execImpl = execFileAsync) { + const secretDir = await mkdtemp(join(tmpdir(), "spaceapp-smoke-")); + const plan = buildSmokePlan({ ...options, secretDir, runId: `${process.pid}-${Date.now()}` }); + const databaseUrl = plan.databaseUrl ?? ""; + await writeFile(join(secretDir, "database-url"), databaseUrl); + // The core entrypoint rejects a setup token shorter than 32 characters and a session secret shorter than 16. + await writeFile(join(secretDir, "session-secret"), "smoke-test-session-secret-0123456789abcdef"); + await writeFile(join(secretDir, "setup-token"), "smoke-test-setup-token-0123456789abcdef"); + + try { + if (plan.requiresPostgres) { + await execImpl("docker", ["network", "create", plan.networkName]); + await execImpl("docker", [ + "run", "-d", + "--name", plan.pgContainer, + "--platform", plan.platform, + "--network", plan.networkName, + "-e", "POSTGRES_USER=spaceapp", + "-e", "POSTGRES_PASSWORD=spaceapp", + "-e", "POSTGRES_DB=spaceapp", + plan.postgresImage + ]); + await pollUntil( + () => execImpl("docker", ["exec", plan.pgContainer, "pg_isready", "-U", "spaceapp", "-d", "spaceapp"]), + { attempts: 40, delayMs: 1_000, description: "PostgreSQL readiness" } + ); + if (plan.migrateArgs) { + await execImpl("docker", plan.migrateArgs, { timeout: 180_000 }); + } + } + + await execImpl("docker", plan.runArgs); + const probe = plan.readinessProbe ?? plan.socketProbe; + const description = plan.target === "core" + ? "core /readyz ok:true" + : plan.target === "cli" + ? "cli host socket connection" + : "browser host socket connection"; + try { + await pollUntil( + async () => { + const state = await execImpl("docker", ["inspect", "--format", "{{.State.Status}}", plan.targetContainer]); + const status = `${state.stdout ?? ""}`.trim(); + if (status && status !== "running") { + throw new Error(`container status is ${status}`); + } + await execImpl("docker", ["exec", plan.targetContainer, "node", "-e", probe]); + }, + { + attempts: plan.target === "core" ? 90 : 40, + delayMs: plan.target === "core" ? 3_000 : 1_000, + description + } + ); + } catch (error) { + const logs = await containerLogs(execImpl, plan.targetContainer); + const message = error instanceof Error ? error.message : String(error); + throw new Error(`${message}\n--- ${plan.targetContainer} logs ---\n${logs}`); + } + + if (plan.chromiumProbe) { + await execImpl("docker", ["exec", plan.targetContainer, ...plan.chromiumProbe], { timeout: 180_000 }); + } + return { ok: true, target: plan.target, image: plan.image, arch: options.arch }; + } finally { + await execImpl("docker", ["rm", "-f", plan.targetContainer]).catch(() => {}); + if (plan.pgContainer) await execImpl("docker", ["rm", "-f", plan.pgContainer]).catch(() => {}); + if (plan.requiresPostgres) await execImpl("docker", ["network", "rm", plan.networkName]).catch(() => {}); + await rm(secretDir, { recursive: true, force: true }); + } +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + const options = parseSmokeArgs(process.argv.slice(2)); + try { + const result = await runContainerSmoke(options); + process.stdout.write(`${JSON.stringify(result)}\n`); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/deploy/docker/stage-container-runtime.mjs b/deploy/docker/stage-container-runtime.mjs new file mode 100644 index 000000000..794912b45 --- /dev/null +++ b/deploy/docker/stage-container-runtime.mjs @@ -0,0 +1,487 @@ +#!/usr/bin/env node +// Stages the runtime tree each container target actually starts. +// The build stage runs this after `npm run build` and `npm prune --omit=dev`. +import { + cpSync, + existsSync, + lstatSync, + mkdirSync, + readFileSync, + readdirSync, + readlinkSync, + realpathSync, + rmSync, + symlinkSync +} from "node:fs"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; + +const NODE_BUILTINS = new Set([ + "assert", "async_hooks", "buffer", "child_process", "cluster", "console", "constants", "crypto", + "dgram", "diagnostics_channel", "dns", "domain", "events", "fs", "http", "http2", "https", + "inspector", "module", "net", "os", "path", "perf_hooks", "process", "punycode", "querystring", + "readline", "repl", "sqlite", "stream", "string_decoder", "timers", "tls", "trace_events", "tty", + "url", "util", "v8", "vm", "wasi", "worker_threads", "zlib" +]); + +// Workspaces the core entrypoint and supervisor load (API, worker, and the +// packages they import). The web UI is a prebuilt static tree, copied separately. +export const CORE_RUNTIME_WORKSPACES = [ + "apps/api", + "apps/worker", + "packages/browser-host", + "packages/cli-host", + "packages/codex-app-server", + "packages/contracts", + "packages/db", + "packages/gateway", + "packages/memory-graph", + "packages/opencode-control", + "packages/runtime", + "packages/streaming" +]; + +// Owner and backup commands the launcher execs inside spaceapp-core, plus the +// proof-room fixture the API imports from the image. +export const CORE_RUNTIME_SCRIPTS = [ + "scripts/reset-owner-password.mjs", + "scripts/rotate-owner-setup-token.mjs", + "scripts/portable-backup.mjs", + "scripts/portable-restore.mjs", + "scripts/cli-render-fixture.mjs" +]; + +const WORKSPACE_RUNTIME_ASSETS = { + "packages/db": ["migrations"], + "packages/gateway": ["grammars", "src/providers/local/space-local-embeddings.py"] +}; + +export function normalizeTargetArch(arch = process.arch) { + const normalized = String(arch || process.arch).trim().toLowerCase(); + if (normalized === "amd64" || normalized === "x64" || normalized === "x86_64") return "amd64"; + if (normalized === "arm64" || normalized === "aarch64") return "arm64"; + throw new Error(`Unsupported container target architecture: ${String(arch)}`); +} + +export function shouldKeepOptionalPackage(packageName, targetArch) { + const name = String(packageName).toLowerCase(); + const platformSpecific = /(darwin|win32|windows|android|freebsd|openbsd|sunos|linux-|musl|gnueabihf|riscv|arm64|aarch64|x86_64|-x64\b)/.test(name); + if (!platformSpecific) return true; + if (/(darwin|win32|windows|android|freebsd|openbsd|sunos|musl|gnueabihf|riscv)/.test(name)) return false; + const arm = name.includes("arm64") || name.includes("aarch64"); + const x64 = name.includes("x64") || name.includes("x86_64") || name.includes("amd64"); + if (/-arm(?!64)/.test(name) && !arm) return false; + if (targetArch === "amd64" && arm && !x64) return false; + if (targetArch === "arm64" && x64 && !arm) return false; + return true; +} + +function prebuildDirAllowed(name, targetArch) { + const match = /^(darwin|win32|linux|android|freebsd|openbsd)-(x64|arm64|ia32|arm)(?:-(musl|gnu))?$/.exec(name); + if (!match) return true; + const [, platform, arch, libc] = match; + if (platform !== "linux" || libc === "musl") return false; + return targetArch === "amd64" ? arch === "x64" : arch === "arm64"; +} + +function temporalReleaseAllowed(name, targetArch) { + return targetArch === "amd64" + ? name === "x86_64-unknown-linux-gnu" + : name === "aarch64-unknown-linux-gnu"; +} + +// Drops declaration files, source maps, foreign-platform native builds, and the +// Temporal Rust sources. Linux node-pty is compiled into build/Release by npm ci +// (the published tarball has no linux prebuild), so that directory stays. +export function shouldKeepRuntimePath(relativePath, isDirectory, targetArch) { + const normalized = relativePath.replaceAll("\\", "/"); + const name = basename(normalized); + if (!isDirectory) { + if (/\.(?:map|pdb)$/.test(name) || /\.d\.[cm]?ts$/.test(name)) return false; + if (/\.(?:test|spec)\.(?:js|mjs|cjs)$/.test(name)) return false; + } + if (/(?:^|\/)node-pty\/(?:third_party|deps|src)(?:\/|$)/.test(normalized)) return false; + if (/@temporalio\/core-bridge\/(?:sdk-core|src)(?:\/|$)/.test(normalized)) return false; + const prebuild = normalized.match(/\/prebuilds\/([^/]+)$/); + if (isDirectory && prebuild && !prebuildDirAllowed(prebuild[1], targetArch)) return false; + const release = normalized.match(/@temporalio\/core-bridge\/releases\/([^/]+)$/); + if (isDirectory && release && !temporalReleaseAllowed(release[1], targetArch)) return false; + return true; +} + +function barePackageName(specifier) { + if (!specifier || specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("node:")) return null; + const parts = specifier.split("/"); + const base = specifier.startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]; + if (!base || NODE_BUILTINS.has(base)) return null; + return base; +} + +function isInside(root, candidate) { + const rel = relative(root, candidate); + return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel)); +} + +function stripComments(source) { + return source.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(^|[^:])\/\/.*$/gm, "$1"); +} + +function resolveLocalFile(fromFile, specifier, distRoot) { + if (!specifier.startsWith(".")) return null; + const base = resolve(dirname(fromFile), specifier); + const candidates = [base]; + if (!/\.(?:js|mjs|cjs|json|node)$/.test(base)) { + candidates.push(`${base}.js`, `${base}.mjs`, `${base}.cjs`, `${base}.json`, join(base, "index.js")); + } + for (const candidate of candidates) { + if (!isInside(distRoot, candidate) || !existsSync(candidate)) continue; + if (lstatSync(candidate).isFile()) return candidate; + } + return null; +} + +export function traceLocalAndExternalImports(entryFile, distRoot) { + const resolvedDistRoot = resolve(distRoot); + const visitedFiles = new Set(); + const externalPackages = new Set(); + const queue = [resolve(entryFile)]; + const importPattern = /(?:from\s+|import\s*\(\s*|require\s*\(\s*|import\s+)["']([^"']+)["']/g; + + while (queue.length > 0) { + const currentFile = queue.shift(); + if (!currentFile || visitedFiles.has(currentFile) || !existsSync(currentFile)) continue; + visitedFiles.add(currentFile); + const source = stripComments(readFileSync(currentFile, "utf8")); + for (const match of source.matchAll(importPattern)) { + const specifier = match[1]; + if (specifier.startsWith(".")) { + const local = resolveLocalFile(currentFile, specifier, resolvedDistRoot); + if (local) queue.push(local); + } else { + const pkg = barePackageName(specifier); + if (pkg) externalPackages.add(pkg); + } + } + } + + return { + localFiles: [...visitedFiles].map((file) => relative(resolvedDistRoot, file)).sort(), + externalPackages: [...externalPackages].sort() + }; +} + +function readManifest(dir) { + const manifestPath = join(dir, "package.json"); + if (!existsSync(manifestPath)) return null; + return JSON.parse(readFileSync(manifestPath, "utf8")); +} + +function productionDependencyNames(manifest) { + if (!manifest) return []; + return Object.keys({ ...(manifest.dependencies ?? {}), ...(manifest.optionalDependencies ?? {}) }); +} + +function resolvePackageDirectory(startDir, sourceRoot, packageName) { + let current = resolve(startDir); + const root = resolve(sourceRoot); + while (isInside(root, current)) { + const candidate = join(current, "node_modules", packageName); + if (existsSync(join(candidate, "package.json"))) return candidate; + if (current === root) break; + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + return null; +} + +export function collectProductionModuleClosure({ sourceRoot, seedSpecs, excludePackages = new Set(), targetArch = "amd64" }) { + const resolvedRoot = resolve(sourceRoot); + const visited = new Map(); + const queue = []; + for (const { fromDir, packageName } of seedSpecs) { + if (excludePackages.has(packageName) || packageName.startsWith("@space/") || !shouldKeepOptionalPackage(packageName, targetArch)) { + continue; + } + const pkgDir = resolvePackageDirectory(fromDir, resolvedRoot, packageName); + if (pkgDir) queue.push(pkgDir); + } + + while (queue.length > 0) { + const pkgDir = queue.shift(); + const realDir = resolve(pkgDir); + if (!isInside(resolvedRoot, realDir) || visited.has(realDir) || !existsSync(join(realDir, "package.json"))) continue; + visited.set(realDir, relative(resolvedRoot, realDir).replaceAll("\\", "/")); + for (const depName of productionDependencyNames(readManifest(realDir))) { + if (excludePackages.has(depName) || depName.startsWith("@space/") || !shouldKeepOptionalPackage(depName, targetArch)) continue; + const nextDir = resolvePackageDirectory(realDir, resolvedRoot, depName); + if (nextDir) queue.push(nextDir); + } + } + return [...visited.values()].sort(); +} + +function copyTree(src, dest, allow) { + if (!existsSync(src)) return; + const stat = lstatSync(src); + const entry = { name: basename(src), isDirectory: () => stat.isDirectory() }; + if (allow && !allow(src, entry)) return; + if (stat.isSymbolicLink()) { + const link = readlinkSync(src); + if (isAbsolute(link)) return; + mkdirSync(dirname(dest), { recursive: true }); + rmSync(dest, { recursive: true, force: true }); + symlinkSync(link, dest); + return; + } + if (stat.isDirectory()) { + mkdirSync(dest, { recursive: true }); + for (const child of readdirSync(src, { withFileTypes: true })) { + copyTree(join(src, child.name), join(dest, child.name), allow); + } + return; + } + if (stat.isFile()) { + mkdirSync(dirname(dest), { recursive: true }); + cpSync(src, dest); + } +} + +function runtimeFilter(sourceRoot, targetArch) { + const root = resolve(sourceRoot); + return (srcPath, entry) => { + if (entry.name === "node_modules" && entry.isDirectory() && relative(root, srcPath).startsWith(`..`)) return false; + const rel = relative(root, srcPath); + if (!rel || rel.startsWith("..")) return true; + return shouldKeepRuntimePath(rel, entry.isDirectory(), targetArch); + }; +} + +function copyPackageClosure(sourceRoot, targetRoot, moduleRelativePaths, targetArch) { + const allow = runtimeFilter(sourceRoot, targetArch); + for (const relPath of moduleRelativePaths) { + const src = join(sourceRoot, relPath); + const dest = join(targetRoot, relPath); + // npm usually materializes packages as real directories. If a package path is a + // symlink, copy the directory it points at so the image does not keep a link + // back into the build tree. + if (existsSync(src) && lstatSync(src).isSymbolicLink()) { + let realSrc; + try { + realSrc = realpathSync(src); + } catch { + continue; + } + if (!isInside(resolve(sourceRoot), realSrc)) continue; + copyTree(realSrc, dest, allow); + continue; + } + copyTree(src, dest, allow); + } +} + +function linkWorkspacePackages(targetRoot, workspaceDirs) { + const scopeDir = join(targetRoot, "node_modules", "@space"); + mkdirSync(scopeDir, { recursive: true }); + for (const wsRel of workspaceDirs) { + const manifest = readManifest(join(targetRoot, wsRel)); + if (!manifest?.name?.startsWith("@space/")) continue; + const linkPath = join(scopeDir, manifest.name.slice("@space/".length)); + rmSync(linkPath, { recursive: true, force: true }); + symlinkSync(relative(scopeDir, join(targetRoot, wsRel)), linkPath); + } +} + +function copyRootManifests(sourceRoot, targetRoot) { + mkdirSync(targetRoot, { recursive: true }); + for (const file of ["package.json", "package-lock.json"]) { + const src = join(sourceRoot, file); + if (existsSync(src)) cpSync(src, join(targetRoot, file)); + } +} + +function copyWorkspaceRuntime(sourceRoot, targetRoot, wsRel) { + const srcWs = join(sourceRoot, wsRel); + const destWs = join(targetRoot, wsRel); + const manifestPath = join(srcWs, "package.json"); + if (!existsSync(manifestPath)) return false; + mkdirSync(destWs, { recursive: true }); + cpSync(manifestPath, join(destWs, "package.json")); + copyTree(join(srcWs, "dist"), join(destWs, "dist"), (_src, entry) => { + if (entry.isDirectory()) return entry.name !== "__tests__" && entry.name !== "tests"; + return !entry.name.endsWith(".d.ts") && !entry.name.endsWith(".map") && !entry.name.endsWith(".test.js"); + }); + for (const asset of WORKSPACE_RUNTIME_ASSETS[wsRel] ?? []) { + copyTree(join(srcWs, asset), join(destWs, asset)); + } + return true; +} + +function copyScripts(sourceRoot, targetRoot, scriptRels) { + for (const scriptRel of scriptRels) { + const src = join(sourceRoot, scriptRel); + if (!existsSync(src)) continue; + const dest = join(targetRoot, scriptRel); + mkdirSync(dirname(dest), { recursive: true }); + cpSync(src, dest); + } +} + +function dependencySeeds(sourceRoot, workspaceRels, excludePackages) { + const seeds = []; + for (const wsRel of workspaceRels) { + const wsDir = join(sourceRoot, wsRel); + for (const packageName of productionDependencyNames(readManifest(wsDir))) { + if (excludePackages.has(packageName)) continue; + seeds.push({ fromDir: wsDir, packageName }); + } + } + return seeds; +} + +function assertOutputOutsideSource(sourceRoot, targetRoot) { + if (isInside(resolve(sourceRoot), resolve(targetRoot))) { + throw new Error(`Refusing to stage ${targetRoot} inside source tree ${sourceRoot}.`); + } +} + +export function stageCoreTarget(sourceRoot, targetRoot, targetArch) { + assertOutputOutsideSource(sourceRoot, targetRoot); + rmSync(targetRoot, { recursive: true, force: true }); + copyRootManifests(sourceRoot, targetRoot); + const copied = []; + for (const wsRel of CORE_RUNTIME_WORKSPACES) { + if (copyWorkspaceRuntime(sourceRoot, targetRoot, wsRel)) copied.push(wsRel); + } + const webSrc = join(sourceRoot, "apps", "web"); + const webDest = join(targetRoot, "apps", "web"); + if (existsSync(join(webSrc, "server.mjs"))) { + mkdirSync(webDest, { recursive: true }); + if (existsSync(join(webSrc, "package.json"))) cpSync(join(webSrc, "package.json"), join(webDest, "package.json")); + cpSync(join(webSrc, "server.mjs"), join(webDest, "server.mjs")); + copyTree(join(webSrc, "dist"), join(webDest, "dist")); + } + // The API serves built-in skills from /app/agent-skills. The CLI image gets its own copy. + copyTree(join(sourceRoot, "agent-skills"), join(targetRoot, "agent-skills"), (_src, entry) => entry.name !== "node_modules"); + copyScripts(sourceRoot, targetRoot, CORE_RUNTIME_SCRIPTS); + const excludePackages = new Set(["node-pty"]); + const closure = collectProductionModuleClosure({ + sourceRoot, + seedSpecs: dependencySeeds(sourceRoot, CORE_RUNTIME_WORKSPACES, excludePackages), + excludePackages, + targetArch + }); + copyPackageClosure(sourceRoot, targetRoot, closure, targetArch); + linkWorkspacePackages(targetRoot, copied); + return { workspaces: copied, closureCount: closure.length }; +} + +export function stageBrowserTarget(sourceRoot, targetRoot, targetArch) { + assertOutputOutsideSource(sourceRoot, targetRoot); + rmSync(targetRoot, { recursive: true, force: true }); + copyRootManifests(sourceRoot, targetRoot); + const apiDistRoot = join(sourceRoot, "apps", "api", "dist"); + const entryFile = join(apiDistRoot, "browser-host-main.js"); + const traced = existsSync(entryFile) + ? traceLocalAndExternalImports(entryFile, apiDistRoot) + : { localFiles: [], externalPackages: [] }; + const destApiDir = join(targetRoot, "apps", "api"); + if (existsSync(join(sourceRoot, "apps", "api", "package.json"))) { + mkdirSync(join(destApiDir, "dist"), { recursive: true }); + cpSync(join(sourceRoot, "apps", "api", "package.json"), join(destApiDir, "package.json")); + for (const relFile of traced.localFiles) { + const destFile = join(destApiDir, "dist", relFile); + mkdirSync(dirname(destFile), { recursive: true }); + cpSync(join(apiDistRoot, relFile), destFile); + } + } + + const spaceWorkspaces = []; + const seedSpecs = []; + const wsQueue = traced.externalPackages.filter((pkg) => pkg.startsWith("@space/")); + for (const extPkg of traced.externalPackages) { + if (!extPkg.startsWith("@space/")) seedSpecs.push({ fromDir: join(sourceRoot, "apps", "api"), packageName: extPkg }); + } + const seen = new Set(); + while (wsQueue.length > 0) { + const pkgName = wsQueue.shift(); + const wsRel = `packages/${pkgName.slice("@space/".length)}`; + if (seen.has(wsRel)) continue; + seen.add(wsRel); + if (!copyWorkspaceRuntime(sourceRoot, targetRoot, wsRel)) continue; + spaceWorkspaces.push(wsRel); + for (const depName of productionDependencyNames(readManifest(join(sourceRoot, wsRel)))) { + if (depName.startsWith("@space/")) wsQueue.push(depName); + else seedSpecs.push({ fromDir: join(sourceRoot, wsRel), packageName: depName }); + } + } + const excludePackages = new Set(["node-pty"]); + const closure = collectProductionModuleClosure({ sourceRoot, seedSpecs, excludePackages, targetArch }); + copyPackageClosure(sourceRoot, targetRoot, closure, targetArch); + linkWorkspacePackages(targetRoot, spaceWorkspaces); + return { localFiles: traced.localFiles, workspaces: spaceWorkspaces.sort(), closureCount: closure.length }; +} + +export function stageCliTarget(sourceRoot, targetRoot, targetArch) { + assertOutputOutsideSource(sourceRoot, targetRoot); + rmSync(targetRoot, { recursive: true, force: true }); + copyRootManifests(sourceRoot, targetRoot); + const copied = copyWorkspaceRuntime(sourceRoot, targetRoot, "packages/cli-host") ? ["packages/cli-host"] : []; + copyTree(join(sourceRoot, "agent-skills"), join(targetRoot, "agent-skills"), (_src, entry) => entry.name !== "node_modules"); + const cliDir = join(sourceRoot, "packages", "cli-host"); + const closure = collectProductionModuleClosure({ + sourceRoot, + seedSpecs: dependencySeeds(sourceRoot, ["packages/cli-host"], new Set()), + excludePackages: new Set(), + targetArch + }); + // cli-host's only production dependency is node-pty; still walk whatever the manifest declares. + if (existsSync(cliDir)) { + copyPackageClosure(sourceRoot, targetRoot, closure, targetArch); + } + linkWorkspacePackages(targetRoot, copied); + return { workspaces: copied, closureCount: closure.length }; +} + +export function stageAllContainerTargets({ sourceRoot, outputRoot, arch = process.arch }) { + const targetArch = normalizeTargetArch(arch); + const resolvedSource = resolve(sourceRoot); + const resolvedOutput = resolve(outputRoot); + assertOutputOutsideSource(resolvedSource, resolvedOutput); + mkdirSync(resolvedOutput, { recursive: true }); + return { + arch: targetArch, + core: stageCoreTarget(resolvedSource, join(resolvedOutput, "core"), targetArch), + browser: stageBrowserTarget(resolvedSource, join(resolvedOutput, "browser"), targetArch), + cli: stageCliTarget(resolvedSource, join(resolvedOutput, "cli"), targetArch) + }; +} + +function parseCliArgs(argv) { + let sourceRoot = process.cwd(); + let outputRoot = "/app-targets"; + let arch = process.env.TARGETARCH || process.arch; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--source") { + sourceRoot = argv[index + 1] ?? ""; + index += 1; + } else if (arg === "--output") { + outputRoot = argv[index + 1] ?? ""; + index += 1; + } else if (arg === "--arch") { + arch = argv[index + 1] || process.arch; + index += 1; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!sourceRoot || !outputRoot) throw new Error("Usage: stage-container-runtime.mjs --source --output [--arch amd64|arm64]"); + return { sourceRoot, outputRoot, arch }; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + const summary = stageAllContainerTargets(parseCliArgs(process.argv.slice(2))); + process.stdout.write(`${JSON.stringify(summary)}\n`); +} diff --git a/package-lock.json b/package-lock.json index 04bf09e25..3e49a9509 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,7 +72,6 @@ "mammoth": "^1.12.0", "mysql2": "^3.24.5", "nanoid": "^5.1.6", - "node-pty": "^1.1.0", "pdfjs-dist": "6.3.289", "sanitize-html": "^2.18.1", "zod": "^4.6.7" diff --git a/packages/cli-host/src/node-pty-spawn.ts b/packages/cli-host/src/node-pty-spawn.ts index 47f18bfad..041f1107b 100644 --- a/packages/cli-host/src/node-pty-spawn.ts +++ b/packages/cli-host/src/node-pty-spawn.ts @@ -1,6 +1,19 @@ -import { spawn } from "node-pty"; +import { createRequire } from "node:module"; import type { CliHostSpawn } from "./types.js"; +const require = createRequire(import.meta.url); +type NodePtyModule = typeof import("node-pty"); +let cachedNodePty: NodePtyModule | null = null; + +// The core image imports CliHostClient from this package. Loading node-pty here +// would make that import fail unless core also shipped the native module. +function loadNodePty(): NodePtyModule { + if (!cachedNodePty) { + cachedNodePty = require("node-pty") as NodePtyModule; + } + return cachedNodePty; +} + /** * Disable the PTY interrupt characters (ISIG) for every CLI pane so that * Ctrl-C is delivered to the CLI as the byte 0x03 (its own interrupt/cancel @@ -9,7 +22,7 @@ import type { CliHostSpawn } from "./types.js"; * and app-level exits (Ctrl-D, /exit) are unaffected. */ export const spawnNodePty: CliHostSpawn = (spec) => - spawn("/bin/sh", ["-c", '/usr/bin/stty -isig; exec "$@"', "space-cli-host", spec.command, ...spec.args], { + loadNodePty().spawn("/bin/sh", ["-c", '/usr/bin/stty -isig; exec "$@"', "space-cli-host", spec.command, ...spec.args], { name: "xterm-256color", cols: spec.cols, rows: spec.rows, diff --git a/scripts/container-image-size-budget.mjs b/scripts/container-image-size-budget.mjs index 978e1262a..0f8925847 100644 --- a/scripts/container-image-size-budget.mjs +++ b/scripts/container-image-size-budget.mjs @@ -4,19 +4,15 @@ import { pathToFileURL } from "node:url"; const mebibyte = 1024 * 1024; -// 1.1.3 raised core and browser (998.7 / 1670.6 MiB measured): the current -// application adds production dependencies pdfjs-dist with @napi-rs/canvas -// (81 MiB, API PDF handling), typescript (23 MiB, peer of @t3-oss/env-core) and -// js-tiktoken (22 MiB, Space Gateway token counting). -// 1.1.5 raised core again: linux/arm64 measured 1066.9 MiB and linux/amd64 1045.0 MiB (sqlite3 in the image, -// the 1.1.5 application); the first public release run failed the arm64 build at 1050 MiB. -// 1.1.8 raised core to 1150 MiB for Python with sqlite3 (the Databases tool's SQLite worker): linux/amd64 measured -// 1090.6 MiB with python3 (rc3 without Python: 1057.9 MiB); the trimmed python3-minimal + libpython3-stdlib variant -// measured 1078.6 MiB, 21.4 MiB under the old 1100 MiB, and arm64 ran about 22 MiB above amd64 in 1.1.5. +// Each image now ships only the workspaces and production node_modules its +// entrypoint runs (issue #64), instead of the full monorepo tree that made the +// 1.1.8 budgets 1150 / 1750 / 2800 MiB. These ceilings are the acceptance +// limits for both linux/amd64 and linux/arm64. `containers.yml` prints the +// measured size next to the budget on every build. export const containerImageSizeBudgets = Object.freeze({ - core: 1150 * mebibyte, - browser: 1750 * mebibyte, - cli: 2800 * mebibyte + core: 850 * mebibyte, + browser: 1350 * mebibyte, + cli: 1000 * mebibyte }); export function evaluateContainerImageSize({ target, sizeBytes }) { diff --git a/scripts/tests/container-image-size-budget.test.mjs b/scripts/tests/container-image-size-budget.test.mjs index df951a364..e3fd9f93d 100644 --- a/scripts/tests/container-image-size-budget.test.mjs +++ b/scripts/tests/container-image-size-budget.test.mjs @@ -9,9 +9,9 @@ const mebibyte = 1024 * 1024; test("public container size budgets preserve the lightweight release ceilings", () => { assert.deepEqual(containerImageSizeBudgets, { - core: 1150 * mebibyte, - browser: 1750 * mebibyte, - cli: 2800 * mebibyte + core: 850 * mebibyte, + browser: 1350 * mebibyte, + cli: 1000 * mebibyte }); }); diff --git a/scripts/tests/public-docker-distribution.test.mjs b/scripts/tests/public-docker-distribution.test.mjs index 444f944f8..4c91c0963 100644 --- a/scripts/tests/public-docker-distribution.test.mjs +++ b/scripts/tests/public-docker-distribution.test.mjs @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { lstat, mkdtemp, readFile, readlink, rm, writeFile } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, readFile, readlink, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import test from "node:test"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); @@ -95,7 +95,7 @@ test("runtime images patch fixable vulnerabilities in npm's bundled dependencies assert.match( normalizedRuntimeStage, - /npm pack --ignore-scripts --no-audit --no-fund --pack-destination \/tmp\/npm-security-patches brace-expansion@5\.0\.12 tar@7\.5\.21 ip-address@10\.7\.4 undici@6\.28\.1 postcss-selector-parser@7\.1\.6/ + /npm pack --ignore-scripts --no-audit --no-fund --pack-destination \/tmp\/npm-security-patches brace-expansion@5\.0\.12 tar@7\.5\.21 ip-address@10\.7\.4 undici@6\.29\.1 postcss-selector-parser@7\.1\.6/ ); assert.match( normalizedRuntimeStage, @@ -115,7 +115,7 @@ test("runtime images patch fixable vulnerabilities in npm's bundled dependencies ); assert.match( normalizedRuntimeStage, - /tar -xzf \/tmp\/npm-security-patches\/undici-6\.28\.1\.tgz -C \/usr\/local\/lib\/node_modules\/npm\/node_modules\/undici --strip-components=1/ + /tar -xzf \/tmp\/npm-security-patches\/undici-6\.29\.1\.tgz -C \/usr\/local\/lib\/node_modules\/npm\/node_modules\/undici --strip-components=1/ ); assert.match( normalizedRuntimeStage, @@ -348,3 +348,243 @@ test("core seeds generic starter memory only when mutable owner memory is absent assert.match(starter, /^# SpaceApp Memory/m); assert.doesNotMatch(starter, /\/etc\/docs|\/srv\/space|spaceapp-user|oll4\.com/i); }); + +test("Dockerfile stages per-service runtime trees instead of shipping the full monorepo in runtime-base", async () => { + const dockerfile = await readFile(join(root, "Dockerfile"), "utf8"); + const stages = Object.fromEntries( + dockerfile.split(/^(?=FROM )/m).map((stage) => [stage.match(/^FROM \S+ AS (\S+)/)?.[1], stage]) + ); + + assert.match( + stages.build, + /node deploy\/docker\/stage-container-runtime\.mjs --source \/app --output \/app-targets --arch "\$\{TARGETARCH:-\}"/ + ); + for (const forbidden of [ + "COPY --from=build --chown=spaceapp:spaceapp /app/node_modules", + "COPY --from=build --chown=spaceapp:spaceapp /app/apps", + "COPY --from=build --chown=spaceapp:spaceapp /app/packages", + "COPY --from=build --chown=spaceapp:spaceapp /app/agent-skills" + ]) { + assert.equal(stages["runtime-base"].includes(forbidden), false, forbidden); + } + assert.match(stages.core, /COPY --from=build --chown=spaceapp:spaceapp \/app-targets\/core\/ \.\//); + assert.match(stages.browser, /COPY --from=build --chown=spaceapp:spaceapp \/app-targets\/browser\/ \.\//); + assert.match(stages.cli, /COPY --from=build --chown=spaceapp:spaceapp \/app-targets\/cli\/ \.\//); + assert.match(stages["runtime-base"], /useradd --uid 10001 --gid 10001/); + assert.match(stages.core, /ENTRYPOINT \["\/app\/deploy\/docker\/core-entrypoint\.sh"\]/); + assert.match(stages.browser, /ENTRYPOINT \["\/app\/deploy\/docker\/browser-entrypoint\.sh"\]/); + assert.match(stages.cli, /ENTRYPOINT \["\/app\/deploy\/docker\/cli-entrypoint\.sh"\]/); +}); + +test("core can import the CLI host client without shipping node-pty", async () => { + const spawnSource = await readFile(join(root, "packages/cli-host/src/node-pty-spawn.ts"), "utf8"); + const apiManifest = JSON.parse(await readFile(join(root, "apps/api/package.json"), "utf8")); + + assert.match(spawnSource, /createRequire\(import\.meta\.url\)/); + assert.match(spawnSource, /require\("node-pty"\)/); + assert.doesNotMatch(spawnSource, /^import\s+\{[^}]*\}\s+from\s+"node-pty"/m); + assert.equal(apiManifest.dependencies["node-pty"], undefined); +}); + +test("staged runtime trees keep only the modules and native builds each target runs", async () => { + const { stageAllContainerTargets } = await import( + pathToFileURL(join(root, "deploy/docker/stage-container-runtime.mjs")).href + ); + const source = await mkdtemp(join(tmpdir(), "spaceapp-stage-src-")); + const output = await mkdtemp(join(tmpdir(), "spaceapp-stage-out-")); + const write = async (rel, contents) => { + await mkdir(join(source, dirname(rel)), { recursive: true }); + await writeFile(join(source, rel), contents); + }; + const pkg = (name, dependencies = {}, optionalDependencies = {}) => + JSON.stringify({ name, version: "1.0.0", dependencies, optionalDependencies }); + + try { + await write("package.json", JSON.stringify({ name: "space", private: true })); + await write("package-lock.json", "{}\n"); + await write("apps/api/package.json", pkg("@space/api", { + "@space/db": "0.2.5", + "@space/cli-host": "0.2.5", + nanoid: "1.0.0", + "node-pty": "1.1.0", + "@temporalio/client": "1.24.0", + "pdfjs-dist": "6.3.289" + })); + await write("apps/api/dist/browser-host-main.js", "import { helper } from \"./lib/helper.js\";\nimport \"@space/db\";\nimport \"nanoid\";\nexport const main = helper;\n"); + await write("apps/api/dist/lib/helper.js", "export const helper = 1;\n"); + await write("apps/api/dist/server.js", "import \"@temporalio/client\";\nimport \"pdfjs-dist/legacy/build/pdf.mjs\";\n"); + await write("apps/web/package.json", pkg("@space/web")); + await write("apps/web/server.mjs", "export const web = 1;\n"); + await write("apps/web/dist/index.html", "

web

\n"); + await write("packages/db/package.json", pkg("@space/db", { pg: "8.0.0" })); + await write("packages/db/dist/index.js", "export const db = 1;\n"); + await write("packages/db/dist/migrate.js", "export const migrate = 1;\n"); + await write("packages/db/migrations/0001_initial.sql", "SELECT 1;\n"); + await write("packages/cli-host/package.json", pkg("@space/cli-host", { "node-pty": "1.1.0" })); + await write("packages/cli-host/dist/main.js", "export const main = 1;\n"); + await write("packages/cli-host/dist/index.d.ts", "export {};\n"); + await write("agent-skills/demo/SKILL.md", "# demo\n"); + await write("scripts/portable-backup.mjs", "export {};\n"); + await write("scripts/reset-owner-password.mjs", "export {};\n"); + for (const ws of [ + "apps/worker", "packages/browser-host", "packages/codex-app-server", "packages/contracts", + "packages/gateway", "packages/memory-graph", "packages/opencode-control", "packages/runtime", "packages/streaming" + ]) { + await write(`${ws}/package.json`, pkg(`@space/${ws.split("/")[1]}`)); + await write(`${ws}/dist/index.js`, "export {};\n"); + } + await write("packages/gateway/grammars/tree-sitter-python.wasm", "wasm"); + await write("packages/gateway/src/providers/local/space-local-embeddings.py", "print('ok')\n"); + await write("node_modules/nanoid/package.json", pkg("nanoid")); + await write("node_modules/nanoid/index.js", "export const id = 1;\n"); + await write("node_modules/pg/package.json", pkg("pg")); + await write("node_modules/pg/index.js", "export const pg = 1;\n"); + await write("node_modules/pdfjs-dist/package.json", pkg("pdfjs-dist", {}, { "@napi-rs/canvas": "1.0.9" })); + await write("node_modules/pdfjs-dist/legacy/build/pdf.mjs", "export const pdf = 1;\n"); + await write("node_modules/@napi-rs/canvas/package.json", pkg("@napi-rs/canvas", {}, { + "@napi-rs/canvas-linux-x64-gnu": "1.0.9", + "@napi-rs/canvas-linux-arm64-gnu": "1.0.9", + "@napi-rs/canvas-darwin-arm64": "1.0.9", + "@napi-rs/canvas-linux-x64-musl": "1.0.9" + })); + await write("node_modules/@napi-rs/canvas/index.js", "export {};\n"); + await write("node_modules/@napi-rs/canvas-linux-x64-gnu/package.json", pkg("@napi-rs/canvas-linux-x64-gnu")); + await write("node_modules/@napi-rs/canvas-linux-x64-gnu/skia.linux-x64-gnu.node", "linux-x64"); + await write("node_modules/@napi-rs/canvas-linux-arm64-gnu/package.json", pkg("@napi-rs/canvas-linux-arm64-gnu")); + await write("node_modules/@napi-rs/canvas-linux-arm64-gnu/skia.linux-arm64-gnu.node", "linux-arm64"); + await write("node_modules/@napi-rs/canvas-darwin-arm64/package.json", pkg("@napi-rs/canvas-darwin-arm64")); + await write("node_modules/@napi-rs/canvas-darwin-arm64/skia.node", "darwin"); + await write("node_modules/@napi-rs/canvas-linux-x64-musl/package.json", pkg("@napi-rs/canvas-linux-x64-musl")); + await write("node_modules/@napi-rs/canvas-linux-x64-musl/skia.node", "musl"); + await write("node_modules/@temporalio/client/package.json", pkg("@temporalio/client", { "@temporalio/core-bridge": "1.24.0" })); + await write("node_modules/@temporalio/client/index.js", "export {};\n"); + await write("node_modules/@temporalio/core-bridge/package.json", pkg("@temporalio/core-bridge")); + await write("node_modules/@temporalio/core-bridge/releases/x86_64-unknown-linux-gnu/bridge.node", "amd64"); + await write("node_modules/@temporalio/core-bridge/releases/aarch64-unknown-linux-gnu/bridge.node", "arm64"); + await write("node_modules/@temporalio/core-bridge/releases/x86_64-apple-darwin/bridge.node", "darwin"); + await write("node_modules/@temporalio/core-bridge/sdk-core/lib.rs", "fn main() {}\n"); + await write("node_modules/node-pty/package.json", pkg("node-pty", { "node-addon-api": "7.0.0" })); + await write("node_modules/node-pty/lib/index.js", "exports.spawn = () => {};\n"); + await write("node_modules/node-pty/lib/index.js.map", "map"); + await write("node_modules/node-pty/build/Release/pty.node", "pty"); + await write("node_modules/node-pty/build/Release/spawn-helper", "helper"); + await write("node_modules/node-pty/prebuilds/linux-x64/pty.node", "linux-x64"); + await write("node_modules/node-pty/prebuilds/linux-arm64/pty.node", "linux-arm64"); + await write("node_modules/node-pty/prebuilds/darwin-arm64/pty.node", "darwin"); + await write("node_modules/node-pty/third_party/conpty/OpenConsole.exe", "conpty"); + await write("node_modules/node-addon-api/package.json", pkg("node-addon-api")); + await write("node_modules/node-addon-api/index.js", "exports.ok = 1;\n"); + + assert.throws( + () => stageAllContainerTargets({ sourceRoot: source, outputRoot: join(source, "inside"), arch: "amd64" }), + /inside source/ + ); + + stageAllContainerTargets({ sourceRoot: source, outputRoot: output, arch: "amd64" }); + const core = join(output, "core"); + const browser = join(output, "browser"); + const cli = join(output, "cli"); + const present = (path) => readFile(path, "utf8"); + const absent = async (path) => { + await assert.rejects(readFile(path, "utf8"), { code: "ENOENT" }); + }; + + await present(join(core, "packages/db/migrations/0001_initial.sql")); + await present(join(core, "packages/db/dist/migrate.js")); + await present(join(core, "apps/api/dist/server.js")); + await present(join(core, "apps/web/server.mjs")); + await present(join(core, "apps/web/dist/index.html")); + await present(join(core, "scripts/portable-backup.mjs")); + await present(join(core, "scripts/reset-owner-password.mjs")); + await present(join(core, "agent-skills/demo/SKILL.md")); + await present(join(core, "packages/gateway/grammars/tree-sitter-python.wasm")); + await present(join(core, "packages/gateway/src/providers/local/space-local-embeddings.py")); + await present(join(core, "node_modules/@temporalio/core-bridge/releases/x86_64-unknown-linux-gnu/bridge.node")); + await present(join(core, "node_modules/@napi-rs/canvas-linux-x64-gnu/skia.linux-x64-gnu.node")); + await present(join(core, "node_modules/pdfjs-dist/legacy/build/pdf.mjs")); + await absent(join(core, "node_modules/node-pty/lib/index.js")); + await absent(join(core, "node_modules/@temporalio/core-bridge/releases/aarch64-unknown-linux-gnu/bridge.node")); + await absent(join(core, "node_modules/@temporalio/core-bridge/releases/x86_64-apple-darwin/bridge.node")); + await absent(join(core, "node_modules/@temporalio/core-bridge/sdk-core/lib.rs")); + await absent(join(core, "node_modules/@napi-rs/canvas-linux-arm64-gnu/skia.linux-arm64-gnu.node")); + await absent(join(core, "node_modules/@napi-rs/canvas-darwin-arm64/skia.node")); + await absent(join(core, "node_modules/@napi-rs/canvas-linux-x64-musl/skia.node")); + assert.match(await readlink(join(core, "node_modules/@space/db")), /\.\.\/\.\.\/packages\/db$/); + + await present(join(browser, "apps/api/dist/browser-host-main.js")); + await present(join(browser, "apps/api/dist/lib/helper.js")); + await present(join(browser, "node_modules/nanoid/index.js")); + await present(join(browser, "node_modules/pg/index.js")); + await present(join(browser, "packages/db/migrations/0001_initial.sql")); + await absent(join(browser, "apps/api/dist/server.js")); + await absent(join(browser, "node_modules/@temporalio/client/index.js")); + await absent(join(browser, "node_modules/pdfjs-dist/legacy/build/pdf.mjs")); + await absent(join(browser, "node_modules/node-pty/lib/index.js")); + await absent(join(browser, "apps/web/server.mjs")); + + await present(join(cli, "packages/cli-host/dist/main.js")); + await present(join(cli, "node_modules/node-pty/build/Release/pty.node")); + await present(join(cli, "node_modules/node-pty/build/Release/spawn-helper")); + await present(join(cli, "node_modules/node-pty/prebuilds/linux-x64/pty.node")); + await present(join(cli, "node_modules/node-pty/lib/index.js")); + await present(join(cli, "node_modules/node-addon-api/index.js")); + await present(join(cli, "agent-skills/demo/SKILL.md")); + await absent(join(cli, "packages/cli-host/dist/index.d.ts")); + await absent(join(cli, "node_modules/node-pty/lib/index.js.map")); + await absent(join(cli, "node_modules/node-pty/prebuilds/linux-arm64/pty.node")); + await absent(join(cli, "node_modules/node-pty/prebuilds/darwin-arm64/pty.node")); + await absent(join(cli, "node_modules/node-pty/third_party/conpty/OpenConsole.exe")); + await absent(join(cli, "apps/api/dist/server.js")); + await absent(join(cli, "node_modules/@temporalio/client/index.js")); + } finally { + await rm(source, { recursive: true, force: true }); + await rm(output, { recursive: true, force: true }); + } +}); + +test("container smoke plans follow each image entrypoint", async () => { + const { buildSmokePlan, parseSmokeArgs, runContainerSmoke, CLI_HOST_SOCKET, BROWSER_HOST_SOCKET, CHROMIUM_PROBE } = await import( + pathToFileURL(join(root, "deploy/docker/smoke-container-image.mjs")).href + ); + + assert.deepEqual( + parseSmokeArgs(["--target", "core", "--image", "spaceapp-ci:core-amd64", "--arch", "amd64"]), + { target: "core", image: "spaceapp-ci:core-amd64", arch: "amd64" } + ); + assert.throws(() => parseSmokeArgs(["--target", "desktop", "--image", "x"]), /Unsupported container target/); + + const corePlan = buildSmokePlan({ target: "core", image: "spaceapp-ci:core-amd64", arch: "amd64", runId: "t1", secretDir: "/secrets" }); + assert.equal(corePlan.requiresPostgres, true); + assert.equal(corePlan.migrateArgs, null); + assert.match(corePlan.readinessProbe, /http:\/\/127\.0\.0\.1:4911\/readyz/); + assert.equal(corePlan.containerEnv.SPACE_RUNTIME_STORE, "postgres"); + assert.equal(corePlan.containerEnv.SPACEAPP_WORKFLOWS_ENABLED, "false"); + assert.ok(corePlan.secretMounts.some((mount) => mount.startsWith("/secrets/database-url:/run/secrets/database-url"))); + assert.ok(corePlan.secretMounts.some((mount) => mount.includes("/run/secrets/setup-token"))); + assert.equal(corePlan.runArgs.at(-1), "spaceapp-ci:core-amd64"); + + const cliPlan = buildSmokePlan({ target: "cli", image: "spaceapp-ci:cli-arm64", arch: "arm64", runId: "t2" }); + assert.equal(cliPlan.requiresPostgres, false); + assert.equal(cliPlan.containerEnv.SPACE_CLI_HOST_SOCKET, CLI_HOST_SOCKET); + assert.match(cliPlan.socketProbe, /\/run\/spaceapp-cli\/host\.sock/); + assert.equal(cliPlan.runArgs.at(-1), "spaceapp-ci:cli-arm64"); + + const browserPlan = buildSmokePlan({ target: "browser", image: "spaceapp-ci:browser-amd64", arch: "amd64", runId: "t3", secretDir: "/secrets" }); + assert.equal(browserPlan.requiresPostgres, true); + assert.equal(browserPlan.containerEnv.SPACE_BROWSER_HOST_SOCKET, BROWSER_HOST_SOCKET); + assert.ok(browserPlan.migrateArgs.includes("packages/db/dist/migrate.js")); + assert.deepEqual(browserPlan.chromiumProbe, CHROMIUM_PROBE); + + const calls = []; + const result = await runContainerSmoke( + { target: "cli", image: "spaceapp-ci:cli-amd64", arch: "amd64" }, + async (cmd, args) => { + calls.push([cmd, ...args]); + return { stdout: "running\n", stderr: "" }; + } + ); + assert.equal(result.ok, true); + assert.ok(calls.some((call) => call[1] === "run" && call.includes("spaceapp-ci:cli-amd64"))); + assert.ok(calls.some((call) => call.includes(`SPACE_CLI_HOST_SOCKET=${CLI_HOST_SOCKET}`))); + assert.ok(calls.some((call) => call[1] === "exec" && call.includes("node"))); +}); diff --git a/scripts/tests/public-workflows.test.mjs b/scripts/tests/public-workflows.test.mjs index 1775371a9..e2c7828f2 100644 --- a/scripts/tests/public-workflows.test.mjs +++ b/scripts/tests/public-workflows.test.mjs @@ -73,6 +73,9 @@ test("container matrices load each image and enforce the public size budgets", a assert.match(content, /node scripts\/container-image-size-budget\.mjs/); assert.match(content, /--target "\$\{\{ matrix\.target \}\}"/); assert.match(content, /--image "spaceapp-ci:\$\{\{ matrix\.target \}\}-\$\{\{ matrix\.arch \}\}"/); + if (name === "containers.yml") { + assert.match(content, /node deploy\/docker\/smoke-container-image\.mjs/); + } } });