diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 9d770c6..410de47 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -1,11 +1,14 @@ +# NOTE: automatic triggering is disabled — see DEPLOYMENT.md "Marketing site". +# The ghost-app Vercel project (VERCEL_PROJECT_ID below) currently has its Root +# Directory overridden to cloud/apps/web, which now serves the live cloud SaaS +# app at ghost.muharafiq.com. Because that override applies regardless of what +# changed in the push, running this workflow as-is would redeploy cloud/apps/web +# again, not public/, no matter what the steps below claim. Do not re-enable the +# push trigger or run this via workflow_dispatch until public/ has its own +# Vercel project (or domain) pointed at this directory, or is retired. name: Deploy Website on: - push: - branches: [main, master] - paths: - - "public/**" - - ".github/workflows/deploy-website.yml" workflow_dispatch: permissions: @@ -13,7 +16,7 @@ permissions: jobs: deploy: - name: Deploy to Vercel (ghost.muharafiq.com) + name: Deploy to Vercel (public/ — target currently misconfigured, see note above) runs-on: ubuntu-latest steps: @@ -62,11 +65,10 @@ jobs: VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} run: | + echo "::warning::VERCEL_PROJECT_ID's Root Directory is overridden to cloud/apps/web, so this will deploy the cloud app, not public/. See the note at the top of this workflow file." echo "Deploying public/ to Vercel…" vercel deploy --prod --token="${VERCEL_TOKEN}" --yes - echo "Live: https://ghost.muharafiq.com" - name: Deployment summary run: | - echo "Deployed $(find public -type f | wc -l) files from public/" - echo "Site: https://ghost.muharafiq.com" + echo "Deployed $(find public -type f | wc -l) files from public/ (see the warning above about where this actually landed)" diff --git a/CLAUDE.md b/CLAUDE.md index 3a8fc3e..46b661e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,8 +106,11 @@ Build the five-part MVP in `cloud/`, in order: Built so far (Phase 1): the execution engine, the deterministic approval gate, per-step screenshots + verification, the hash-chained audit log, the run → approval → verify UI, and the agent HTTP/MCP surface (agents propose; humans -approve — `cloud/docs/AGENT_PLUGIN.md`). Recording (steps 1–2) is next. See -`cloud/docs/PHASE_1_PLAN.md` and `cloud/docs/CURSOR_HANDOFF.md`. +approve — `cloud/docs/AGENT_PLUGIN.md`). Phase 2 (recording → editable steps) is +in progress: the convert side (upload → deterministic compile → review) is +built and off by default; a Chrome extension (`cloud/apps/extension`) captures +the browser session. See `cloud/docs/PHASE_1_PLAN.md` and +`cloud/docs/CURSOR_HANDOFF.md` for current status. Required behavior (unchanged in spirit from the desktop trust pipeline): @@ -117,6 +120,49 @@ Required behavior (unchanged in spirit from the desktop trust pipeline): - verify each step's outcome; - write audit events (hash-chained) for every run and step. +### Cloud workspace layout & commands + +`cloud/` is a self-contained pnpm + Turborepo workspace — it does not share +tooling with the repo root. `cd cloud` before running anything below. + +```text +cloud/ + apps/ + web/ Next.js 15 (App Router) — UI + API + /api/agent/* → deployed to Vercel + worker/ Node worker: BullMQ consumers + Playwright execution → deployed as a container + mcp/ Stdio MCP bridge for Cursor/Claude (no approve tools) + extension/ Chrome extension — records a browser session for Phase 2 capture + packages/ + core/ Prisma schema, Zod step types, classifyStep (approval gate), audit chain, agent catalog +``` + +Quickstart: `cd cloud && pnpm demo` (writes `.env`, brings up Postgres/Redis, +migrates, installs Chromium, starts web + worker — idempotent, safe to rerun). +Manual steps and the two load-bearing env vars (`GHOST_ARTIFACT_DIR` must be an +absolute path shared by web+worker; `GHOST_SESSION_KEY` must decode to 32 bytes) +are in `cloud/README.md`. + +Validation from inside `cloud/`: + +```bash +pnpm typecheck # the real static gate — run this even if lint is clean +pnpm lint # apps/web only; worker/mcp/core have no lint script yet +pnpm test # ~90 of ~239 tests need DATABASE_URL set or they skip silently +pnpm build +``` + +`pnpm test` also needs `REDIS_URL` and `GHOST_SESSION_KEY` set (not just +`DATABASE_URL`) or the DB-gated tests run instead of skipping and fail on +status rather than on anything naming the missing var: without `REDIS_URL`, +rate-limited routes (e.g. invite acceptance) fail closed with 429; without +`GHOST_SESSION_KEY`, the worker skips session capture at an approval gate, so +every gated run refuses to resume and ends `INCIDENT`. Both read like product +bugs and are actually a missing local env var — `.github/workflows/cloud.yml` +sets all three for exactly this reason. + +Don't reach for the root-level `cargo`/`make` commands when working in `cloud/` +— they build the unrelated legacy desktop app. + ## Engineering rules Every meaningful operation should pass through: @@ -211,7 +257,7 @@ Current structure: src/ # Tauri desktop frontend (ES-module JS/HTML/CSS, bundled by Vite; main.js holds most UI logic; compression-review.js/.css is the split-out event-review timeline; src/public/ holds pass-through static assets) apps/macos/ # Ghost 2.0 native macOS app (SwiftUI): App/, Views/, Features/, Services/, RustBridge/, AppKitBridge/ — UI only; all trust decisions stay in the Rust core over a JSON stdin/stdout bridge (docs/legacy/native-macos-preview.md) native/macos/ # GhostAXHelper.swift — read-only macOS Accessibility helper (list_matches op) -public/ # marketing/download site (static vanilla JS with in-browser demos; ships Ghost.dmg / Ghost_Setup.exe under downloads/; auto-deployed to Vercel by deploy-website.yml) +public/ # marketing/download site for the legacy desktop app (static vanilla JS with in-browser demos; ships Ghost.dmg / Ghost_Setup.exe under downloads/); NOT currently deployed anywhere — deploy-website.yml's auto-trigger is disabled because the Vercel project it targets now serves cloud/apps/web at ghost.muharafiq.com instead (see DEPLOYMENT.md) src-tauri/ # Rust backend docs/ # planning and technical docs .github/workflows/ # CI (rust.yml), release (release.yml), site deploy (deploy-website.yml) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 35b61a7..137d940 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -26,12 +26,28 @@ Production sketch: Cloud CI workflow is staged at `cloud/ci/cloud.yml` until installed under `.github/workflows/`. -Details: `cloud/README.md`, `cloud/docs/CURSOR_HANDOFF.md`. +**Live today:** the production domain **`ghost.muharafiq.com`** points at the +`ghost-app` Vercel project with its Root Directory set to `cloud/apps/web` — it +serves the cloud SaaS app directly (deployed manually today via +`vercel --prod --scope muharafiq --cwd cloud/apps/web`, not by a checked-in CI +workflow). This is a deliberate change from the domain's original use as the +static marketing site — see "Marketing site" below. -## Marketing site +Details: `cloud/README.md`, `cloud/docs/CURSOR_HANDOFF.md`. -Static files in `public/` deploy via `.github/workflows/deploy-website.yml` to Vercel. -This is the public marketing surface — keep it aligned with the cloud product. +## Marketing site (currently not deployed anywhere) + +`public/` is a separate static site (vanilla JS, ships the legacy desktop +Ghost.dmg/Ghost_Setup.exe installers) for the superseded desktop product, not +the cloud SaaS. `.github/workflows/deploy-website.yml` still exists to deploy +it, but it targets the same `ghost-app` Vercel project that now serves +`cloud/apps/web` above — because Vercel's Root Directory override applies +regardless of which files actually changed, running this workflow today would +redeploy the cloud app again, not `public/`, despite its build log claiming +otherwise. Its automatic trigger is disabled for that reason (see the workflow +file). Before re-enabling it: either point it at a separate Vercel +project/domain for the legacy site, or retire `public/` outright if the legacy +desktop product no longer needs a public download page. ## Legacy desktop diff --git a/cloud/.env.example b/cloud/.env.example index d334935..240f15d 100644 --- a/cloud/.env.example +++ b/cloud/.env.example @@ -4,7 +4,16 @@ # Redis with the defaults below out of the box. # --------------------------------------------------------------------------- -# Postgres (matches cloud/docker-compose.yml) +# Postgres (matches cloud/docker-compose.yml). +# +# Both apps load THIS file — the workspace root one — via packages/core/src/env.ts. +# +# If 5432 is already taken by an unrelated Postgres (Homebrew, Postgres.app, +# another project), `pnpm demo` moves Ghost's to 55432 and rewrites this line. +# It probes by running a query as the ghost user, because a port that merely +# accepts connections is not evidence: a foreign Postgres answers the socket and +# then denies every query, which surfaces much later as unexplained 500s. +# `pnpm check` reports the same thing without changing anything. DATABASE_URL="postgresql://ghost:ghost@localhost:5432/ghost?schema=public" # Redis / BullMQ (matches cloud/docker-compose.yml) @@ -46,6 +55,14 @@ AUTH_GITHUB_SECRET="" AUTH_GOOGLE_ID="" AUTH_GOOGLE_SECRET="" +# Email magic-link sign-in via Resend (optional in dev, same reason). Needs +# the sending domain verified in Resend (SPF/DKIM DNS records) before real +# emails will deliver — see docs/DEPLOY.md. Setting this also activates the +# Prisma adapter (see auth.ts), which GitHub/Google reuse for account linking. +# Install: https://vercel.com/marketplace/resend, or https://resend.com directly. +RESEND_API_KEY="" +RESEND_EMAIL_DOMAIN="" + # S3-compatible object storage for run screenshots + recording traces. # Works with AWS S3, Cloudflare R2, or MinIO. Not required for local dev. # diff --git a/cloud/README.md b/cloud/README.md index 0db23ab..d86642b 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -90,6 +90,8 @@ cp .env.example .env # a working local config as-is # GHOST_ARTIFACT_DIR -> e.g. $PWD/.artifacts pnpm install # runs `prisma generate` via core postinstall docker compose up -d # Postgres :5432, Redis :6379 + # (set GHOST_PG_PORT / GHOST_REDIS_PORT if + # those ports are already taken) pnpm db:migrate # apply the Prisma schema pnpm --filter @ghost/worker exec playwright install chromium pnpm dev # web on http://localhost:3000 + worker @@ -116,6 +118,34 @@ pnpm --filter @ghost/web dev pnpm --filter @ghost/worker dev ``` +Both read `cloud/.env` directly (`packages/core/src/env.ts`), so either one +works on its own. A real environment variable always wins over the file, and in +deployment there is no `.env` at all. + +## When something is wrong + +```bash +pnpm check +``` + +Read-only; it names the problem rather than leaving you to infer it. Ghost +fails locally in two ways that look like nothing at all: + +- **No worker running.** The UI is fine, "Run" appears to work, and the run sits + there forever, because the process that executes runs is not up. `pnpm dev` + starts both; `pnpm --filter @ghost/worker dev` starts just the worker. +- **`DATABASE_URL` pointing at the wrong Postgres.** A Postgres that is merely + *listening* on 5432 is not Ghost's — Homebrew's, Postgres.app's, another + project's container will all accept the connection and deny the user. `pnpm + demo` probes with real credentials, moves to a free port if it must, and + repairs `.env`. + +A stalled run is no longer permanent either: the worker reclaims runs whose +lease expired (`apps/worker/src/jobs/reclaimRuns.ts`) on boot and every minute, +so a crash or a redeploy mid-run resumes from the journal instead of leaving a +row `RUNNING` forever. After five failed restarts it becomes an `INCIDENT` for a +human, rather than looping. + ## Smoke test (Phase 1) 1. Open http://localhost:3000 and sign in (dev-credentials accepts any email; @@ -144,7 +174,7 @@ and `turbo run lint` skips packages that define no `lint` script — silently, a with a green summary. Treat `typecheck` as the real static gate until the other three packages have configs. -A full green run is **424 tests**. Roughly 90 of them are gated on +A full green run is **430 tests**. Roughly 90 of them are gated on `Boolean(process.env.DATABASE_URL)` and **skip silently** without it — so a green run with no database covers none of the execution engine. If the worker suite reports 45 tests rather than 90, the database is not being reached. diff --git a/cloud/apps/web/.gitignore b/cloud/apps/web/.gitignore index 245259b..d3dd205 100644 --- a/cloud/apps/web/.gitignore +++ b/cloud/apps/web/.gitignore @@ -1,2 +1,8 @@ .vercel .env* + +# Agent-skill docs auto-fetched by `vercel integration add` (Resend usage +# reference for AI assistants). Not application code; safe to re-fetch. +.agents/ +.claude/ +skills-lock.json diff --git a/cloud/apps/web/next.config.ts b/cloud/apps/web/next.config.ts index 045cfbd..320149a 100644 --- a/cloud/apps/web/next.config.ts +++ b/cloud/apps/web/next.config.ts @@ -1,3 +1,7 @@ +// Loads cloud/.env before Next reads anything. Next only looks for .env in its +// own project directory (apps/web), so the workspace-root file the README tells +// people to create was never picked up — see packages/core/src/env.ts. +import "@ghost/core/env"; import type { NextConfig } from "next"; const nextConfig: NextConfig = { diff --git a/cloud/apps/web/package.json b/cloud/apps/web/package.json index 96fdc68..ca5a354 100644 --- a/cloud/apps/web/package.json +++ b/cloud/apps/web/package.json @@ -13,6 +13,7 @@ "test": "vitest run" }, "dependencies": { + "@auth/prisma-adapter": "^2.11.3", "@ghost/core": "workspace:*", "bullmq": "^5.34.4", "clsx": "^2.1.1", diff --git a/cloud/apps/web/src/app/(app)/recordings/page.tsx b/cloud/apps/web/src/app/(app)/recordings/page.tsx index 1236129..27c86f3 100644 --- a/cloud/apps/web/src/app/(app)/recordings/page.tsx +++ b/cloud/apps/web/src/app/(app)/recordings/page.tsx @@ -32,7 +32,7 @@ export default async function RecordingsPage() { return (
-
+

Recordings

@@ -61,8 +61,8 @@ export default async function RecordingsPage() {

{recordings.map((r) => ( - -
+ +
{r.rawTraceFilename ?? r.id} diff --git a/cloud/apps/web/src/app/(app)/workflows/page.tsx b/cloud/apps/web/src/app/(app)/workflows/page.tsx index 96ff2af..3b089ee 100644 --- a/cloud/apps/web/src/app/(app)/workflows/page.tsx +++ b/cloud/apps/web/src/app/(app)/workflows/page.tsx @@ -25,7 +25,7 @@ export default async function WorkflowsPage() { return (
-
+

Workflows

@@ -57,18 +57,18 @@ export default async function WorkflowsPage() {

{workflows.map((w) => ( - -
+ +
{w.name} {w.description && ( -
+
{w.description}
)}
-
+
{w._count.versions} version{w._count.versions === 1 ? "" : "s"} diff --git a/cloud/apps/web/src/app/signin/page.tsx b/cloud/apps/web/src/app/signin/page.tsx index f77fc37..8b43706 100644 --- a/cloud/apps/web/src/app/signin/page.tsx +++ b/cloud/apps/web/src/app/signin/page.tsx @@ -6,14 +6,15 @@ import { SOURCE_URL } from "@/lib/source-url"; const githubEnabled = Boolean(process.env.AUTH_GITHUB_ID && process.env.AUTH_GITHUB_SECRET); const googleEnabled = Boolean(process.env.AUTH_GOOGLE_ID && process.env.AUTH_GOOGLE_SECRET); +const resendEnabled = Boolean(process.env.RESEND_API_KEY && process.env.RESEND_EMAIL_DOMAIN); const devEnabled = process.env.NODE_ENV !== "production"; -// All three false means production with no OAuth provider configured: no +// All four false means production with no sign-in method configured: no // form below has anything to render, and a card with a title and no buttons // looks like a bug rather than a missing deploy step. Whoever hits this is // more likely to be the person standing up the deployment than an end user, // so name the exact fix rather than failing silently. See docs/DEPLOY.md's // "sign-in trap". -const misconfigured = !githubEnabled && !googleEnabled && !devEnabled; +const misconfigured = !githubEnabled && !googleEnabled && !resendEnabled && !devEnabled; export default async function SignInPage({ searchParams, @@ -44,11 +45,11 @@ export default async function SignInPage({ No sign-in method is configured

- This deployment has NODE_ENV=production and no OAuth app - configured, so there is no way to sign in. Set either{" "} - AUTH_GITHUB_ID/AUTH_GITHUB_SECRET or{" "} - AUTH_GOOGLE_ID/AUTH_GOOGLE_SECRET, with the - app's callback at{" "} + This deployment has NODE_ENV=production and no sign-in method + configured. Set AUTH_GITHUB_ID/AUTH_GITHUB_SECRET,{" "} + AUTH_GOOGLE_ID/AUTH_GOOGLE_SECRET, or{" "} + RESEND_API_KEY/RESEND_EMAIL_DOMAIN for email + magic links. OAuth apps need their callback at{" "} https://<this-domain>/api/auth/callback/<github|google>. See docs/DEPLOY.md.

@@ -81,6 +82,35 @@ export default async function SignInPage({ )} + {resendEnabled && ( +
{ + "use server"; + await signIn("resend", { + email: String(formData.get("email") ?? ""), + redirectTo, + }); + }} + className="space-y-3" + > + + +
+ )} + {devEnabled && (
{ diff --git a/cloud/apps/web/src/app/signout/page.tsx b/cloud/apps/web/src/app/signout/page.tsx new file mode 100644 index 0000000..8fcbb16 --- /dev/null +++ b/cloud/apps/web/src/app/signout/page.tsx @@ -0,0 +1,36 @@ +import { signOut } from "@/auth"; +import { Button } from "@/components/ui/button"; +import { Card, CardBody } from "@/components/ui/card"; + +/** + * Auth.js's own `/api/auth/signout` confirmation page renders its + * unstyled built-in theme — the only unbranded surface in an otherwise + * consistently dark-themed app. `pages.signOut` in `@/auth` points here + * instead so a visitor never sees it, even though the real in-app + * sign-out (`components/sign-out-button.tsx`) bypasses this page entirely + * via the same server action. + */ +export default function SignOutPage() { + return ( +
+ + +
+
Ghost
+

Sign out of your workspace.

+
+ { + "use server"; + await signOut({ redirectTo: "/signin" }); + }} + > + + + + +
+ ); +} diff --git a/cloud/apps/web/src/auth-providers.test.ts b/cloud/apps/web/src/auth-providers.test.ts index 9f3f02b..3201f4f 100644 --- a/cloud/apps/web/src/auth-providers.test.ts +++ b/cloud/apps/web/src/auth-providers.test.ts @@ -70,8 +70,12 @@ describe("auth providers", () => { ["GitHub", "AUTH_GITHUB_ID"], ["Google", "AUTH_GOOGLE_ID"], ] as const) { + // `providers.push(GitHub)` (bare reference) and `providers.push(GitHub({ ... }))` + // (invoked with options, e.g. `allowDangerousEmailAccountLinking`) are both a + // provider gated on its own env var — only match on the provider name, not on + // whether it's called. const registration = new RegExp( - `process\\.env\\.${envVar}[\\s\\S]{0,120}?providers\\.push\\(${provider}\\)`, + `process\\.env\\.${envVar}[\\s\\S]{0,120}?providers\\.push\\(\\s*${provider}\\b`, ); expect( registration.test(CODE), diff --git a/cloud/apps/web/src/auth.ts b/cloud/apps/web/src/auth.ts index 80fa368..1d10c97 100644 --- a/cloud/apps/web/src/auth.ts +++ b/cloud/apps/web/src/auth.ts @@ -1,9 +1,12 @@ import { randomUUID } from "node:crypto"; +import { PrismaAdapter } from "@auth/prisma-adapter"; import NextAuth, { type NextAuthConfig } from "next-auth"; import Credentials from "next-auth/providers/credentials"; import GitHub from "next-auth/providers/github"; import Google from "next-auth/providers/google"; +import Resend from "next-auth/providers/resend"; import { assertAuthSecretUsable, devSignInAllowed, sessionMaxAgeSeconds } from "@/lib/auth-env"; +import { prisma } from "@/lib/db"; import { ensureUserOrg } from "@/lib/org"; /** @@ -14,24 +17,52 @@ import { ensureUserOrg } from "@/lib/org"; * (`ensureUserOrg`) and stamp `userId`/`orgId` into the token, so every request * is scoped to a tenant. * - * GitHub and Google OAuth are each enabled only when their own env vars are - * present — a deployment can offer one, both, or neither (falling back to the - * dev-only provider below). Locally, the "Dev sign-in" provider accepts any - * email. + * GitHub, Google, and Resend (email magic link) are each enabled only when + * their own env vars are present — a deployment can offer any combination + * (falling back to the dev-only provider below). Locally, the "Dev sign-in" + * provider accepts any email. */ // Before anything else: in production, refuse to start on a session secret // that cannot protect a session. See lib/auth-env.ts. assertAuthSecretUsable(); +// The Email/magic-link provider is the only one that needs a database +// adapter (Auth.js stores the one-time token via it) — GitHub/Google/dev +// sign-in all work adapter-less today via `ensureUserOrg`'s manual upsert. +// Only wire the adapter in when Resend is actually configured, so a +// deployment without it is byte-for-byte the same as before this provider +// existed. +const resendEnabled = Boolean(process.env.RESEND_API_KEY && process.env.RESEND_EMAIL_DOMAIN); + const providers: NextAuthConfig["providers"] = []; if (process.env.AUTH_GITHUB_ID && process.env.AUTH_GITHUB_SECRET) { - providers.push(GitHub); + providers.push( + GitHub({ + // Once the adapter above is present, Auth.js refuses an OAuth sign-in + // whose email already belongs to a different provider's account, + // unless a provider opts in here. `ensureUserOrg` has always merged + // accounts by email with no such check, so this keeps that existing + // behavior instead of silently locking out a user who e.g. signed up + // with GitHub and later tries Google. Inert (and unread) when the + // adapter isn't wired. + allowDangerousEmailAccountLinking: true, + }), + ); } if (process.env.AUTH_GOOGLE_ID && process.env.AUTH_GOOGLE_SECRET) { - providers.push(Google); + providers.push(Google({ allowDangerousEmailAccountLinking: true })); +} + +if (resendEnabled) { + providers.push( + Resend({ + apiKey: process.env.RESEND_API_KEY, + from: `Ghost `, + }), + ); } // Passwordless "any email" sign-in for local development. `devSignInAllowed` @@ -53,9 +84,12 @@ if (devSignInAllowed()) { } export const authConfig: NextAuthConfig = { + adapter: resendEnabled ? PrismaAdapter(prisma) : undefined, providers, session: { strategy: "jwt", maxAge: sessionMaxAgeSeconds() }, - pages: { signIn: "/signin" }, + // `signOut` keeps `/api/auth/signout` off Auth.js's unstyled built-in + // confirmation page — see app/signout/page.tsx. + pages: { signIn: "/signin", signOut: "/signout" }, callbacks: { async jwt({ token, user }) { // Runs with `user` only on initial sign-in. diff --git a/cloud/apps/web/src/lib/recording-ingest.ts b/cloud/apps/web/src/lib/recording-ingest.ts index 61f8080..e57d004 100644 --- a/cloud/apps/web/src/lib/recording-ingest.ts +++ b/cloud/apps/web/src/lib/recording-ingest.ts @@ -1,137 +1,7 @@ -import { appendAuditEvent } from "@ghost/core/audit-log"; -import { artifactStore } from "@ghost/core/storage/artifacts"; -import { Prisma } from "@ghost/core/db"; -import { compileTrace } from "@ghost/core/recording/compile"; -import { parseRecordingTrace } from "@ghost/core/recording/trace"; -import { prisma } from "@/lib/db"; - /** - * Storing an uploaded recording trace, shared by the browser upload form and - * the extension's ingest endpoint so the two cannot drift on limits, - * sanitisation, or what gets audited. - * - * The interesting part is what happens to a *structured* trace. A Ghost - * recorder reads the accessible role and name off each element while it is - * still on screen, so the trace already contains what the worker's resolution - * chain wants. `compileTrace` then turns it into typed steps deterministically - * — no model, no network, no configured compiler — and the recording lands - * `READY` for review in a single request. - * - * That is what makes recording work in production, where there is deliberately - * no compiler configured at all (see `docs/DEPLOY.md`). Anything that is *not* - * a structured trace — a HAR, a Playwright zip — is stored as before and left - * for whatever compiler exists, if any. + * Re-export: the implementation moved to `@ghost/core/recording/ingest` when + * the worker's remote capture browser needed it too. Kept as a module so the + * web app's existing import sites read naturally. */ - -/** Small event logs, not video. Well under Vercel's 100MB body limit. */ -export const MAX_TRACE_BYTES = 25 * 1024 * 1024; - -export function sanitizeFilename(name: string): string { - const base = name.split(/[/\\]/).pop() || "recording-trace"; - return base.replace(/[^a-zA-Z0-9._-]/g, "_").slice(-200) || "recording-trace"; -} - -export interface IngestResult { - id: string; - /** READY when the trace compiled deterministically, NONE when it awaits a compiler. */ - compileStatus: "READY" | "NONE"; - stepCount: number; - notes: string[]; -} - -export async function ingestTrace(args: { - orgId: string; - userId: string | null; - filename: string; - contentType: string; - buffer: Buffer; -}): Promise { - const { orgId, userId, buffer } = args; - const filename = sanitizeFilename(args.filename); - - const recording = await prisma.recording.create({ - data: { orgId, status: "STOPPED" }, - }); - - try { - const key = `recordings/${recording.id}/trace-${filename}`; - await artifactStore().put(key, buffer, args.contentType || "application/octet-stream"); - - const compiled = tryCompile(buffer); - - await prisma.$transaction(async (tx) => { - await tx.recording.update({ - where: { id: recording.id }, - data: { - rawTraceKey: key, - rawTraceFilename: filename, - ...(compiled - ? { - compileStatus: "READY" as const, - compiledSteps: compiled.steps as unknown as Prisma.InputJsonValue, - compileNotes: compiled.notes.length > 0 ? compiled.notes.join("\n\n") : null, - compileError: null, - } - : {}), - }, - }); - await appendAuditEvent( - orgId, - userId, - { - action: "recording.uploaded", - entityType: "Recording", - entityId: recording.id, - metadata: { filename, bytes: buffer.byteLength }, - }, - tx, - ); - if (compiled) { - // Audited as a compile in its own right. A reviewer looking at how a - // workflow came to exist should see that a compiler ran, even though - // it ran inline and deterministically rather than as a queued task. - await appendAuditEvent( - orgId, - userId, - { - action: "recording.compile_ready", - entityType: "Recording", - entityId: recording.id, - metadata: { stepCount: compiled.steps.length, compiler: "deterministic" }, - }, - tx, - ); - } - }); - - return { - id: recording.id, - compileStatus: compiled ? "READY" : "NONE", - stepCount: compiled?.steps.length ?? 0, - notes: compiled?.notes ?? [], - }; - } catch (err) { - // Storage failed after the row was created — don't leave an unusable - // Recording with no trace behind for the org to trip over. - await prisma.recording.delete({ where: { id: recording.id } }).catch(() => undefined); - throw err; - } -} - -/** Returns compiled steps for a structured Ghost trace, or null for anything else. */ -function tryCompile(buffer: Buffer): { steps: unknown[]; notes: string[] } | null { - let json: unknown; - try { - json = JSON.parse(buffer.toString("utf8")); - } catch { - return null; // a zip, a HAR that isn't ours, or not JSON at all - } - const parsed = parseRecordingTrace(json); - if (!parsed.ok) return null; - - const { steps, notes } = compileTrace(parsed.trace); - // A trace with no replayable action compiles to just the opening navigate. - // Storing that as READY would present an empty workflow as a result. - if (steps.length <= 1) return null; - return { steps, notes }; -} +export { ingestTrace, sanitizeFilename, MAX_TRACE_BYTES } from "@ghost/core/recording/ingest"; +export type { IngestResult } from "@ghost/core/recording/ingest"; diff --git a/cloud/apps/worker/package.json b/cloud/apps/worker/package.json index 1c88617..92ab261 100644 --- a/cloud/apps/worker/package.json +++ b/cloud/apps/worker/package.json @@ -17,10 +17,12 @@ "@sentry/node": "^10.69.0", "bullmq": "^5.34.4", "ioredis": "^5.4.2", - "playwright": "^1.55.0" + "playwright": "^1.55.0", + "ws": "^8.18.0" }, "devDependencies": { "@types/node": "^22.10.2", + "@types/ws": "^8.5.13", "tsup": "^8.3.5", "tsx": "^4.19.2", "typescript": "^5.7.2", diff --git a/cloud/apps/worker/src/capture/server.ts b/cloud/apps/worker/src/capture/server.ts new file mode 100644 index 0000000..5118f34 --- /dev/null +++ b/cloud/apps/worker/src/capture/server.ts @@ -0,0 +1,202 @@ +import { createServer, type Server } from "node:http"; +import { WebSocketServer, type WebSocket } from "ws"; +import { + CAPTURE_LIMITS, + captureConfigured, + verifyCaptureTicket, + type CaptureClientMessage, + type CaptureServerMessage, + type CaptureTicketClaims, +} from "@ghost/core/recording/capture"; +import { createLogger, serializeError } from "@ghost/core/logger"; +import { captureException } from "@ghost/core/sentry"; +import { CaptureSession } from "./session.js"; + +const log = createLogger("capture-server"); + +/** + * The socket the user's browser drives a capture session over. + * + * It lives in the worker because that is where a browser can live: `apps/web` + * runs on request-scoped serverless functions, and a person demonstrating a + * workflow holds a page for minutes. So the worker — already long-running, + * already holding Playwright for replay — accepts one WebSocket per session + * and pumps it into a `CaptureSession`. + * + * ## Authentication + * + * The worker has no session table and no way to ask the web app who is + * connecting, so it does not try: the web app signs a short-lived ticket with a + * key both processes hold, and this server verifies it offline. The ticket + * arrives as the **first message on the socket** rather than in the URL, so it + * never reaches an access log or a proxy trace. A socket that has not + * authenticated within `authTimeoutMs` is dropped, and an unauthenticated + * socket can do nothing else at all — there is no state for it to touch. + * + * With no `GHOST_CAPTURE_KEY` set, this server does not listen. That is the + * whole feature switch: not "capture but unauthenticated", which is a browser + * anyone who can reach the port may drive. + */ + +export interface CaptureServer { + port: number; + close(): Promise; +} + +/** Sessions currently holding a browser. Bounded by `maxConcurrentSessions`. */ +const live = new Set(); +/** Sessions whose browser is still starting — they count against the cap too. */ +let opening = 0; + +export function liveCaptureSessions(): number { + return live.size; +} + +export function startCaptureServer(): CaptureServer | null { + if (!captureConfigured()) { + log.info("remote capture disabled (GHOST_CAPTURE_KEY not set)"); + return null; + } + + const port = Number(process.env.GHOST_CAPTURE_PORT ?? 8787); + const http = createServer((req, res) => { + // A health endpoint, because a container that cannot be probed gets + // restarted by its scheduler at the worst possible moment. + if (req.url === "/health") { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: true, sessions: live.size, max: CAPTURE_LIMITS.maxConcurrentSessions })); + return; + } + res.writeHead(404).end(); + }); + + const wss = new WebSocketServer({ + server: http, + path: "/capture", + // Input messages are small; the ceiling exists so a malformed or hostile + // client cannot hand the worker a 100MB string to parse. + maxPayload: 256 * 1024, + }); + + wss.on("connection", (socket) => void accept(socket)); + http.listen(port, () => log.info("capture server listening", { port })); + + return { + port, + close: async () => { + // End sessions before closing the server: each holds a browser process, + // and a container that exits without closing them leaks them. + await Promise.all([...live].map((s) => s.finish("disconnected").catch(() => undefined))); + wss.close(); + await new Promise((resolve) => http.close(() => resolve())); + }, + }; +} + +async function accept(socket: WebSocket): Promise { + let session: CaptureSession | null = null; + + const send = (msg: CaptureServerMessage): void => { + if (socket.readyState !== socket.OPEN) return; + // Frames are droppable and everything else is not. A client on a slow link + // would otherwise accumulate a frame backlog in the worker's memory, and a + // frame that arrives late is worthless anyway — the next one is right + // behind it. Control messages are never dropped: `stopped` carries the + // recording id, and losing it would strand a recording the user made. + if (msg.t === "frame" && socket.bufferedAmount > CAPTURE_LIMITS.maxBufferedBytes) return; + socket.send(JSON.stringify(msg)); + }; + + const fail = (message: string): void => { + send({ t: "error", message }); + socket.close(); + }; + + // Nothing may happen on this socket until it proves who it is. + const authTimer = setTimeout(() => { + if (!session) fail("Capture session did not authenticate in time."); + }, CAPTURE_LIMITS.authTimeoutMs); + authTimer.unref(); + + socket.on("message", (raw) => { + let msg: CaptureClientMessage; + try { + msg = JSON.parse(String(raw)) as CaptureClientMessage; + } catch { + fail("Unreadable message."); + return; + } + + if (!session) { + if (msg.t !== "auth") { + // Deliberately refused rather than buffered. Accepting input for a + // session that does not exist yet is how an unauthenticated socket + // gets to influence one that later does. + fail("The first message on a capture socket must be an auth ticket."); + return; + } + clearTimeout(authTimer); + void open(msg.ticket); + return; + } + + // Every handler is a browser operation and may reject; a rejection must + // close the session rather than become an unhandled rejection that takes + // the worker down with replay still running on it. + void session.handle(msg).catch((err: unknown) => { + log.error("capture input failed", serializeError(err)); + send({ t: "error", message: "That action could not be sent to the browser." }); + }); + }); + + socket.on("close", () => { + clearTimeout(authTimer); + void session?.finish("disconnected").catch(() => undefined); + }); + socket.on("error", (err) => log.warn("capture socket error", serializeError(err))); + + async function open(ticket: string): Promise { + const verified = verifyCaptureTicket(ticket); + if (!verified.ok) { + log.warn("capture ticket rejected", { reason: verified.reason }); + fail("This capture session is no longer valid. Start a new recording."); + return; + } + + // Counted *before* the browser launches, and including sessions still + // launching. Checking `live.size` alone would let every socket that + // arrived during a slow Chromium start find room, and the cap would hold + // everywhere except under the load it exists for. + if (live.size + opening >= CAPTURE_LIMITS.maxConcurrentSessions) { + // Refused with a reason a person can act on, rather than a browser + // launched into a worker that has no memory left for it. + fail( + "Ghost is already running as many recording sessions as this worker allows. " + + "Try again in a few minutes.", + ); + return; + } + + const claims: CaptureTicketClaims = verified.claims; + opening++; + try { + const opened = await CaptureSession.open(claims, send, () => { + live.delete(opened); + if (socket.readyState === socket.OPEN) socket.close(); + }); + session = opened; + live.add(opened); + log.info("capture session opened", { + sessionId: claims.sessionId, + orgId: claims.orgId, + sessions: live.size, + }); + } catch (err) { + log.error("capture session failed to open", serializeError(err)); + captureException(err, { phase: "capture.open", sessionId: claims.sessionId }); + fail("Ghost could not start a browser for this recording. Try again."); + } finally { + opening--; + } + } +} diff --git a/cloud/apps/worker/src/capture/session.ts b/cloud/apps/worker/src/capture/session.ts new file mode 100644 index 0000000..e058dbc --- /dev/null +++ b/cloud/apps/worker/src/capture/session.ts @@ -0,0 +1,339 @@ +import type { BrowserContext, CDPSession, Page } from "playwright"; +import { chromium } from "playwright"; +import { + CAPTURE_LIMITS, + CAPTURE_VIEWPORT, + type CaptureClientMessage, + type CaptureServerMessage, + type CaptureTicketClaims, +} from "@ghost/core/recording/capture"; +import { CAPTURE_BINDING, recorderInitScript } from "@ghost/core/recording/recorder"; +import { recordingTraceSchema, type RecordingTrace } from "@ghost/core/recording/trace"; +import { ingestTrace, type IngestResult } from "@ghost/core/recording/ingest"; +import { createLogger } from "@ghost/core/logger"; +import { launchOptions } from "../browser/driver.js"; + +const log = createLogger("capture"); + +/** + * Why a session ended. Only `stopped` keeps what was recorded. + * + * `disconnected` is separate from `canceled` so the logs say what happened: a + * user who pressed Cancel made a decision, and a user whose socket dropped did + * not. They have the same effect on the trace and different meanings. + */ +export type CaptureEndReason = "stopped" | "canceled" | "idle" | "expired" | "disconnected"; + +/** + * One remote capture session: a real browser in the worker, driven by a person + * through their own browser. + * + * This is the "record" half of Phase 2, and it is a cloud browser rather than + * an extension for one reason — an extension is a thing the user has to + * install, in developer mode, before Ghost can do anything for them at all. A + * cloud browser is a page they open. + * + * ## What crosses which boundary + * + * **Coordinates go in; they never come out.** A person moving a mouse produces + * pixel positions, and forwarding them is the only way to drive a remote + * browser. But the recorder injected into the page reads the *accessible role + * and name* of whatever the pointer landed on, off the live element, and that + * is what the trace carries. Coordinates drive the session and stop at the + * page; they are not automation identity, and there is no step field that + * could hold one. + * + * **Frames go out; they are never stored.** The live view is a CDP screencast + * relayed to the socket and dropped. A capture session shows the customer's + * real systems — their inbox, their ERP — and writing that to the artifact + * store would create a video of it that nobody asked for. Run screenshots are + * different: they are evidence for an approval, deliberately retained, and + * covered by the retention window. + * + * **Secrets never enter the trace.** Enforced in the page by the recorder, + * which is the only place it can be done honestly. A password typed here is + * dispatched into the remote browser and recorded as `redacted: true` with no + * value. + * + * **No session state is kept.** The browser context is destroyed when the + * session ends. If the user signed into their bank to demonstrate a workflow, + * Ghost does not keep those cookies — holding live credentials for a system + * needs an explicit, scoped grant, and demonstrating a task is not one. + */ +export class CaptureSession { + private events: unknown[] = []; + private droppedEvents = 0; + private closed = false; + private startedAt = Date.now(); + private lastClientMessageAt = Date.now(); + private cdp: CDPSession | null = null; + private timers: NodeJS.Timeout[] = []; + + private constructor( + private readonly context: BrowserContext, + private readonly page: Page, + private readonly claims: CaptureTicketClaims, + private readonly send: (msg: CaptureServerMessage) => void, + private readonly onClosed: () => void, + ) {} + + static async open( + claims: CaptureTicketClaims, + send: (msg: CaptureServerMessage) => void, + onClosed: () => void, + ): Promise { + const browser = await chromium.launch(launchOptions()); + const context = await browser.newContext({ viewport: { ...CAPTURE_VIEWPORT } }); + + // Order matters: the binding installs its own init script, so exposing it + // first means it exists by the time the recorder's script runs. The + // recorder also queues and retries, because "first" is not a guarantee + // worth betting a whole recording on. + const page = await context.newPage(); + const session = new CaptureSession(context, page, claims, send, onClosed); + + await context.exposeBinding(CAPTURE_BINDING, (_source, event: unknown) => { + session.collect(event); + }); + await context.addInitScript({ content: recorderInitScript(CAPTURE_BINDING) }); + + await session.start(browser); + return session; + } + + private async start(browser: { close(): Promise }): Promise { + this.onBrowserClose = () => browser.close(); + + // Report the address bar back so the user can see where they are, and so a + // redirect they did not initiate is visible rather than silent. + this.page.on("framenavigated", (frame) => { + if (frame === this.page.mainFrame()) { + this.send({ t: "url", url: frame.url() }); + } + }); + + await this.page + .goto(this.claims.startUrl, { waitUntil: "domcontentloaded", timeout: 30_000 }) + .catch((err: unknown) => { + // A bad start URL must not kill the session — the user can type + // another one. Report it and stay open. + this.send({ + t: "error", + message: `Could not open ${this.claims.startUrl}: ${describeError(err)}`, + }); + }); + + this.cdp = await this.context.newCDPSession(this.page); + this.cdp.on("Page.screencastFrame", (frame: { data: string; sessionId: number }) => { + this.send({ t: "frame", data: frame.data }); + // Acked immediately rather than on the client's ack: an unacked + // screencast simply stops, and a dropped frame is better than a frozen + // live view that makes the user think the session died. + this.cdp?.send("Page.screencastFrameAck", { sessionId: frame.sessionId }).catch(() => undefined); + }); + await this.cdp.send("Page.startScreencast", { + format: "jpeg", + quality: 60, + maxWidth: CAPTURE_LIMITS.frameWidth, + maxHeight: CAPTURE_LIMITS.frameHeight, + everyNthFrame: 1, + }); + + this.send({ + t: "ready", + url: this.page.url(), + width: CAPTURE_VIEWPORT.width, + height: CAPTURE_VIEWPORT.height, + deadline: this.startedAt + CAPTURE_LIMITS.maxSessionMs, + }); + + // Both ceilings exist so an abandoned tab cannot pin a browser forever; + // enough abandoned sessions would take the worker down and replay with it. + this.timers.push( + setInterval(() => { + if (Date.now() - this.lastClientMessageAt > CAPTURE_LIMITS.idleTimeoutMs) { + void this.finish("idle"); + } else if (Date.now() - this.startedAt > CAPTURE_LIMITS.maxSessionMs) { + void this.finish("expired"); + } + }, 10_000), + ); + } + + private onBrowserClose: (() => Promise) | null = null; + + /** An event from the page-side recorder. */ + private collect(event: unknown): void { + if (this.closed) return; + if (this.events.length >= CAPTURE_LIMITS.maxEvents) { + // Counted rather than silently ignored: the reviewer is told the + // recording is incomplete instead of being handed a truncated workflow + // that looks whole. + this.droppedEvents++; + return; + } + this.events.push(event); + // Cheap progress signal. The user is watching a page, not a log, and needs + // to know Ghost is actually seeing what they do. + this.send({ t: "events", count: this.events.length }); + } + + /** A message from the user's browser. */ + async handle(msg: CaptureClientMessage): Promise { + this.lastClientMessageAt = Date.now(); + if (this.closed) return; + + switch (msg.t) { + case "mouse": + await this.cdp?.send("Input.dispatchMouseEvent", { + type: msg.type, + x: msg.x, + y: msg.y, + button: msg.button ?? "left", + clickCount: msg.clickCount ?? (msg.type === "mouseMoved" ? 0 : 1), + deltaX: msg.deltaX ?? 0, + deltaY: msg.deltaY ?? 0, + modifiers: msg.modifiers ?? 0, + }); + return; + case "key": + await this.cdp?.send("Input.dispatchKeyEvent", { + type: msg.type, + key: msg.key, + code: msg.code, + text: msg.text, + modifiers: msg.modifiers ?? 0, + windowsVirtualKeyCode: msg.windowsVirtualKeyCode, + }); + return; + case "text": + await this.cdp?.send("Input.insertText", { text: msg.value }); + return; + case "navigate": + await this.page + .goto(normalizeUrl(msg.url), { waitUntil: "domcontentloaded", timeout: 30_000 }) + .catch((err: unknown) => + this.send({ t: "error", message: `Could not open that page: ${describeError(err)}` }), + ); + return; + case "back": + await this.page.goBack().catch(() => undefined); + return; + case "forward": + await this.page.goForward().catch(() => undefined); + return; + case "reload": + await this.page.reload().catch(() => undefined); + return; + case "stop": + await this.finish("stopped"); + return; + case "cancel": + await this.finish("canceled"); + return; + case "ack": + return; + case "auth": + // Consumed by the server before this session existed. A second one is + // not a way to change orgs mid-session. + return; + } + } + + /** + * End the session. + * + * `stopped` saves and compiles; everything else throws the trace away. A + * session that timed out or was abandoned produced a partial demonstration, + * and a half-recorded workflow presented as a result is worse than none — + * the reviewer has no way to tell which half is missing. + */ + async finish(reason: CaptureEndReason): Promise { + if (this.closed) return; + this.closed = true; + for (const t of this.timers) clearInterval(t); + + try { + if (reason === "stopped") { + const result = await this.save(); + this.send({ + t: "stopped", + recordingId: result.id, + compileStatus: result.compileStatus, + stepCount: result.stepCount, + notes: this.droppedEvents > 0 + ? [ + ...result.notes, + `This recording hit Ghost's ${CAPTURE_LIMITS.maxEvents}-event ceiling and ` + + `${this.droppedEvents} later actions were not captured. Review the end of the ` + + "workflow carefully, or record it in smaller pieces.", + ] + : result.notes, + }); + } else if (reason === "canceled" || reason === "disconnected") { + // Nothing is sent for `disconnected` in practice — the socket that + // would receive it is what went away — but the trace is dropped for + // the same reason a cancel drops it: nobody pressed Stop. + this.send({ t: "canceled" }); + } else { + this.send({ + t: "error", + message: + reason === "idle" + ? "Capture ended: nothing happened for two minutes, so the browser was released." + : "Capture ended: sessions are capped at 15 minutes. Record the workflow in smaller pieces.", + }); + } + } catch (err) { + log.error("capture finish failed", { sessionId: this.claims.sessionId, err }); + this.send({ t: "error", message: `Could not save this recording: ${describeError(err)}` }); + } finally { + await this.dispose(); + this.onClosed(); + } + } + + private async save(): Promise { + const trace: RecordingTrace = recordingTraceSchema.parse({ + version: 1, + sessionId: this.claims.sessionId, + startUrl: this.claims.startUrl, + capturedAt: this.startedAt, + recorder: "ghost-cloud-browser", + events: this.events, + }); + + // Through the same path an uploaded trace takes, deliberately: one + // definition of what a Recording is, one set of limits, one audit shape. + // `ingestTrace` is also what creates the `Recording` row — nothing exists + // in the database until this moment, so an abandoned capture leaves no + // empty recording for someone to find later and have to explain. + return ingestTrace({ + orgId: this.claims.orgId, + userId: this.claims.userId, + filename: `cloud-capture-${this.claims.sessionId}.json`, + contentType: "application/json", + buffer: Buffer.from(JSON.stringify(trace), "utf8"), + }); + } + + private async dispose(): Promise { + await this.cdp?.send("Page.stopScreencast").catch(() => undefined); + await this.cdp?.detach().catch(() => undefined); + // Closing the context discards cookies and storage with it. Nothing the + // user signed into during the demonstration is kept. + await this.context.close().catch(() => undefined); + await this.onBrowserClose?.().catch(() => undefined); + } +} + +function describeError(err: unknown): string { + return err instanceof Error ? err.message.split("\n")[0]! : String(err); +} + +/** Accepts what a person types into an address bar. */ +export function normalizeUrl(input: string): string { + const trimmed = input.trim(); + if (/^https?:\/\//i.test(trimmed)) return trimmed; + return `https://${trimmed}`; +} diff --git a/cloud/apps/worker/src/crash-drive.ts b/cloud/apps/worker/src/crash-drive.ts index f6c2d2c..7236e9f 100644 --- a/cloud/apps/worker/src/crash-drive.ts +++ b/cloud/apps/worker/src/crash-drive.ts @@ -40,6 +40,7 @@ */ import { spawn, type ChildProcess } from "node:child_process"; import { fileURLToPath } from "node:url"; +import "@ghost/core/env"; import { Queue } from "bullmq"; import IORedis from "ioredis"; import { prisma, Prisma } from "@ghost/core/db"; diff --git a/cloud/apps/worker/src/e2e-drive.ts b/cloud/apps/worker/src/e2e-drive.ts index 9ec9a0d..bf6a40d 100644 --- a/cloud/apps/worker/src/e2e-drive.ts +++ b/cloud/apps/worker/src/e2e-drive.ts @@ -6,6 +6,7 @@ * * Run with: pnpm --filter @ghost/worker exec tsx src/e2e-drive.ts */ +import "@ghost/core/env"; import { Queue } from "bullmq"; import IORedis from "ioredis"; import { prisma, Prisma } from "@ghost/core/db"; diff --git a/cloud/apps/worker/src/index.ts b/cloud/apps/worker/src/index.ts index 51b0b74..fee0401 100644 --- a/cloud/apps/worker/src/index.ts +++ b/cloud/apps/worker/src/index.ts @@ -1,3 +1,6 @@ +// First, before anything reads process.env: load cloud/.env. The worker runs +// from apps/worker and nothing else puts that file into its environment. +import "@ghost/core/env"; import { Queue, Worker } from "bullmq"; import { QUEUE_NAMES, @@ -12,6 +15,8 @@ import { createRedisConnection } from "./redis.js"; import { runWorkflowJob } from "./jobs/runWorkflow.js"; import { compensateRunJob } from "./jobs/compensateRun.js"; import { purgeArtifactsJob } from "./jobs/purgeArtifacts.js"; +import { reclaimStalledRuns, RECLAIM_INTERVAL_MS } from "./jobs/reclaimRuns.js"; +import { startCaptureServer } from "./capture/server.js"; /** * Ghost worker entrypoint. @@ -91,11 +96,45 @@ await purgeQueue.upsertJobScheduler( { name: "purge-artifacts", data: {} }, ); +// Pick up runs whose worker died. Immediately on boot — a redeploy is the +// commonest way to orphan a run, and the new process is standing right where +// the old one fell — and then on an interval for crashes that happen while +// this process is up. Safe to run in every replica: the job id is derived from +// the expired lease, so concurrent sweeps collapse into one job, and the run +// lease still admits exactly one executor. +const reclaimQueue = new Queue(QUEUE_NAMES.runWorkflow, { connection }); +const sweepStalledRuns = async (): Promise => { + try { + await reclaimStalledRuns(reclaimQueue); + } catch (err) { + // A failed sweep must never take the worker down with it: the queues above + // are still executing real runs. + log.error("reclaim sweep failed", serializeError(err)); + captureException(err, { queue: QUEUE_NAMES.runWorkflow, phase: "reclaim" }); + } +}; +await sweepStalledRuns(); +const reclaimTimer = setInterval(() => void sweepStalledRuns(), RECLAIM_INTERVAL_MS); +// Do not hold the process open on this timer alone. +reclaimTimer.unref(); + +// The remote capture browser: a WebSocket the user drives a real browser +// through to demonstrate a workflow. Hosted here rather than in apps/web +// because a capture session outlives any serverless request, and because the +// Playwright this process already carries for replay is the same Playwright. +// Returns null and listens on nothing when GHOST_CAPTURE_KEY is unset. +const captureServer = startCaptureServer(); + log.info("Ghost worker started", { queues: Object.values(QUEUE_NAMES) }); async function shutdown(signal: string): Promise { log.info("shutting down", { signal }); + clearInterval(reclaimTimer); + // First, so in-flight capture sessions release their browsers instead of + // being orphaned by the exit below. + await captureServer?.close().catch(() => undefined); await Promise.all([ + reclaimQueue.close(), noopWorker.close(), runWorker.close(), compensateWorker.close(), diff --git a/cloud/apps/worker/src/jobs/reclaimRuns.test.ts b/cloud/apps/worker/src/jobs/reclaimRuns.test.ts new file mode 100644 index 0000000..839b152 --- /dev/null +++ b/cloud/apps/worker/src/jobs/reclaimRuns.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "vitest"; +import { prisma, Prisma } from "@ghost/core/db"; +import type { RunWorkflowJob } from "@ghost/core/queue"; +import type { WorkflowSteps } from "@ghost/core/schema/step"; +import { MAX_ATTEMPTS, STALL_GRACE_MS, reclaimStalledRuns } from "./reclaimRuns.js"; + +/** + * Covers the gap that made a dead worker look like a dead product: a run left + * `RUNNING` with an expired lease and no job, which nothing ever picked up. + * + * Requires DATABASE_URL; skips cleanly without it, like the other DB-backed + * suites here. + */ + +const hasDb = Boolean(process.env.DATABASE_URL); + +interface Added { + jobId?: string; + data: RunWorkflowJob; +} + +/** Records what would have been enqueued instead of touching Redis. */ +function fakeQueue() { + const added: Added[] = []; + return { + added, + add: async (_name: string, data: RunWorkflowJob, opts?: { jobId?: string }) => { + added.push({ jobId: opts?.jobId, data }); + return {} as never; + }, + }; +} + +const STEPS: WorkflowSteps = [ + { id: "nav", type: "navigate", url: "http://127.0.0.1/never-fetched", label: "Open" }, +]; + +async function seedRun(args: { + status: "QUEUED" | "RUNNING" | "AWAITING_APPROVAL"; + leaseExpiresAt: Date | null; + createdAt: Date; + attempt?: number; +}): Promise<{ runId: string; orgId: string }> { + const slug = `reclaim-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const org = await prisma.organization.create({ data: { name: "Reclaim test", slug } }); + const workflow = await prisma.workflow.create({ + data: { orgId: org.id, name: "Reclaim test workflow" }, + }); + const version = await prisma.workflowVersion.create({ + data: { + workflowId: workflow.id, + version: 1, + steps: STEPS as unknown as Prisma.InputJsonValue, + }, + }); + const run = await prisma.run.create({ + data: { + orgId: org.id, + workflowVersionId: version.id, + status: args.status, + cursor: 0, + attempt: args.attempt ?? 0, + leaseOwner: args.leaseExpiresAt ? "dead-worker" : null, + leaseExpiresAt: args.leaseExpiresAt, + createdAt: args.createdAt, + }, + }); + return { runId: run.id, orgId: org.id }; +} + +describe.skipIf(!hasDb)("reclaimStalledRuns (Postgres)", () => { + const longAgo = () => new Date(Date.now() - 10 * 60_000); + + it("re-enqueues a RUNNING run whose lease expired", async () => { + const { runId } = await seedRun({ + status: "RUNNING", + leaseExpiresAt: longAgo(), + createdAt: longAgo(), + }); + + const queue = fakeQueue(); + const result = await reclaimStalledRuns(queue); + + expect(result.requeued).toContain(runId); + expect(queue.added.some((a) => a.data.runId === runId)).toBe(true); + }); + + it("leaves a run alone while its lease is still live", async () => { + const { runId } = await seedRun({ + status: "RUNNING", + // Comfortably inside the grace window, as a healthy worker's would be. + leaseExpiresAt: new Date(Date.now() + 30_000), + createdAt: longAgo(), + }); + + const queue = fakeQueue(); + const result = await reclaimStalledRuns(queue); + + expect(result.requeued).not.toContain(runId); + }); + + it("does not disturb a run waiting on a human", async () => { + // The case that would be actively harmful: a gate is *meant* to sit for + // days with no lease. Re-driving it would reopen an approval nobody touched. + const { runId } = await seedRun({ + status: "AWAITING_APPROVAL", + leaseExpiresAt: null, + createdAt: longAgo(), + }); + + const queue = fakeQueue(); + const result = await reclaimStalledRuns(queue); + + expect(result.requeued).not.toContain(runId); + expect(result.incidents).not.toContain(runId); + }); + + it("respects the grace period rather than racing a slow renewal", async () => { + const { runId } = await seedRun({ + status: "RUNNING", + // Expired, but only just — a worker mid-renewal looks like this. + leaseExpiresAt: new Date(Date.now() - Math.floor(STALL_GRACE_MS / 2)), + createdAt: longAgo(), + }); + + const queue = fakeQueue(); + const result = await reclaimStalledRuns(queue); + + expect(result.requeued).not.toContain(runId); + }); + + it("stops restarting after MAX_ATTEMPTS and raises an incident instead", async () => { + const { runId } = await seedRun({ + status: "RUNNING", + leaseExpiresAt: longAgo(), + createdAt: longAgo(), + attempt: MAX_ATTEMPTS, + }); + + const queue = fakeQueue(); + const result = await reclaimStalledRuns(queue); + + expect(result.incidents).toContain(runId); + expect(result.requeued).not.toContain(runId); + + const run = await prisma.run.findUniqueOrThrow({ where: { id: runId } }); + expect(run.status).toBe("INCIDENT"); + expect(run.error).toMatch(/stopped restarting it automatically/); + // The lease must be released, or a human retry could not take the run. + expect(run.leaseOwner).toBeNull(); + }); + + it("gives two concurrent sweeps the same job id so they collapse", async () => { + const { runId } = await seedRun({ + status: "RUNNING", + leaseExpiresAt: longAgo(), + createdAt: longAgo(), + }); + + const a = fakeQueue(); + const b = fakeQueue(); + await reclaimStalledRuns(a); + await reclaimStalledRuns(b); + + const idA = a.added.find((x) => x.data.runId === runId)?.jobId; + const idB = b.added.find((x) => x.data.runId === runId)?.jobId; + expect(idA).toBeDefined(); + expect(idA).toBe(idB); + }); +}); diff --git a/cloud/apps/worker/src/jobs/reclaimRuns.ts b/cloud/apps/worker/src/jobs/reclaimRuns.ts new file mode 100644 index 0000000..4e86250 --- /dev/null +++ b/cloud/apps/worker/src/jobs/reclaimRuns.ts @@ -0,0 +1,141 @@ +import type { Queue } from "bullmq"; +import { prisma } from "@ghost/core/db"; +import { appendAuditEvent } from "@ghost/core/audit-log"; +import { runWorkflowJobId, type RunWorkflowJob } from "@ghost/core/queue"; +import { createLogger } from "@ghost/core/logger"; + +const log = createLogger("reclaim-runs"); + +/** + * Restarts runs whose worker died. + * + * The lease in `runWorkflow` prevents two workers from executing one run at + * once, and it correctly lets a *new* job take over a lease that has expired. + * What was missing is the thing that produces that new job. When a worker + * process disappeared mid-run — a crash, a redeploy, a laptop closing — the + * BullMQ job died with it, so nothing ever re-enqueued the run. The row stayed + * `RUNNING` with an expired lease **forever**, and the UI showed a run in + * progress that no process was working on. That is indistinguishable, to the + * person watching, from Ghost simply not working. + * + * Restarting is safe rather than merely convenient, and only because of how + * the run journal is built: position is folded from an append-only, + * hash-chained log, so a step that already completed is not executed again on + * resume. This function does not re-run anything; it re-enters a run at the + * position its own journal proves it reached. + * + * Runs that have stalled too many times are not restarted forever. Past + * `MAX_ATTEMPTS` the run is moved to `INCIDENT`, where a human decides — + * exactly what an incident is for. Looping instead would keep a workflow that + * reliably kills its worker doing so. + * + * Touches: database (run rows, audit log), Redis (enqueue). No browser, no + * network to customer systems. + */ + +/** + * How far past lease expiry a run must be before it is considered abandoned. + * + * The lease is renewed every LEASE_MS/3 (20s), so brief scheduling delays and + * clock skew must not look like death. A live worker that is merely slow to + * renew keeps its run; only one that has been silent well beyond its renewal + * interval loses it. + */ +export const STALL_GRACE_MS = 90_000; + +/** Restarts beyond this become an incident for a human instead. */ +export const MAX_ATTEMPTS = 5; + +/** How often the worker sweeps. Cheap: one indexed query. */ +export const RECLAIM_INTERVAL_MS = 60_000; + +export interface ReclaimResult { + requeued: string[]; + incidents: string[]; +} + +export async function reclaimStalledRuns( + queue: Pick, "add">, + now: Date = new Date(), +): Promise { + const cutoff = new Date(now.getTime() - STALL_GRACE_MS); + + const stalled = await prisma.run.findMany({ + where: { + // Only states where a worker is supposed to be actively holding the run. + // AWAITING_APPROVAL is deliberately absent: a run waiting on a human is + // meant to sit for days with no lease, and re-driving one would reopen + // gates nobody touched. + status: { in: ["QUEUED", "RUNNING"] }, + OR: [{ leaseExpiresAt: null }, { leaseExpiresAt: { lt: cutoff } }], + createdAt: { lt: cutoff }, + }, + select: { id: true, orgId: true, cursor: true, attempt: true, leaseExpiresAt: true }, + // Bounded so one sweep cannot enqueue an unbounded burst after a long + // outage; the next sweep takes the rest. + take: 50, + }); + + const result: ReclaimResult = { requeued: [], incidents: [] }; + + for (const run of stalled) { + if (run.attempt >= MAX_ATTEMPTS) { + // updateMany with the status in the filter: if a worker revived and + // finished this run since the query above, do not overwrite its outcome. + const moved = await prisma.run.updateMany({ + where: { id: run.id, status: { in: ["QUEUED", "RUNNING"] } }, + data: { + status: "INCIDENT", + error: + `Run stopped making progress ${MAX_ATTEMPTS} times without finishing. ` + + "Ghost stopped restarting it automatically. Retry, skip the failing step, or cancel.", + leaseOwner: null, + leaseExpiresAt: null, + }, + }); + if (moved.count === 1) { + result.incidents.push(run.id); + await appendAuditEvent(run.orgId, null, { + action: "run.reclaim_exhausted", + entityType: "Run", + entityId: run.id, + metadata: { attempts: run.attempt }, + }).catch((err) => log.error("audit failed for reclaim_exhausted", { runId: run.id, err })); + } + continue; + } + + // Derived from the expired lease rather than from the clock, so two worker + // replicas sweeping at the same moment produce the *same* job id and + // collapse into one job instead of racing. A later stall of the same run + // has a different expiry and so gets a fresh id, which is what stops the + // retained completed job from swallowing it. + const resumeToken = `reclaim-${run.leaseExpiresAt?.getTime() ?? "none"}`; + + await queue.add( + "run-workflow", + { runId: run.id, orgId: run.orgId, fromStepIndex: run.cursor }, + { + jobId: runWorkflowJobId(run.id, run.cursor, resumeToken), + removeOnComplete: 100, + removeOnFail: 500, + }, + ); + result.requeued.push(run.id); + + await appendAuditEvent(run.orgId, null, { + action: "run.reclaimed", + entityType: "Run", + entityId: run.id, + metadata: { cursor: run.cursor, attempt: run.attempt, reason: "lease expired" }, + }).catch((err) => log.error("audit failed for reclaim", { runId: run.id, err })); + } + + if (result.requeued.length > 0 || result.incidents.length > 0) { + log.info("reclaimed stalled runs", { + requeued: result.requeued.length, + incidents: result.incidents.length, + }); + } + return result; +} diff --git a/cloud/docker-compose.yml b/cloud/docker-compose.yml index bc7e8f3..1d63ae6 100644 --- a/cloud/docker-compose.yml +++ b/cloud/docker-compose.yml @@ -1,6 +1,11 @@ # Local development dependencies for Ghost Cloud. # docker compose up -d -# Brings up Postgres (5432) and Redis (6379) matching cloud/.env.example. +# +# Host ports are overridable because 5432 and 6379 are frequently already taken +# by an unrelated Postgres or Redis on a developer machine. `scripts/lib.sh` +# probes for a port that genuinely answers as the ghost user and sets these, +# rather than assuming an open port is ours — see the comment at the top of that +# file for the failure that caused. services: postgres: image: postgres:16-alpine @@ -11,7 +16,7 @@ services: POSTGRES_PASSWORD: ghost POSTGRES_DB: ghost ports: - - "5432:5432" + - "${GHOST_PG_PORT:-5432}:5432" volumes: - ghost-postgres-data:/var/lib/postgresql/data healthcheck: @@ -25,7 +30,7 @@ services: container_name: ghost-redis restart: unless-stopped ports: - - "6379:6379" + - "${GHOST_REDIS_PORT:-6379}:6379" volumes: - ghost-redis-data:/data healthcheck: diff --git a/cloud/docs/CURSOR_HANDOFF.md b/cloud/docs/CURSOR_HANDOFF.md index 9959c66..c1141e5 100644 --- a/cloud/docs/CURSOR_HANDOFF.md +++ b/cloud/docs/CURSOR_HANDOFF.md @@ -435,10 +435,46 @@ is scaffolding, not the product. working Chrome extension (records clicks/typing/selects/submits/navigation via accessible role+name, redacts secret-shaped fields at capture, uploads to `POST /api/agent/recordings` with a revocable bearer token) — see its own -`README.md` for the trust boundary. It lands on `feat/browser-recording-extension`, -not yet merged, so `README.md`'s Phase 2 status line ("capture is next") is -correct as of `master` but stale the moment this branch merges. Update it in -the same PR. +`README.md` for the trust boundary. **Merged to `master` in PR #409** (this +paragraph previously said "not yet merged" — that was stale the moment the PR +landed; `README.md`'s Phase 2 status line needs the same correction if it +still says "capture is next"). + +The Capture → Convert pipeline is real and wired end-to-end: extension records +→ `POST /api/agent/recordings` → compile (HarnessRouter, optional) → review in +`WorkflowEditor` → `POST /api/workflows` → the normal approve/execute/verify +loop. What has never happened is anyone actually *using* it to produce a +workflow worth showing someone — see "Make Ghost demo-able" below. + +## Make Ghost demo-able (do this next) + +As of 2026-08-05 `ghost.muharafiq.com` is a real, working deployment: three +sign-in methods (GitHub/Google/email magic link), a public landing page +instead of a forced auth wall, real Postgres/Redis/worker infra. None of that +is the gap anymore. The gap is that nothing on the live site *shows* the trust +loop running. The only workflow ever exercised end-to-end is the seeded +`fixtures/order` demo, and it is not reachable or visible from the public +landing page — a visitor who signs in sees an empty dashboard with a bullet +list describing what Ghost does, not something doing it. + +Ghost is genuinely early-stage. The fix is not to fabricate a fake "real +customer workflow" to look further along than it is — it's to make the +capability that already exists **visible and walkable**, honestly labeled as +a demo: + +- A demo entry point (linked from the landing page and/or dashboard) that runs + `fixtures/order` — or an equally simple, self-contained workflow purpose-built + for showing the loop — through Capture/Author → Review → Approve → + Execute → Verify → Audit, with each stage visible as it happens rather than + requiring someone to already know where `/runs/[id]` or `/audit` live. +- Consider recording *this* workflow through the now-merged Chrome extension + instead of hand-authoring it, as a way of exercising Capture → Convert with + something real before the next customer does. +- The approval gate should be genuinely clickable by whoever's watching, not + narrated — that's the entire point of the product being demoed. +- Keep it honestly scoped: a labeled demo, not a claim that this is a live + customer integration. Marketing/docs must not promise capabilities the app + cannot support (Engineering rule 10 in the root `CLAUDE.md`). ## Worker container: built, but had never actually run diff --git a/cloud/docs/DEPLOY.md b/cloud/docs/DEPLOY.md index 7a36653..1b77a91 100644 --- a/cloud/docs/DEPLOY.md +++ b/cloud/docs/DEPLOY.md @@ -53,6 +53,7 @@ B2, or MinIO. | `AUTH_SECRET` | ● | | sessions cannot be signed | | `AUTH_GITHUB_ID` / `AUTH_GITHUB_SECRET` | ● | | one fewer sign-in option; **no way to sign in at all** if Google is also unset — see below | | `AUTH_GOOGLE_ID` / `AUTH_GOOGLE_SECRET` | ● | | same as above, independently — offer either, both, or neither | +| `RESEND_API_KEY` / `RESEND_EMAIL_DOMAIN` | ● | | same as above; also activates the Prisma adapter (`auth.ts`), which GitHub/Google reuse for account linking | | `S3_BUCKET`, `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY` | ● | ● | silent fallback to local disk; artifacts lost | | `S3_REGION`, `S3_ENDPOINT` | ○ | ○ | needed for non-AWS S3-compatible stores | | `GHOST_SESSION_KEY` | | ● | gated runs cannot resume after approval | @@ -69,14 +70,16 @@ reason to hold that power. `apps/web/src/app/signin/page.tsx` shows the dev email form only when `NODE_ENV !== "production"`, the GitHub button only when both GitHub -variables are set, and the Google button only when both Google variables are -set — each provider is independent. In production with all three unset, the -page renders an explicit "no sign-in method is configured" error rather than -silently showing nothing (see `signin/page.tsx`'s `misconfigured` check), but -that is a diagnostic, not a fix: configure at least one real OAuth app before -the first deploy, with its callback at +variables are set, the Google button only when both Google variables are set, +and the email-magic-link form only when both Resend variables are set — each +provider is independent. In production with all four unset, the page renders +an explicit "no sign-in method is configured" error rather than silently +showing nothing (see `signin/page.tsx`'s `misconfigured` check), but that is a +diagnostic, not a fix: configure at least one real sign-in method before the +first deploy. OAuth apps need their callback at `https:///api/auth/callback/github` or -`https:///api/auth/callback/google`. +`https:///api/auth/callback/google`; Resend needs its sending +domain verified — see "Resend (email magic link)" below. ### The domain trap @@ -213,6 +216,27 @@ market (ops-heavy SMBs) who doesn't have or want a GitHub account: - Note the Client ID and Client Secret; these become `AUTH_GOOGLE_ID` / `AUTH_GOOGLE_SECRET`. +**Resend (email magic link)** (resend.com, or `vercel integration add +resend/resend-email` — the Vercel Marketplace path auto-injects +`RESEND_API_KEY` into the linked project) — a third independent option, for +anyone who'd rather not use either OAuth provider: +- Pick a **subdomain** to send from (e.g. `mail.`), not your + root domain, if the root already has real mail (Google Workspace, iCloud, + etc.) — Resend's SPF/DKIM records can conflict with an existing mail setup + otherwise. +- Add that domain in Resend, then add the DKIM (TXT) and SPF (MX + TXT) + records it gives you to your DNS zone. If the zone is Vercel-managed + (`vercel domains ls`), `vercel dns add ` + works directly; verification is near-instant once records propagate — check + with `GET https://api.resend.com/domains/` (`Authorization: Bearer + `). +- `RESEND_API_KEY` and `RESEND_EMAIL_DOMAIN` become the two env vars. Setting + both also activates the Prisma adapter in `auth.ts` (the Email provider + needs it to store one-time tokens) — GitHub and Google get + `allowDangerousEmailAccountLinking: true` at the same time, to keep their + existing merge-by-email behavior instead of Auth.js rejecting a sign-in + whose email already belongs to a different provider. + **Vercel project for `cloud/apps/web`** — this is "The domain trap" above made concrete: 1. Create a **new, second** Vercel project. Do not add `cloud/apps/web` to @@ -242,8 +266,8 @@ concrete: marketing site's). 6. Set every `web`-column environment variable from the table above on this project (`DATABASE_URL`, `REDIS_URL`, `AUTH_SECRET`, `AUTH_GITHUB_ID`/ - `AUTH_GITHUB_SECRET`, `AUTH_GOOGLE_ID`/`AUTH_GOOGLE_SECRET`, the `S3_*` - set, `APP_URL`). + `AUTH_GITHUB_SECRET`, `AUTH_GOOGLE_ID`/`AUTH_GOOGLE_SECRET`, + `RESEND_API_KEY`/`RESEND_EMAIL_DOMAIN`, the `S3_*` set, `APP_URL`). 7. **Disable Vercel's own Deployment Protection (SSO/Vercel Authentication)** for this project, or it gates every page — including `/signin` — behind a Vercel-account login wall on top of Ghost's own auth, blocking real users diff --git a/cloud/package.json b/cloud/package.json index 451ee9f..6b06a85 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -10,6 +10,7 @@ }, "scripts": { "demo": "bash scripts/demo.sh", + "check": "bash scripts/doctor.sh", "dev": "turbo run dev", "build": "turbo run build", "lint": "turbo run lint", diff --git a/cloud/packages/core/package.json b/cloud/packages/core/package.json index 045f197..9f59151 100644 --- a/cloud/packages/core/package.json +++ b/cloud/packages/core/package.json @@ -29,9 +29,13 @@ "./crypto/mfa-secret": "./src/crypto/mfa-secret.ts", "./recording/trace": "./src/recording/trace.ts", "./recording/compile": "./src/recording/compile.ts", + "./recording/capture": "./src/recording/capture.ts", + "./recording/recorder": "./src/recording/recorder.ts", + "./recording/ingest": "./src/recording/ingest.ts", "./retention": "./src/retention.ts", "./logger": "./src/logger.ts", - "./sentry": "./src/sentry.ts" + "./sentry": "./src/sentry.ts", + "./env": "./src/env.ts" }, "scripts": { "build": "prisma generate && tsc --noEmit", @@ -46,11 +50,13 @@ "db:migrate:deploy": "prisma migrate deploy" }, "dependencies": { - "@prisma/client": "^6.1.0", - "zod": "^3.24.1", "@aws-sdk/client-s3": "3.1101.0", "@aws-sdk/s3-request-presigner": "3.1101.0", - "@sentry/node": "^10.69.0" + "@prisma/client": "^6.1.0", + "@sentry/node": "^10.69.0", + "@vercel/blob": "^2.7.0", + "dotenv": "^16.4.7", + "zod": "^3.24.1" }, "devDependencies": { "prisma": "^6.1.0", diff --git a/cloud/packages/core/src/env.ts b/cloud/packages/core/src/env.ts new file mode 100644 index 0000000..d8eb7cf --- /dev/null +++ b/cloud/packages/core/src/env.ts @@ -0,0 +1,53 @@ +import { existsSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { config as loadDotenv } from "dotenv"; + +/** + * Loads `cloud/.env` into `process.env`. + * + * The README has always said `cp .env.example .env` and then `pnpm dev`, and + * that could not work: `.env` lives at the workspace root while the apps run + * from `apps/web` and `apps/worker`. Next.js reads `.env` relative to its own + * project directory and the worker read nothing at all, so the worker died on + * "REDIS_URL is not set" and the web app ran against whatever happened to be + * exported in the shell that started it. Every local setup that appeared to + * work was one where someone had exported the variables by hand — which is + * also why the failure never reproduced for whoever had done so. + * + * Import this **first**, before any module that reads `process.env` at import + * time. ES module evaluation follows import order, so a first import is + * enough. + * + * Deliberately non-overriding: a real environment (Vercel, a container, + * `DATABASE_URL=... pnpm dev`) always wins over the file. In deployment there + * is no `.env` and this is a no-op. + */ + +/** Walks up from `startDir` looking for the workspace `.env`. */ +export function findEnvFile(startDir: string = process.cwd()): string | null { + let dir = resolve(startDir); + // Bounded by reaching the filesystem root, where `dirname` is a fixed point. + for (;;) { + const candidate = join(dir, ".env"); + if (existsSync(candidate)) return candidate; + // Stop at the workspace root rather than escaping into the user's home + // directory and picking up an unrelated project's .env. + if (existsSync(join(dir, "pnpm-workspace.yaml"))) return null; + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +let loaded: string | null | undefined; + +/** Returns the path loaded, or null when there was no file to load. */ +export function loadEnv(startDir?: string): string | null { + if (loaded !== undefined && startDir === undefined) return loaded; + const file = findEnvFile(startDir); + if (file) loadDotenv({ path: file, override: false }); + if (startDir === undefined) loaded = file; + return file; +} + +loadEnv(); diff --git a/cloud/packages/core/src/recording/capture.ts b/cloud/packages/core/src/recording/capture.ts new file mode 100644 index 0000000..e361002 --- /dev/null +++ b/cloud/packages/core/src/recording/capture.ts @@ -0,0 +1,277 @@ +import { createHmac, randomUUID, timingSafeEqual } from "node:crypto"; + +/** + * The contract between the web app and the worker's remote capture browser. + * + * Recording needs a browser, and the browser cannot live where the UI does: + * `apps/web` runs on serverless functions that end with the request, while a + * capture session is a human driving a page for several minutes. So the + * browser lives in the worker — the same long-running container that already + * holds Playwright for replay — and the user drives it over a WebSocket. + * + * That splits one session across two processes which share no memory, so this + * module defines the three things both sides must agree on: who is allowed to + * connect (`mintCaptureTicket` / `verifyCaptureTicket`), what may be said over + * the socket (the message unions), and when a session is cut off + * (`CAPTURE_LIMITS`). + * + * ## Why a ticket rather than the session cookie + * + * The worker has no Auth.js, no session table, and no reason to grow either. + * The web app is the only thing that knows who is signed in, so it says so + * once, in a form the worker can check offline: an HMAC over the fields that + * matter, with a short expiry. The worker holds the same key and verifies; it + * never calls back into the web app, so a capture cannot be opened by anyone + * who has not just been authenticated by it. + * + * The ticket is deliberately narrow. It authorises **one** capture session for + * **one** recording in **one** organization, for about as long as it takes a + * browser to open a socket. It is not a session token: it cannot be replayed + * into any other endpoint, and it grants nothing after the worker has bound it + * to a live session. + */ + +export const CAPTURE_TICKET_VERSION = "gcap1"; + +/** What the web app asserts to the worker about a capture session. */ +export interface CaptureTicketClaims { + /** + * Correlates the session across the two processes and names its trace. + * + * Deliberately *not* a `Recording` row id. The row is created by + * `ingestTrace` at the moment a trace exists, so a capture the user abandons + * — a closed tab, a session that timed out — leaves nothing behind to + * explain. The real recording id comes back in the `stopped` message. + */ + sessionId: string; + orgId: string; + /** Who started it, for the audit trail. Null for org-scoped callers. */ + userId: string | null; + /** Where the browser opens. Bound into the ticket so it cannot be swapped. */ + startUrl: string; + /** Epoch seconds. */ + exp: number; +} + +export class CaptureKeyMissingError extends Error { + constructor() { + super( + "GHOST_CAPTURE_KEY is not set. Generate one with `openssl rand -base64 32` and set the " + + "same value on both the web app and the worker — it is what lets the worker trust that " + + "a capture session belongs to a signed-in user.", + ); + this.name = "CaptureKeyMissingError"; + } +} + +/** + * Whether remote capture is available at all. + * + * Absent key means the feature is off, not degraded: the UI does not offer it + * and the worker does not listen. Ghost refuses to accept unauthenticated + * capture connections rather than quietly running a browser for anyone who can + * reach the port. + */ +export function captureConfigured(): boolean { + return Boolean(process.env.GHOST_CAPTURE_KEY); +} + +function captureKey(): Buffer { + const raw = process.env.GHOST_CAPTURE_KEY; + if (!raw) throw new CaptureKeyMissingError(); + return Buffer.from(raw, "utf8"); +} + +function b64url(input: Buffer): string { + return input.toString("base64url"); +} + +/** + * How long a freshly minted ticket may sit unused. + * + * Short on purpose: the only gap it has to cover is the browser opening a + * WebSocket to a URL it was just handed. Anything longer is a bearer token + * sitting in a page's memory for no reason. + */ +export const CAPTURE_TICKET_TTL_SECONDS = 120; + +/** A fresh correlation id for one capture session. */ +export function newCaptureSessionId(): string { + return randomUUID(); +} + +export function mintCaptureTicket( + claims: Omit & { exp?: number }, +): string { + const payload: CaptureTicketClaims = { + sessionId: claims.sessionId, + orgId: claims.orgId, + userId: claims.userId, + startUrl: claims.startUrl, + exp: claims.exp ?? Math.floor(Date.now() / 1000) + CAPTURE_TICKET_TTL_SECONDS, + }; + const body = b64url(Buffer.from(JSON.stringify(payload), "utf8")); + const mac = b64url( + createHmac("sha256", captureKey()).update(`${CAPTURE_TICKET_VERSION}.${body}`).digest(), + ); + return `${CAPTURE_TICKET_VERSION}.${body}.${mac}`; +} + +export type CaptureTicketResult = + | { ok: true; claims: CaptureTicketClaims } + | { ok: false; reason: string }; + +export function verifyCaptureTicket(ticket: string): CaptureTicketResult { + const parts = ticket.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed ticket" }; + const [version, body, mac] = parts as [string, string, string]; + if (version !== CAPTURE_TICKET_VERSION) return { ok: false, reason: "unknown ticket version" }; + + let expected: string; + try { + expected = b64url( + createHmac("sha256", captureKey()).update(`${version}.${body}`).digest(), + ); + } catch { + return { ok: false, reason: "capture key is not configured" }; + } + + // Compare in constant time, and only after a length check — `timingSafeEqual` + // throws on a length mismatch rather than returning false. + const a = Buffer.from(mac, "utf8"); + const b = Buffer.from(expected, "utf8"); + if (a.length !== b.length || !timingSafeEqual(a, b)) { + return { ok: false, reason: "signature does not match" }; + } + + let claims: CaptureTicketClaims; + try { + claims = JSON.parse(Buffer.from(body, "base64url").toString("utf8")) as CaptureTicketClaims; + } catch { + return { ok: false, reason: "ticket payload is not readable" }; + } + + if ( + typeof claims?.sessionId !== "string" || + typeof claims?.orgId !== "string" || + typeof claims?.startUrl !== "string" || + typeof claims?.exp !== "number" + ) { + return { ok: false, reason: "ticket payload is incomplete" }; + } + if (claims.exp * 1000 < Date.now()) return { ok: false, reason: "ticket has expired" }; + + return { ok: true, claims }; +} + +/** + * Ceilings a capture session cannot exceed. + * + * Every one of these exists because a capture session holds a real browser in + * a container with finite memory, and the thing driving it is a person who may + * simply close the tab. Without a cap an abandoned session pins a browser + * forever, and enough abandoned sessions take the worker down — taking replay, + * which is the part customers depend on, with it. + */ +export const CAPTURE_LIMITS = { + /** Wall clock from first connect. A workflow demonstration is minutes, not hours. */ + maxSessionMs: 15 * 60 * 1000, + /** No message from the client at all — the tab is gone, or the network is. */ + idleTimeoutMs: 2 * 60 * 1000, + /** + * Trace events retained. A runaway page (an animation loop firing `change`, + * a redirect storm) must not grow the trace without bound in memory. + */ + maxEvents: 5_000, + /** Concurrent sessions per worker process, each holding its own browser. */ + maxConcurrentSessions: Number(process.env.GHOST_CAPTURE_MAX_SESSIONS ?? 3), + /** Screencast frame size. Bigger costs bandwidth for no reviewing benefit. */ + frameWidth: 1280, + frameHeight: 800, + /** + * How long a connected socket has to send its `auth` message before it is + * dropped. An unauthenticated socket costs nothing but a file descriptor — + * but only because it is never allowed to hold one for long. + */ + authTimeoutMs: 5_000, + /** + * Bytes of unsent frame data past which new frames are dropped instead of + * queued. A client on a slow link cannot be allowed to turn the worker's + * memory into a frame backlog, and a stale frame has no value anyway — the + * next one is 30ms behind it. + */ + maxBufferedBytes: 4 * 1024 * 1024, +} as const; + +/** Default viewport of the remote browser, and of the live view showing it. */ +export const CAPTURE_VIEWPORT = { width: 1280, height: 800 } as const; + +// --------------------------------------------------------------------------- +// Wire protocol +// --------------------------------------------------------------------------- + +/** + * What the user's browser may ask the remote one to do. + * + * Input is forwarded as coordinates because that is what a human moving a + * mouse produces, and it never reaches the trace: the recorder inside the page + * reads the accessible role and name off whatever the pointer landed on, and + * the compiled step carries those. Coordinates drive the live session; they + * are not, and must never become, automation identity. + */ +export type CaptureClientMessage = + /** + * First frame on the socket, always. The ticket travels in the body rather + * than a query string so it never lands in an access log, a proxy trace, or + * a `Referer` — and because the browser `WebSocket` API cannot set headers. + */ + | { t: "auth"; ticket: string } + | { + t: "mouse"; + type: "mousePressed" | "mouseReleased" | "mouseMoved" | "mouseWheel"; + x: number; + y: number; + button?: "none" | "left" | "middle" | "right"; + clickCount?: number; + deltaX?: number; + deltaY?: number; + modifiers?: number; + } + | { + t: "key"; + type: "keyDown" | "keyUp" | "rawKeyDown" | "char"; + key?: string; + code?: string; + text?: string; + modifiers?: number; + windowsVirtualKeyCode?: number; + } + /** Bulk text (paste, IME commit) — cheaper and more reliable than key-by-key. */ + | { t: "text"; value: string } + | { t: "navigate"; url: string } + | { t: "back" } + | { t: "forward" } + | { t: "reload" } + /** Finish: save the trace, compile it, and end the session. */ + | { t: "stop" } + /** Abandon: throw the trace away and end the session. */ + | { t: "cancel" } + /** Frame received — the worker sends the next only after this. */ + | { t: "ack" }; + +export type CaptureServerMessage = + | { t: "ready"; url: string; width: number; height: number; deadline: number } + /** A JPEG screencast frame, base64. Relayed live and never stored. */ + | { t: "frame"; data: string } + | { t: "url"; url: string } + /** How much the recorder has captured so far, so the user sees it working. */ + | { t: "events"; count: number } + | { + t: "stopped"; + recordingId: string; + compileStatus: "READY" | "NONE"; + stepCount: number; + notes: string[]; + } + | { t: "canceled" } + | { t: "error"; message: string }; diff --git a/cloud/packages/core/src/recording/ingest.ts b/cloud/packages/core/src/recording/ingest.ts new file mode 100644 index 0000000..f0d7daa --- /dev/null +++ b/cloud/packages/core/src/recording/ingest.ts @@ -0,0 +1,141 @@ +import { appendAuditEvent } from "../auditLog.js"; +import { artifactStore } from "../storage/artifacts.js"; +import { prisma, Prisma } from "../db.js"; +import { compileTrace } from "./compile.js"; +import { parseRecordingTrace } from "./trace.js"; + +/** + * Storing a recording trace, shared by every producer — the browser upload + * form, the extension's ingest endpoint, and the worker's remote capture + * browser — so they cannot drift on limits, sanitisation, or what gets audited. + * + * It lives in core rather than in the web app because the worker needs it too: + * a cloud capture session ends inside the worker, and a second copy of "how a + * trace becomes a Recording" is how the two would stop agreeing about what a + * recording is. + * + * The interesting part is what happens to a *structured* trace. A Ghost + * recorder reads the accessible role and name off each element while it is + * still on screen, so the trace already contains what the worker's resolution + * chain wants. `compileTrace` then turns it into typed steps deterministically + * — no model, no network, no configured compiler — and the recording lands + * `READY` for review in a single request. + * + * That is what makes recording work in production, where there is deliberately + * no compiler configured at all (see `docs/DEPLOY.md`). Anything that is *not* + * a structured trace — a HAR, a Playwright zip — is stored as before and left + * for whatever compiler exists, if any. + */ + +/** Small event logs, not video. Well under Vercel's 100MB body limit. */ +export const MAX_TRACE_BYTES = 25 * 1024 * 1024; + +export function sanitizeFilename(name: string): string { + const base = name.split(/[/\\]/).pop() || "recording-trace"; + return base.replace(/[^a-zA-Z0-9._-]/g, "_").slice(-200) || "recording-trace"; +} + +export interface IngestResult { + id: string; + /** READY when the trace compiled deterministically, NONE when it awaits a compiler. */ + compileStatus: "READY" | "NONE"; + stepCount: number; + notes: string[]; +} + +export async function ingestTrace(args: { + orgId: string; + userId: string | null; + filename: string; + contentType: string; + buffer: Buffer; +}): Promise { + const { orgId, userId, buffer } = args; + const filename = sanitizeFilename(args.filename); + + const recording = await prisma.recording.create({ + data: { orgId, status: "STOPPED" }, + }); + + try { + const key = `recordings/${recording.id}/trace-${filename}`; + await artifactStore().put(key, buffer, args.contentType || "application/octet-stream"); + + const compiled = tryCompile(buffer); + + await prisma.$transaction(async (tx) => { + await tx.recording.update({ + where: { id: recording.id }, + data: { + rawTraceKey: key, + rawTraceFilename: filename, + ...(compiled + ? { + compileStatus: "READY" as const, + compiledSteps: compiled.steps as unknown as Prisma.InputJsonValue, + compileNotes: compiled.notes.length > 0 ? compiled.notes.join("\n\n") : null, + compileError: null, + } + : {}), + }, + }); + await appendAuditEvent( + orgId, + userId, + { + action: "recording.uploaded", + entityType: "Recording", + entityId: recording.id, + metadata: { filename, bytes: buffer.byteLength }, + }, + tx, + ); + if (compiled) { + // Audited as a compile in its own right. A reviewer looking at how a + // workflow came to exist should see that a compiler ran, even though + // it ran inline and deterministically rather than as a queued task. + await appendAuditEvent( + orgId, + userId, + { + action: "recording.compile_ready", + entityType: "Recording", + entityId: recording.id, + metadata: { stepCount: compiled.steps.length, compiler: "deterministic" }, + }, + tx, + ); + } + }); + + return { + id: recording.id, + compileStatus: compiled ? "READY" : "NONE", + stepCount: compiled?.steps.length ?? 0, + notes: compiled?.notes ?? [], + }; + } catch (err) { + // Storage failed after the row was created — don't leave an unusable + // Recording with no trace behind for the org to trip over. + await prisma.recording.delete({ where: { id: recording.id } }).catch(() => undefined); + throw err; + } +} + +/** Returns compiled steps for a structured Ghost trace, or null for anything else. */ +function tryCompile(buffer: Buffer): { steps: unknown[]; notes: string[] } | null { + let json: unknown; + try { + json = JSON.parse(buffer.toString("utf8")); + } catch { + return null; // a zip, a HAR that isn't ours, or not JSON at all + } + const parsed = parseRecordingTrace(json); + if (!parsed.ok) return null; + + const { steps, notes } = compileTrace(parsed.trace); + // A trace with no replayable action compiles to just the opening navigate. + // Storing that as READY would present an empty workflow as a result. + if (steps.length <= 1) return null; + return { steps, notes }; +} diff --git a/cloud/packages/core/src/recording/recorder.ts b/cloud/packages/core/src/recording/recorder.ts new file mode 100644 index 0000000..bc4ca07 --- /dev/null +++ b/cloud/packages/core/src/recording/recorder.ts @@ -0,0 +1,377 @@ +/// +/** + * The page-side recorder, as a function that can be shipped into a page. + * + * The `dom` lib reference above belongs in this file rather than in a + * `tsconfig`: this is the only file in `@ghost/core` that is browser code, and + * every package that imports from core type-checks core's *sources*. Putting + * DOM in core's compiler options would have fixed core's own build and left + * the worker's failing on `document` — and would have handed every other + * server-side file in this package a `window` it must never touch. + * + * This is the same job `apps/extension/src/content.js` does for the Chrome + * extension, written so a Playwright context can inject it with + * `addInitScript` — which is how the remote cloud browser records. The + * extension keeps its own copy because it is loaded as a raw content script + * with no build step; `recorder.test.ts` pins the security-critical constants + * of the two together so they cannot drift apart silently. + * + * `installRecorder` is serialised with `Function.prototype.toString()`, so it + * must be **completely self-contained**: no imports, no module-scope + * references, no closures. Nothing outside this function body exists in the + * page. TypeScript annotations are erased before serialisation and are safe; + * anything that compiles to a runtime helper (enums, decorators, `class` + * fields with initialisers that lower) is not. + * + * The two rules from the extension hold here identically, and for the same + * reasons: + * + * 1. **A secret never enters the trace.** Not encrypted, not truncated — + * absent. Redaction happens at capture because it is the only place it can + * be done honestly. `redacted: true` records that something was typed + * without recording what. + * 2. **No coordinates, ever.** The user drives the remote browser with a + * mouse, so coordinates cross the socket — but they stop at the page. What + * is recorded is the accessible role and name of whatever the pointer + * landed on, read off the live element while it is still on screen. An + * element that cannot be named is reported as unidentifiable rather than + * approximated by its position. + */ + +/** Name of the Playwright binding the recorder emits through. */ +export const CAPTURE_BINDING = "__ghostCaptureEmit"; + +/** + * Installs the recorder in the current page. + * + * Exported for the type-checker and for direct use in tests; production + * injection goes through `recorderInitScript`. + */ +export function installRecorder(bindingName: string): void { + const w = window as unknown as Record; + if (w.__ghostRecorderInstalled) return; + // Only the top frame. A selector carries no frame identity, so a step + // compiled from an iframe's DOM would resolve against the wrong document on + // replay — worse than not recording it. + if (window.top !== window.self) return; + w.__ghostRecorderInstalled = true; + + const pending: unknown[] = []; + + function flush(): void { + const emit = w[bindingName]; + if (typeof emit !== "function") return; + while (pending.length > 0) { + const event = pending.shift(); + try { + (emit as (arg: unknown) => unknown)(event); + } catch { + // The binding disappears during navigation teardown. Losing the last + // event of a page is better than throwing inside a page's own handler, + // which would break the site the user is demonstrating. + return; + } + } + } + + function send(event: unknown): void { + pending.push(event); + flush(); + } + + // --- sensitivity ------------------------------------------------------- + + const SECRET_HINTS = + /pass(word|code)|otp|one[-_ ]?time|2fa|mfa|cvv|cvc|card[-_ ]?number|credit|account[-_ ]?number|routing|ssn|social[-_ ]?security|\bpin\b|secret|token|security[-_ ]?code/i; + + const SECRET_AUTOCOMPLETE = /current-password|new-password|one-time-code|cc-number|cc-csc|cc-exp/i; + + function isSecretField(el: Element): boolean { + if (!el) return false; + const type = (el.getAttribute("type") || "").toLowerCase(); + if (type === "password") return true; + if (SECRET_AUTOCOMPLETE.test(el.getAttribute("autocomplete") || "")) return true; + + const haystack = [ + el.getAttribute("name"), + el.getAttribute("id"), + el.getAttribute("placeholder"), + el.getAttribute("aria-label"), + accessibleName(el), + ] + .filter(Boolean) + .join(" "); + return SECRET_HINTS.test(haystack); + } + + function isIgnorableField(el: Element): boolean { + return (el.getAttribute("type") || "").toLowerCase() === "hidden"; + } + + // --- accessible role and name ----------------------------------------- + + function implicitRole(el: Element): string | undefined { + const tag = el.tagName.toLowerCase(); + const type = (el.getAttribute("type") || "").toLowerCase(); + if (tag === "a" && el.hasAttribute("href")) return "link"; + if (tag === "button") return "button"; + if (tag === "select") return "combobox"; + if (tag === "textarea") return "textbox"; + if (tag === "input") { + if (["submit", "button", "reset", "image"].indexOf(type) !== -1) return "button"; + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + if (["text", "email", "tel", "url", "search", "password", "number", ""].indexOf(type) !== -1) { + return "textbox"; + } + } + if (/^h[1-6]$/.test(tag)) return "heading"; + return undefined; + } + + function roleOf(el: Element): string | undefined { + return el.getAttribute("role") || implicitRole(el); + } + + function textOf(el: Element | null): string { + return ((el && el.textContent) || "").replace(/\s+/g, " ").trim(); + } + + function accessibleName(el: Element): string { + if (!el || el.nodeType !== 1) return ""; + + const labelledBy = el.getAttribute("aria-labelledby"); + if (labelledBy) { + const parts = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)) + .filter(Boolean) + .map(textOf) + .filter(Boolean); + if (parts.length) return parts.join(" "); + } + + const ariaLabel = el.getAttribute("aria-label"); + if (ariaLabel && ariaLabel.trim()) return ariaLabel.trim(); + + if (el.id) { + const label = document.querySelector('label[for="' + CSS.escape(el.id) + '"]'); + if (label) { + const t = textOf(label); + if (t) return t; + } + } + + const wrapping = el.closest("label"); + if (wrapping) { + const t = textOf(wrapping); + if (t) return t; + } + + const tag = el.tagName.toLowerCase(); + if (tag === "input") { + const type = (el.getAttribute("type") || "").toLowerCase(); + if (["submit", "button", "reset"].indexOf(type) !== -1) { + const v = el.getAttribute("value"); + if (v && v.trim()) return v.trim(); + } + const placeholder = el.getAttribute("placeholder"); + if (placeholder && placeholder.trim()) return placeholder.trim(); + } + + if (tag === "img") { + const alt = el.getAttribute("alt"); + if (alt && alt.trim()) return alt.trim(); + } + + const title = el.getAttribute("title"); + if (title && title.trim()) return title.trim(); + + if ( + ["button", "a", "summary"].indexOf(tag) !== -1 || + el.getAttribute("role") === "button" + ) { + const t = textOf(el); + if (t) return t.slice(0, 200); + } + + return ""; + } + + // --- selector candidates ---------------------------------------------- + + function cssPath(el: Element): string { + const parts: string[] = []; + let node: Element | null = el; + let depth = 0; + while (node && node.nodeType === 1 && depth < 5) { + let part = node.tagName.toLowerCase(); + if (node.id) { + parts.unshift("#" + CSS.escape(node.id)); + break; + } + const parent: Element | null = node.parentElement; + if (parent) { + const current: Element = node; + const siblings = Array.prototype.slice + .call(parent.children) + .filter((c: Element) => c.tagName === current.tagName); + if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")"; + } + parts.unshift(part); + node = node.parentElement; + depth++; + } + return parts.join(" > "); + } + + function testIdOf(el: Element): string | undefined { + return ( + el.getAttribute("data-testid") || + el.getAttribute("data-test-id") || + el.getAttribute("data-test") || + undefined + ); + } + + function selectorCandidates(el: Element): string[] { + const out: string[] = []; + const testId = testIdOf(el); + if (testId) out.push('[data-testid="' + CSS.escape(testId) + '"]'); + if (el.id) out.push("#" + CSS.escape(el.id)); + const name = el.getAttribute("name"); + if (name) out.push(el.tagName.toLowerCase() + '[name="' + CSS.escape(name) + '"]'); + const path = cssPath(el); + if (path) out.push(path); + return out.slice(0, 5); + } + + function describeTarget(el: Element): Record { + return { + role: roleOf(el) || undefined, + name: accessibleName(el) || undefined, + testId: testIdOf(el), + text: textOf(el).slice(0, 120) || undefined, + selectorCandidates: selectorCandidates(el), + tagName: el.tagName.toLowerCase(), + inputType: (el.getAttribute("type") || "").toLowerCase() || undefined, + }; + } + + const now = (): number => Date.now(); + + // --- listeners --------------------------------------------------------- + + // Capture phase: a page that calls stopPropagation in its own handlers would + // otherwise make the workflow unrecordable. + document.addEventListener( + "click", + (e) => { + const raw = e.target; + const el = + raw instanceof Element + ? raw.closest("a,button,[role],input,summary,label") || raw + : null; + if (!el || el.nodeType !== 1) return; + if (isIgnorableField(el)) return; + send({ type: "click", url: location.href, timestamp: now(), target: describeTarget(el) }); + }, + true, + ); + + document.addEventListener( + "change", + (e) => { + const el = e.target; + if (!(el instanceof Element) || isIgnorableField(el)) return; + const tag = el.tagName.toLowerCase(); + + if (tag === "select") { + send({ + type: "select", + url: location.href, + timestamp: now(), + target: describeTarget(el), + value: (el as HTMLSelectElement).value || "", + }); + return; + } + + if (tag === "input" || tag === "textarea") { + const type = (el.getAttribute("type") || "").toLowerCase(); + if (["checkbox", "radio", "submit", "button", "file"].indexOf(type) !== -1) return; + + const secret = isSecretField(el); + const event: Record = { + type: "input", + url: location.href, + timestamp: now(), + target: describeTarget(el), + redacted: secret, + }; + // The whole point: on a secret field the value is not read at all. + if (!secret) event.value = (el as HTMLInputElement).value || ""; + send(event); + } + }, + true, + ); + + document.addEventListener( + "submit", + (e) => { + const el = e.target instanceof Element ? e.target : null; + const event: Record = { + type: "submit", + url: location.href, + timestamp: now(), + }; + if (el) event.target = describeTarget(el); + send(event); + }, + true, + ); + + // SPA navigation: pushState/replaceState fire no event of their own. + let lastUrl = location.href; + const reportNavigation = (): void => { + if (location.href === lastUrl) return; + lastUrl = location.href; + send({ type: "navigate", url: location.href, timestamp: now() }); + }; + + const history = window.history as unknown as Record; + for (const method of ["pushState", "replaceState"]) { + const original = history[method] as (...args: unknown[]) => unknown; + history[method] = function patched(this: unknown, ...args: unknown[]): unknown { + const result = original.apply(this, args); + reportNavigation(); + return result; + }; + } + window.addEventListener("popstate", reportNavigation); + window.addEventListener("hashchange", reportNavigation); + + // The page this document loaded on, so the compiled workflow opens somewhere. + // Emitted on every document, not just the first: a full page load creates a + // new document and re-runs this script, and that transition is exactly the + // `navigate` step a replay needs. + send({ type: "navigate", url: location.href, timestamp: now() }); + + // The binding may be installed after this script runs. Retry the queue on the + // next tick and once the document is interactive rather than dropping the + // opening navigate. + setTimeout(flush, 0); + document.addEventListener("DOMContentLoaded", flush); +} + +/** + * The recorder as a source string, ready for `context.addInitScript`. + * + * Serialising the function rather than keeping a parallel string literal means + * the injected code is the code above — type-checked, lint-covered, and + * reviewable — instead of a blob nothing checks. + */ +export function recorderInitScript(bindingName: string = CAPTURE_BINDING): string { + return `(${installRecorder.toString()})(${JSON.stringify(bindingName)});`; +} diff --git a/cloud/packages/core/src/storage/artifacts.ts b/cloud/packages/core/src/storage/artifacts.ts index 3c00ebc..f5381c2 100644 --- a/cloud/packages/core/src/storage/artifacts.ts +++ b/cloud/packages/core/src/storage/artifacts.ts @@ -1,5 +1,7 @@ import { mkdir, writeFile, readFile, rm } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; +import { Readable } from "node:stream"; +import type { ReadableStream as NodeWebReadableStream } from "node:stream/web"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import { PutObjectCommand, @@ -9,6 +11,7 @@ import { ListObjectsV2Command, S3Client, } from "@aws-sdk/client-s3"; +import { put, get as blobGet, del as blobDel, list as blobList } from "@vercel/blob"; /** * Storage for run artifacts. Returns a stable key stored on the run row; the @@ -161,17 +164,80 @@ class S3ArtifactStore implements ArtifactStore { } } +/** + * Vercel Blob-backed store, for Vercel deployments with no S3-compatible + * bucket. The store is private: `put`/`get`/`del` all require the read-write + * token, so an object's URL alone (unlike S3's presigned links) grants no + * access. That means `signedUrl` cannot produce a browser-fetchable link — + * same as `DiskArtifactStore`, every read goes through the app's own artifact + * route, which calls `get()` server-side and streams the bytes back. + */ +class VercelBlobArtifactStore implements ArtifactStore { + constructor(private readonly token: string) {} + + async put(key: string, body: Buffer, contentType: string): Promise { + await put(key, body, { + access: "private", + addRandomSuffix: false, + allowOverwrite: true, + contentType, + token: this.token, + }); + return key; + } + + async get(key: string): Promise { + const result = await blobGet(key, { access: "private", token: this.token }); + if (!result?.stream) throw new Error(`artifact ${key} not found`); + const chunks: Buffer[] = []; + for await (const chunk of Readable.fromWeb(result.stream as unknown as NodeWebReadableStream)) { + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks); + } + + async delete(key: string): Promise { + await blobDel(key, { token: this.token }); + } + + async deletePrefix(prefix: string): Promise { + let cursor: string | undefined; + do { + const listed = await blobList({ + prefix: prefix.endsWith("/") ? prefix : `${prefix}/`, + cursor, + token: this.token, + }); + if (listed.blobs.length > 0) { + await blobDel( + listed.blobs.map((b) => b.pathname), + { token: this.token }, + ); + } + cursor = listed.hasMore ? listed.cursor : undefined; + } while (cursor); + } + + /** Private-store URLs need the read-write token; nothing is safe to hand a browser. */ + async signedUrl(): Promise { + return null; + } +} + let store: ArtifactStore | undefined; export function artifactStore(): ArtifactStore { if (store) return store; const bucket = process.env.S3_BUCKET; + const blobToken = process.env.BLOB_READ_WRITE_TOKEN; if (bucket && process.env.S3_ACCESS_KEY_ID && process.env.S3_SECRET_ACCESS_KEY) { store = new S3ArtifactStore( bucket, process.env.S3_REGION ?? "auto", process.env.S3_ENDPOINT || undefined, ); + } else if (blobToken) { + store = new VercelBlobArtifactStore(blobToken); } else { const dir = process.env.GHOST_ARTIFACT_DIR ?? resolve(process.cwd(), ".artifacts"); store = new DiskArtifactStore(dir); diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml index 20f9df0..e92e0df 100644 --- a/cloud/pnpm-lock.yaml +++ b/cloud/pnpm-lock.yaml @@ -45,6 +45,9 @@ importers: apps/web: dependencies: + '@auth/prisma-adapter': + specifier: ^2.11.3 + version: 2.11.3(@prisma/client@6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3)) '@ghost/core': specifier: workspace:* version: link:../../packages/core @@ -130,10 +133,16 @@ importers: playwright: specifier: ^1.55.0 version: 1.62.0 + ws: + specifier: ^8.18.0 + version: 8.21.3 devDependencies: '@types/node': specifier: ^22.10.2 version: 22.20.1 + '@types/ws': + specifier: ^8.5.13 + version: 8.18.1 tsup: specifier: ^8.3.5 version: 8.5.1(jiti@2.7.0)(postcss@8.5.24)(tsx@4.23.1)(typescript@5.9.3) @@ -161,6 +170,12 @@ importers: '@sentry/node': specifier: ^10.69.0 version: 10.69.0(@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1)) + '@vercel/blob': + specifier: ^2.7.0 + version: 2.7.0 + dotenv: + specifier: ^16.4.7 + version: 16.6.1 zod: specifier: ^3.24.1 version: 3.25.76 @@ -206,6 +221,11 @@ packages: nodemailer: optional: true + '@auth/prisma-adapter@2.11.3': + resolution: {integrity: sha512-jZbpVAO6PTc9zNtdTWc0RLWG8qap4iMc54/3oWaWbuKdj92wHxzPrs3HivWB2mB9975GPW0l3YM/wFUWiUlTlg==} + peerDependencies: + '@prisma/client': '>=2.26.0 || >=3 || >=4 || >=5 || >=6' + '@aws-sdk/checksums@3.1000.24': resolution: {integrity: sha512-7TWLjypP8kk3savsDBRuhZJx7mBuFFA2136BQhwwLllsAnO4Tmq/p+SXZaNxbuulkzUFz3BZzj0bb4YzexZcNQ==} engines: {node: '>=20.0.0'} @@ -1524,6 +1544,9 @@ packages: '@types/react@19.2.17': resolution: {integrity: sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@typescript-eslint/eslint-plugin@8.65.0': resolution: {integrity: sha512-IEgob78X12rHpUmtcwFsXhZdVGJtwTVP8FiCLZkR6GlYVrl2PcuB+KhCE5BlVC/eQpQnu8WXRtkHZuPar+gCRA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -1703,6 +1726,21 @@ packages: cpu: [x64] os: [win32] + '@vercel/blob@2.7.0': + resolution: {integrity: sha512-cWo8XeRI+eq+pTAQOGZEljSeARw/PZfvNCez9xioGcz/KFvwTR5ESGuKd4wPkGnhbPXyxJJMEgP4oOkwQ4uNGQ==} + engines: {node: '>=20.0.0'} + + '@vercel/cli-config@0.2.2': + resolution: {integrity: sha512-kAy35eymNzRBfmcqEViVQge0KJ59FYEKsPqGNCSJQ7M9HTuFGWd3qPcZos1A5cZpAWRBdiYe82Bcz9P7pIZvUQ==} + + '@vercel/cli-exec@1.0.1': + resolution: {integrity: sha512-g9XerViJ/paZujufXYcu5XYI2vU2rtB4sgdpjUHde5RnOkdmpu0ngH46LCFGHoPXO/C+qDPSczIHIRN+8Q2YKQ==} + engines: {node: '>= 18'} + + '@vercel/oidc@3.8.2': + resolution: {integrity: sha512-nmVSeQ7tewCkqYBNB/MNL8aPB9sTvtjvqIE6+U17U4IuTyehuWVERDlWrSn0DVfiFAstRlRkGLHBlKHB2FyzbQ==} + engines: {node: '>= 20'} + '@vitest/expect@2.1.9': resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} @@ -1806,6 +1844,9 @@ packages: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + available-typed-arrays@1.0.7: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} @@ -2231,6 +2272,10 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + execa@5.1.1: + resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} + engines: {node: '>=10'} + expect-type@1.4.0: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} @@ -2325,6 +2370,10 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-stream@6.0.1: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + get-symbol-description@1.1.0: resolution: {integrity: sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==} engines: {node: '>= 0.4'} @@ -2386,6 +2435,10 @@ packages: resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} engines: {node: '>= 0.4'} + human-signals@2.1.0: + resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} + engines: {node: '>=10.17.0'} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -2430,6 +2483,10 @@ packages: resolution: {integrity: sha512-wa56o2/ElJMYqjCjGkXri7it5FbebW5usLw/nPmCMs5DeZ7eziSYZhSmPRn0txqeW4LnAmQQU7FgqLpsEFKM4A==} engines: {node: '>= 0.4'} + is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + is-bun-module@2.0.0: resolution: {integrity: sha512-gNCGbnnnnFAUGKeZ9PdbyeGYJqewpmc2aKHUEMO5nQPWU9lOmv7jcmQIv+qHD8fXW6W7qfuCwX4rY9LNRjXrkQ==} @@ -2477,6 +2534,9 @@ packages: resolution: {integrity: sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==} engines: {node: '>= 0.4'} + is-node-process@1.2.0: + resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==} + is-number-object@1.1.1: resolution: {integrity: sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==} engines: {node: '>= 0.4'} @@ -2497,6 +2557,10 @@ packages: resolution: {integrity: sha512-ISWac8drv4ZGfwKl5slpHG9OwPNty4jOWPRIhBpxOoD+hqITiwuipOQ2bNthAzwA3B4fIjO4Nln74N0S9byq8A==} engines: {node: '>= 0.4'} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-string@1.1.1: resolution: {integrity: sha512-BtEeSsoaQjlSPBemMQIrY1MY0uM6vnS1g5fmufYOtnxLGUZM2178PKbhsk7Ffv58IX+ZtcvoGwccYsh0PglkAA==} engines: {node: '>= 0.4'} @@ -2535,6 +2599,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + jose@6.2.4: resolution: {integrity: sha512-N8acGzVsQy6M/fjFcxtysNc4Q379TcM5dM/qKkNtsHFji88yANnXTr7BLeP75iPnFwBfQzM/jg2BZ9+HZrHCZA==} @@ -2695,6 +2762,9 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -2707,6 +2777,10 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + minimatch@10.2.6: resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} engines: {node: 18 || 20 || >=22} @@ -2800,6 +2874,10 @@ packages: resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} hasBin: true + npm-run-path@4.0.1: + resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} + engines: {node: '>=8'} + nypm@0.6.9: resolution: {integrity: sha512-zxlE2yvSWZWmHcNdT3+5zV2lrCogeE9YOklHrR3dFjqutq5wO7GFDYLFDRXLsYnJzwvy/im9fYoxePvS0VTW0w==} engines: {node: '>=18'} @@ -2843,10 +2921,18 @@ packages: ohash@2.0.11: resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} + onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} + os-paths@4.4.0: + resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} + engines: {node: '>= 6.0'} + own-keys@1.0.2: resolution: {integrity: sha512-19YVAg7T+WTrxggPukVq7DjTv6+PJ867TmhCvBsYwmbFCsZd344rq2Ld1p0wo8f8Qrrhgp82c6FJRqdXWtSEhg==} engines: {node: '>= 0.4'} @@ -3043,6 +3129,10 @@ packages: engines: {node: '>= 0.4'} hasBin: true + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -3125,6 +3215,9 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -3180,6 +3273,10 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -3227,6 +3324,10 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + throttleit@2.1.0: + resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} + engines: {node: '>=18'} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3339,6 +3440,10 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@6.28.0: + resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==} + engines: {node: '>=18.17'} + unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} @@ -3436,6 +3541,26 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + ws@8.21.3: + resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + xdg-app-paths@5.5.1: + resolution: {integrity: sha512-hI3flOB4PLZIy5prbtTpirobtPE2ZtZ52szO+2mM9Efp6ErM398La+C1lIpNWDfNoQk+6Lsi6nMcCwVB7pxeMQ==} + engines: {node: '>= 6.0'} + + xdg-portable@7.3.0: + resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} + engines: {node: '>= 6.0'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -3443,6 +3568,9 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.1.11: + resolution: {integrity: sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==} + snapshots: '@alloc/quick-lru@5.2.0': {} @@ -3479,6 +3607,15 @@ snapshots: preact: 10.24.3 preact-render-to-string: 6.5.11(preact@10.24.3) + '@auth/prisma-adapter@2.11.3(@prisma/client@6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3))': + dependencies: + '@auth/core': 0.41.3 + '@prisma/client': 6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3) + transitivePeerDependencies: + - '@simplewebauthn/browser' + - '@simplewebauthn/server' + - nodemailer + '@aws-sdk/checksums@3.1000.24': dependencies: '@aws-sdk/core': 3.977.4 @@ -4508,6 +4645,10 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/ws@8.18.1': + dependencies: + '@types/node': 22.20.1 + '@typescript-eslint/eslint-plugin@8.65.0(@typescript-eslint/parser@8.65.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 @@ -4669,6 +4810,30 @@ snapshots: '@unrs/resolver-binding-win32-x64-msvc@1.12.2': optional: true + '@vercel/blob@2.7.0': + dependencies: + '@vercel/oidc': 3.8.2 + async-retry: 1.3.3 + is-buffer: 2.0.5 + is-node-process: 1.2.0 + throttleit: 2.1.0 + undici: 6.28.0 + + '@vercel/cli-config@0.2.2': + dependencies: + xdg-app-paths: 5.5.1 + zod: 4.1.11 + + '@vercel/cli-exec@1.0.1': + dependencies: + execa: 5.1.1 + + '@vercel/oidc@3.8.2': + dependencies: + '@vercel/cli-config': 0.2.2 + '@vercel/cli-exec': 1.0.1 + jose: 5.10.0 + '@vitest/expect@2.1.9': dependencies: '@vitest/spy': 2.1.9 @@ -4807,6 +4972,10 @@ snapshots: async-function@1.0.0: {} + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + available-typed-arrays@1.0.7: dependencies: possible-typed-array-names: 1.1.0 @@ -5432,6 +5601,18 @@ snapshots: esutils@2.0.3: {} + execa@5.1.1: + dependencies: + cross-spawn: 7.0.6 + get-stream: 6.0.1 + human-signals: 2.1.0 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + expect-type@1.4.0: {} exsolve@1.1.1: {} @@ -5534,6 +5715,8 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.2 + get-stream@6.0.1: {} + get-symbol-description@1.1.0: dependencies: call-bound: 1.0.4 @@ -5594,6 +5777,8 @@ snapshots: dependencies: function-bind: 1.1.2 + human-signals@2.1.0: {} + ignore@5.3.2: {} ignore@7.0.6: {} @@ -5652,6 +5837,8 @@ snapshots: call-bound: 1.0.4 has-tostringtag: 1.0.2 + is-buffer@2.0.5: {} + is-bun-module@2.0.0: dependencies: semver: 7.8.5 @@ -5699,6 +5886,8 @@ snapshots: is-negative-zero@2.0.3: {} + is-node-process@1.2.0: {} + is-number-object@1.1.1: dependencies: call-bound: 1.0.4 @@ -5719,6 +5908,8 @@ snapshots: dependencies: call-bound: 1.0.4 + is-stream@2.0.1: {} + is-string@1.1.1: dependencies: call-bound: 1.0.4 @@ -5760,6 +5951,8 @@ snapshots: jiti@2.7.0: {} + jose@5.10.0: {} + jose@6.2.4: {} joycon@3.1.1: {} @@ -5881,6 +6074,8 @@ snapshots: math-intrinsics@1.1.0: {} + merge-stream@2.0.0: {} + merge2@1.4.1: {} meriyah@6.1.4: {} @@ -5890,6 +6085,8 @@ snapshots: braces: 3.0.3 picomatch: 2.3.2 + mimic-fn@2.1.0: {} + minimatch@10.2.6: dependencies: brace-expansion: 5.0.8 @@ -5985,6 +6182,10 @@ snapshots: detect-libc: 2.1.2 optional: true + npm-run-path@4.0.1: + dependencies: + path-key: 3.1.1 + nypm@0.6.9: dependencies: citty: 0.2.2 @@ -6037,6 +6238,10 @@ snapshots: ohash@2.0.11: {} + onetime@5.1.2: + dependencies: + mimic-fn: 2.1.0 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -6046,6 +6251,8 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 + os-paths@4.4.0: {} + own-keys@1.0.2: dependencies: call-bound: 1.0.4 @@ -6224,6 +6431,8 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + retry@0.13.1: {} + reusify@1.1.0: {} rollup@4.62.3: @@ -6378,6 +6587,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@3.0.7: {} + source-map-js@1.2.1: {} source-map@0.6.1: {} @@ -6450,6 +6661,8 @@ snapshots: strip-bom@3.0.0: {} + strip-final-newline@2.0.0: {} + strip-json-comments@3.1.1: {} styled-jsx@5.1.6(react@19.2.8): @@ -6487,6 +6700,8 @@ snapshots: dependencies: any-promise: 1.3.0 + throttleit@2.1.0: {} + tinybench@2.9.0: {} tinyexec@0.3.2: {} @@ -6618,6 +6833,8 @@ snapshots: undici-types@6.21.0: {} + undici@6.28.0: {} + unrs-resolver@1.12.2: dependencies: napi-postinstall: 0.3.4 @@ -6764,6 +6981,19 @@ snapshots: word-wrap@1.2.5: {} + ws@8.21.3: {} + + xdg-app-paths@5.5.1: + dependencies: + os-paths: 4.4.0 + xdg-portable: 7.3.0 + + xdg-portable@7.3.0: + dependencies: + os-paths: 4.4.0 + yocto-queue@0.1.0: {} zod@3.25.76: {} + + zod@4.1.11: {} diff --git a/cloud/scripts/demo.sh b/cloud/scripts/demo.sh index dd45467..3ac0730 100755 --- a/cloud/scripts/demo.sh +++ b/cloud/scripts/demo.sh @@ -15,11 +15,8 @@ set -euo pipefail cd "$(dirname "$0")/.." ROOT="$PWD" -bold() { printf '\033[1m%s\033[0m\n' "$1"; } -info() { printf ' \033[36m→\033[0m %s\n' "$1"; } -ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; } -warn() { printf ' \033[33m!\033[0m %s\n' "$1"; } -die() { printf ' \033[31m✗\033[0m %s\n' "$1" >&2; exit 1; } +# shellcheck source=scripts/lib.sh +source "$ROOT/scripts/lib.sh" bold "Ghost Cloud demo" echo @@ -36,7 +33,7 @@ ok "node $(node -v), pnpm $(pnpm -v)" # Without that key every approved run ends INCIDENT instead of SUCCEEDED, which # is precisely the flow this demo exists to show. if [ -f .env ]; then - ok ".env already present (left untouched)" + ok ".env already present" else cp .env.example .env ok "created .env from .env.example" @@ -56,33 +53,19 @@ if grep -q '^GHOST_ARTIFACT_DIR=""' .env 2>/dev/null; then fi mkdir -p "$ROOT/.artifacts" -# --- 3. Data plane ---------------------------------------------------------- -# Reuse whatever is already listening; only reach for Docker if nothing is. -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") >/dev/null 2>&1; } - -if port_open 5432 && port_open 6379; then - ok "Postgres :5432 and Redis :6379 already reachable" -elif command -v docker >/dev/null && docker info >/dev/null 2>&1; then - info "starting Postgres + Redis via docker compose" - docker compose up -d - for _ in $(seq 1 30); do - port_open 5432 && port_open 6379 && break - sleep 1 - done - port_open 5432 || die "Postgres did not come up on :5432. Check: docker compose logs postgres" - port_open 6379 || die "Redis did not come up on :6379. Check: docker compose logs redis" - ok "Postgres and Redis are up" -else - die "Need Postgres on :5432 and Redis on :6379. - Either start Docker and re-run, or point DATABASE_URL and REDIS_URL in - cloud/.env at instances you already have." -fi - -# --- 4. Dependencies, schema, browser --------------------------------------- +# --- 3. Dependencies -------------------------------------------------------- +# Before the data plane: probing Postgres runs the Prisma CLI, which has to be +# installed first. info "installing dependencies" pnpm install --silent ok "dependencies installed" +# --- 4. Data plane, schema, browser ----------------------------------------- +# `ensure_data_plane` reuses a Postgres/Redis that genuinely answers, starts +# Docker otherwise, and repairs .env to match. It never trusts an open port — +# see scripts/lib.sh. +ensure_data_plane + info "applying database schema" # `migrate deploy`, not `migrate dev` — the latter is interactive and will offer # to reset the database. diff --git a/cloud/scripts/doctor.sh b/cloud/scripts/doctor.sh new file mode 100755 index 0000000..1c78159 --- /dev/null +++ b/cloud/scripts/doctor.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# +# Ghost Cloud — say plainly what is wrong. +# +# cd cloud && pnpm doctor +# +# Read-only. It changes nothing; `pnpm demo` is what repairs. This exists +# because the way Ghost breaks locally is almost never a visible error: the web +# app renders fine while pointed at a database that denies it, or while no +# worker is running to execute anything, so "Run" simply does nothing forever. +# Both are one-line diagnoses that were previously invisible. +set -uo pipefail + +cd "$(dirname "$0")/.." +ROOT="$PWD" +# shellcheck source=scripts/lib.sh +source "$ROOT/scripts/lib.sh" + +PROBLEMS=0 +note_problem() { PROBLEMS=$((PROBLEMS + 1)); } + +bold "Ghost Cloud doctor" +echo + +# --- config ---------------------------------------------------------------- +if [ -f .env ]; then + ok ".env present" +else + fail ".env missing — run: pnpm demo" + note_problem + exit 1 +fi + +ARTIFACT_DIR="$(env_get GHOST_ARTIFACT_DIR || true)" +case "$ARTIFACT_DIR" in + /*) [ -d "$ARTIFACT_DIR" ] && ok "GHOST_ARTIFACT_DIR $ARTIFACT_DIR" || { + fail "GHOST_ARTIFACT_DIR points at $ARTIFACT_DIR, which does not exist" + note_problem + } ;; + "") fail "GHOST_ARTIFACT_DIR is empty — run screenshots will 404, so approvals show no evidence" + note_problem ;; + *) fail "GHOST_ARTIFACT_DIR must be an absolute path (got '$ARTIFACT_DIR')" + note_problem ;; +esac + +SESSION_KEY="$(env_get GHOST_SESSION_KEY || true)" +if [ -z "$SESSION_KEY" ]; then + warn "GHOST_SESSION_KEY is empty — approved runs will end INCIDENT, not SUCCEEDED" + note_problem +else + ok "GHOST_SESSION_KEY set" +fi + +# --- data plane ------------------------------------------------------------ +DB_URL="$(env_get DATABASE_URL || true)" +DB_PORT="$(port_of_url "$DB_URL")" +if [ -z "$DB_URL" ]; then + fail "DATABASE_URL is not set" + note_problem +elif pg_usable "$DB_URL"; then + ok "Postgres :$DB_PORT accepts the ghost user" +else + fail "Postgres on :$DB_PORT is not usable by Ghost." + if port_open "${DB_PORT:-5432}"; then + fail " Something IS listening there — most likely a different Postgres" + fail " (Homebrew, Postgres.app, another project). An open port is not enough." + else + fail " Nothing is listening on :$DB_PORT." + fi + fail " Fix: pnpm demo (it finds a port that actually works and repairs .env)" + note_problem +fi + +REDIS_URL="$(env_get REDIS_URL || true)" +REDIS_PORT="$(port_of_url "$REDIS_URL")" +: "${REDIS_PORT:=6379}" +if redis_usable "$REDIS_PORT"; then + ok "Redis :$REDIS_PORT answers PING" +else + fail "Redis on :$REDIS_PORT did not answer PING — runs can be created but never queued" + fail " Fix: pnpm demo" + note_problem +fi + +# --- processes ------------------------------------------------------------- +if port_open 3000; then + ok "web app listening on :3000" +else + warn "nothing on :3000 — the web app is not running (pnpm dev)" + note_problem +fi + +# The worker binds no port, so presence is judged by process, and by whether +# anything it should have consumed is still sitting in the queue. +if pgrep -f "@ghost/worker|worker/dist/index.js|worker/src/index.ts" >/dev/null 2>&1; then + ok "worker process running" +else + fail "NO WORKER RUNNING. Runs will stay PENDING forever — nothing executes." + fail " Fix: pnpm dev (starts web and worker together)" + fail " Or: pnpm --filter @ghost/worker dev" + note_problem +fi + +echo +if [ "$PROBLEMS" -eq 0 ]; then + ok "no problems found" +else + bold "$PROBLEMS problem(s) above." +fi +exit 0 diff --git a/cloud/scripts/lib.sh b/cloud/scripts/lib.sh new file mode 100755 index 0000000..0f4cc9d --- /dev/null +++ b/cloud/scripts/lib.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# +# Shared setup logic for `pnpm demo` and `pnpm doctor`. +# +# The one idea in this file: **never conclude a service is usable because a port +# is open.** The original demo script tested `port_open 5432` and, on any +# machine with an unrelated Postgres installed — Homebrew's, Postgres.app, +# another project's container — declared success, wrote that port into .env, and +# handed the user an app whose every database call was denied. The failure +# surfaced later as unexplained 500s, so the setup step that caused it looked +# like the one thing that had gone right. +# +# So each check here actually speaks the protocol: Postgres is probed by running +# a query as the `ghost` user, Redis by sending a PING and reading the PONG. + +# --- output ---------------------------------------------------------------- + +bold() { printf '\033[1m%s\033[0m\n' "$1"; } +info() { printf ' \033[36m→\033[0m %s\n' "$1"; } +ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; } +warn() { printf ' \033[33m!\033[0m %s\n' "$1"; } +fail() { printf ' \033[31m✗\033[0m %s\n' "$1"; } +die() { fail "$1"; exit 1; } + +# --- ports ----------------------------------------------------------------- + +port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") >/dev/null 2>&1; } + +# --- .env ------------------------------------------------------------------ + +# Read a single value out of .env, stripping surrounding quotes. +env_get() { + local key="$1" + [ -f .env ] || return 1 + sed -n "s/^${key}=//p" .env | head -1 | sed -e 's/^"//' -e 's/"$//' +} + +# Set (or add) a key in .env. Portable across BSD and GNU sed. +env_set() { + local key="$1" value="$2" + touch .env + if grep -q "^${key}=" .env; then + # `|` as the delimiter, and the value escaped, so a URL's slashes and any + # `&` in a password cannot corrupt the file being repaired. + local escaped + escaped="$(printf '%s' "$value" | sed -e 's/[\\|&]/\\&/g')" + sed -i.bak "s|^${key}=.*|${key}=\"${escaped}\"|" .env + rm -f .env.bak + else + printf '%s="%s"\n' "$key" "$value" >> .env + fi +} + +# --- service probes -------------------------------------------------------- + +pg_url_for_port() { printf 'postgresql://ghost:ghost@localhost:%s/ghost?schema=public' "$1"; } + +# True when a real Postgres answers a query as the ghost user on this URL. +pg_usable() { + local url="$1" + echo 'SELECT 1;' | pnpm --filter @ghost/core exec prisma db execute --stdin --url "$url" \ + >/dev/null 2>&1 +} + +# True when Redis answers PING. Inline commands are valid RESP input, so this +# needs no client — and unlike an open port, a PONG proves it is Redis. +redis_usable() { + local port="$1" reply="" + exec 3<>"/dev/tcp/127.0.0.1/${port}" 2>/dev/null || return 1 + printf 'PING\r\n' >&3 + # `read -t` bounds the wait: a port held open by something that never + # answers must fail the check rather than hang the setup. + IFS= read -r -t 3 reply <&3 || true + exec 3<&- 3>&- + case "$reply" in *PONG*) return 0 ;; *) return 1 ;; esac +} + +port_of_url() { + # Pulls the port out of scheme://user:pass@host:PORT/db, empty if absent. + printf '%s' "$1" | sed -n 's|.*://[^/]*:\([0-9][0-9]*\)/.*|\1|p' +} + +# --- data plane ------------------------------------------------------------ + +# Find a Postgres that actually accepts the ghost credentials, starting Docker +# only if none does. Echoes the chosen port. +# +# Candidates in order: whatever .env already says (so a deliberate choice is +# kept), the compose default, then the fallback the compose file uses when 5432 +# is taken. Trying the configured value first also means a working setup +# re-runs with no changes at all. +resolve_pg_port() { + local configured candidates=() c + configured="$(port_of_url "$(env_get DATABASE_URL || true)")" + [ -n "$configured" ] && candidates+=("$configured") + candidates+=(5432 55432) + + for c in "${candidates[@]}"; do + if pg_usable "$(pg_url_for_port "$c")"; then + printf '%s' "$c" + return 0 + fi + done + return 1 +} + +resolve_redis_port() { + local configured candidates=() c + configured="$(port_of_url "$(env_get REDIS_URL || true)")" + [ -n "$configured" ] && candidates+=("$configured") + candidates+=(6379 56379) + + for c in "${candidates[@]}"; do + if redis_usable "$c"; then + printf '%s' "$c" + return 0 + fi + done + return 1 +} + +docker_available() { command -v docker >/dev/null && docker info >/dev/null 2>&1; } + +# Bring up Postgres and Redis on ports that are genuinely free, and write those +# ports into .env. Sets PG_PORT and REDIS_PORT for the caller. +ensure_data_plane() { + PG_PORT="$(resolve_pg_port || true)" + REDIS_PORT="$(resolve_redis_port || true)" + + if [ -n "$PG_PORT" ] && [ -n "$REDIS_PORT" ]; then + ok "Postgres :$PG_PORT and Redis :$REDIS_PORT answering as ghost" + else + docker_available || die "Need Postgres and Redis. + Start Docker and re-run, or point DATABASE_URL and REDIS_URL in cloud/.env + at instances you already have. A Postgres that is merely running is not + enough — it must accept user 'ghost' on database 'ghost'." + + # Bind to a port nothing else holds. Publishing onto an occupied port fails + # the whole compose run, and reusing one occupied by a *different* Postgres + # is the bug this file exists to prevent. + if [ -z "$PG_PORT" ]; then + GHOST_PG_PORT=5432 + port_open 5432 && GHOST_PG_PORT=55432 + export GHOST_PG_PORT + info "starting Postgres on :$GHOST_PG_PORT" + fi + if [ -z "$REDIS_PORT" ]; then + GHOST_REDIS_PORT=6379 + port_open 6379 && GHOST_REDIS_PORT=56379 + export GHOST_REDIS_PORT + info "starting Redis on :$GHOST_REDIS_PORT" + fi + + docker compose up -d >/dev/null + + local i + for i in $(seq 1 45); do + [ -n "$PG_PORT" ] || PG_PORT="$(pg_usable "$(pg_url_for_port "${GHOST_PG_PORT:-5432}")" && echo "${GHOST_PG_PORT:-5432}" || true)" + [ -n "$REDIS_PORT" ] || REDIS_PORT="$(redis_usable "${GHOST_REDIS_PORT:-6379}" && echo "${GHOST_REDIS_PORT:-6379}" || true)" + [ -n "$PG_PORT" ] && [ -n "$REDIS_PORT" ] && break + sleep 1 + done + + [ -n "$PG_PORT" ] || die "Postgres did not accept the ghost user on :${GHOST_PG_PORT:-5432}. + Check: docker compose logs postgres" + [ -n "$REDIS_PORT" ] || die "Redis did not answer PING on :${GHOST_REDIS_PORT:-6379}. + Check: docker compose logs redis" + ok "Postgres :$PG_PORT and Redis :$REDIS_PORT are up" + fi + + # Write back what actually worked, repairing a stale .env rather than leaving + # the app pointed at a database that denies it. + local want_db want_redis + want_db="$(pg_url_for_port "$PG_PORT")" + want_redis="redis://localhost:${REDIS_PORT}" + [ "$(env_get DATABASE_URL || true)" = "$want_db" ] || { env_set DATABASE_URL "$want_db"; ok "pointed DATABASE_URL at :$PG_PORT"; } + [ "$(env_get REDIS_URL || true)" = "$want_redis" ] || { env_set REDIS_URL "$want_redis"; ok "pointed REDIS_URL at :$REDIS_PORT"; } +} diff --git a/cloud/turbo.json b/cloud/turbo.json index 1abe0c5..c84ac09 100644 --- a/cloud/turbo.json +++ b/cloud/turbo.json @@ -21,7 +21,11 @@ "APP_URL", "NODE_ENV", "GHOST_ACCESS_TOKEN", - "GHOST_API_URL" + "GHOST_API_URL", + "GHOST_CAPTURE_KEY", + "GHOST_CAPTURE_PORT", + "GHOST_CAPTURE_MAX_SESSIONS", + "NEXT_PUBLIC_GHOST_CAPTURE_URL" ], "globalPassThroughEnv": [ "PLAYWRIGHT_BROWSERS_PATH", @@ -31,21 +35,33 @@ ], "tasks": { "build": { - "dependsOn": ["^build"], - "outputs": [".next/**", "!.next/cache/**", "dist/**"] + "dependsOn": [ + "^build" + ], + "outputs": [ + ".next/**", + "!.next/cache/**", + "dist/**" + ] }, "dev": { "cache": false, "persistent": true }, "lint": { - "dependsOn": ["^build"] + "dependsOn": [ + "^build" + ] }, "typecheck": { - "dependsOn": ["^build"] + "dependsOn": [ + "^build" + ] }, "test": { - "dependsOn": ["^build"] + "dependsOn": [ + "^build" + ] } } }