test(worker): run Temporal sandbox tests and gate replay compatibility

.github/workflows/ci.yml

@@ -15,6 +15,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  temporal-replay:
+    name: Temporal sandbox and replay
+    runs-on: ubuntu-latest
+    timeout-minutes: 12
+    env:
+      ELECTRON_SKIP_BINARY_DOWNLOAD: "1"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm ci
+      - name: Build fixture dependencies
+        run: >-
+          npm run build -w @space/contracts &&
+          npm run build -w @space/memory-graph &&
+          npm run build -w @space/runtime
+      # Independent of export provenance so fork PRs exercise replay as well.
+      # The ordinary worker/public suites also discover this test file.
+      - name: Replay every baseline and exercise real workflow scheduling
+        timeout-minutes: 3
+        run: npm test -w @space/worker -- tests/workflow-sandbox.test.ts
+
   public-quality:
     name: Sanitized public quality gates
     runs-on: ubuntu-latest

apps/worker/package.json

@@ -10,7 +10,8 @@
     "start": "node dist/worker.js",
     "build": "tsc -p tsconfig.json",
     "check": "tsc -p tsconfig.json --noEmit",
-    "test": "vitest run"
+    "test": "vitest run",
+    "temporal:record-histories": "node scripts/record-workflow-histories.mjs"
   },
   "dependencies": {
     "@space/codex-app-server": "0.2.5",
@@ -26,6 +27,8 @@
     "undici": "8.12.3"
   },
   "devDependencies": {
+    "@temporalio/proto": "1.24.0",
+    "@temporalio/testing": "1.24.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.12"
   },

apps/worker/scripts/record-workflow-histories.mjs

@@ -0,0 +1,35 @@
+import { mkdtemp, mkdir, readdir, copyFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { createRequire } from "node:module";
+import { spawnSync } from "node:child_process";
+
+const version = process.argv[2];
+if (!version || !/^[a-z0-9][a-z0-9-]{0,70}$/.test(version)) throw new Error("Usage: npm run temporal:record-histories -- <reviewed-patch-version>");
+const root = fileURLToPath(new URL("../tests/fixtures/workflow-histories/", import.meta.url));
+const destination = path.join(root, version);
+try { await readdir(destination); throw new Error("History versions are immutable; choose a new reviewed patch version"); }
+catch (error) { if (error.code !== "ENOENT") throw error; }
+const staging = await mkdtemp(path.join(tmpdir(), "space-temporal-history-"));
+try {
+  const require = createRequire(import.meta.url);
+  const vitest = path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs");
+  const result = spawnSync(process.execPath, [vitest, "run", "tests/workflow-sandbox.test.ts", "--reporter=verbose"], {
+    cwd: fileURLToPath(new URL("../", import.meta.url)), stdio: "inherit", timeout: 180_000,
+    env: { ...process.env, SPACE_TEMPORAL_RECORD_DIR: staging }
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) throw new Error(`Recording tests failed (${result.status}); no baseline was published`);
+  const files = (await readdir(staging)).filter((name) => name.endsWith(".json"));
+  if (files.length < 9) throw new Error("Incomplete recording: need every workflow and the legacy supervisor");
+  await mkdir(root, { recursive: true });
+  await mkdir(destination);
+  for (const name of files) await copyFile(path.join(staging, name), path.join(destination, name), 1);
+  console.log(`Recorded ${files.length} replay-verified synthetic histories in ${destination}`);
+} finally {
+  const owned = path.resolve(staging);
+  if (!owned.startsWith(path.resolve(tmpdir()) + path.sep) || !path.basename(owned).startsWith("space-temporal-history-"))
+    throw new Error("Refusing to remove an unowned recording directory");
+  await rm(owned, { recursive: true, force: true });
+}

apps/worker/tests/fixtures/workflow-histories/README.md

@@ -0,0 +1,42 @@
+# Workflow replay baselines
+
+These histories contain synthetic inputs and activities from the real Temporal
+time-skipping server. Normal worker tests replay every committed version against
+the current production workflow bundles. The suite also exercises a test-only
+extra timer and requires a `DeterminismViolationError`, proving that replay is
+an effective compatibility gate.
+
+The test-only legacy bundle suppresses `room-background-continuation-v1` while
+recording. Its histories are replayed against the unchanged production bundle,
+where the absence of the patch marker selects the legacy implementation.
+
+Run from `apps/worker` with Node 22 and locked dependencies installed:
+
+```sh
+npm test -- tests/workflow-sandbox.test.ts
+npm run temporal:record-histories -- reviewed-patch-v2
+```
+
+The first run downloads the SDK-compatible Temporal test server. Subsequent
+runs reuse its cache. No Docker, database, provider or operator credentials are
+used. The server and workers are stopped after the suite, including failures.
+The committed baselines cover all eight workflow types, continuation runs,
+the pre-patch supervisor branch and prepared work. The 250-poll and
+300-iteration boundaries run as live sandbox tests but are not committed as
+replay histories, keeping each version reviewable. Each JSON event occupies one
+line. The public CI has an independent Ubuntu replay job, including fork PRs,
+with a three-minute limit on the sandbox/replay step. Fixture dependencies are
+built before that step. The ordinary worker and public suites also run these
+tests.
+
+**Recording is an explicit migration operation.** After the initial baseline,
+add a new version only together with a reviewed, replay-safe `patched()` change.
+Keep all previous versions and their old branches. Never regenerate a baseline
+to hide an incompatibility. The recorder replays all existing versions first,
+refuses an existing destination, and publishes only after the complete sandbox
+suite and replay of the new histories pass. Ordinary tests never write fixtures.
+
+Worker identities and failure source paths are normalized to fixed synthetic
+strings before publication; the normalized histories are replayed too. Activity
+results, commands, timers, signals, patch markers and continuation inputs are
+preserved.

apps/worker/tests/fixtures/workflow-histories/initial-v1/action-failure--0.json

@@ -0,0 +1,23 @@
+{
+  "events": [
+    {"eventId":"1","eventTime":{"seconds":"4946587763","nanos":784000000},"eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_STARTED","workflowExecutionStartedEventAttributes":{"workflowType":{"name":"roomAgentActionWorkflow"},"taskQueue":{"name":"sandbox-60-main","kind":"TASK_QUEUE_KIND_NORMAL"},"input":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"eyJicmlkZ2UiOnsicm9vbUlkIjoicm9vbTpzeW50aGV0aWMiLCJtaXNzaW9uSWQiOiJtaXNzaW9uOmFjdGlvbiJ9LCJ0cmFjZUlkIjoidHJhY2U6YWN0aW9uIn0="}]},"workflowExecutionTimeout":{"seconds":"315360000"},"workflowRunTimeout":{"seconds":"315360000"},"workflowTaskTimeout":{"seconds":"10"},"originalExecutionRunId":"702e7e85-dce7-4c7b-9bae-78c978d36b16","identity":"space-synthetic-tests","firstExecutionRunId":"702e7e85-dce7-4c7b-9bae-78c978d36b16","attempt":1,"firstWorkflowTaskBackoff":{},"header":{}}},
+    {"eventId":"2","eventTime":{"seconds":"4946587763","nanos":784000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"sandbox-60-main","kind":"TASK_QUEUE_KIND_NORMAL"},"startToCloseTimeout":{"seconds":"10"},"attempt":1}},
+    {"eventId":"3","eventTime":{"seconds":"4946587763","nanos":784000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"2","identity":"space-synthetic-tests","historySizeBytes":"367"}},
+    {"eventId":"4","eventTime":{"seconds":"4946587764","nanos":343000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"2","identity":"space-synthetic-tests","binaryChecksum":"@temporalio/worker@1.24.0+1bda3f8171c686154b9ae83f1228126364d43fe31da1622f07aff7a2bdb6b26a","sdkMetadata":{"coreUsedFlags":[3,1,2],"sdkName":"temporal-typescript","sdkVersion":"1.24.0"},"meteringMetadata":{}}},
+    {"eventId":"5","eventTime":{"seconds":"4946587764","nanos":343000000},"eventType":"EVENT_TYPE_ACTIVITY_TASK_SCHEDULED","activityTaskScheduledEventAttributes":{"activityId":"1","activityType":{"name":"runRoomAgentBackgroundAction"},"taskQueue":{"name":"sandbox-60-main","kind":"TASK_QUEUE_KIND_NORMAL"},"header":{},"input":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"eyJicmlkZ2UiOnsicm9vbUlkIjoicm9vbTpzeW50aGV0aWMiLCJtaXNzaW9uSWQiOiJtaXNzaW9uOmFjdGlvbiJ9LCJ0cmFjZUlkIjoidHJhY2U6YWN0aW9uIn0="}]},"scheduleToCloseTimeout":{"seconds":"315360000"},"scheduleToStartTimeout":{"seconds":"315360000"},"startToCloseTimeout":{"seconds":"86400"},"heartbeatTimeout":{"seconds":"15"},"workflowTaskCompletedEventId":"3","retryPolicy":{"initialInterval":{"seconds":"1"},"backoffCoefficient":2,"maximumInterval":{"seconds":"100"},"maximumAttempts":5}}},
+    {"eventId":"6","eventTime":{"seconds":"4946587779","nanos":379000000},"eventType":"EVENT_TYPE_ACTIVITY_TASK_STARTED","activityTaskStartedEventAttributes":{"scheduledEventId":"5","identity":"space-synthetic-tests","attempt":5}},
+    {"eventId":"7","eventTime":{"seconds":"4946587779","nanos":381000000},"eventType":"EVENT_TYPE_ACTIVITY_TASK_FAILED","activityTaskFailedEventAttributes":{"failure":{"message":"Synthetic action failure","source":"TypeScriptSDK","stackTrace":"Synthetic test failure","applicationFailureInfo":{"type":"Error"}},"scheduledEventId":"5","startedEventId":"6","identity":"space-synthetic-tests","retryState":"RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED"}},
+    {"eventId":"8","eventTime":{"seconds":"4946587779","nanos":381000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"sandbox-60-main","kind":"TASK_QUEUE_KIND_NORMAL"},"startToCloseTimeout":{"seconds":"10"},"attempt":1}},
+    {"eventId":"9","eventTime":{"seconds":"4946587779","nanos":381000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"8","identity":"space-synthetic-tests","historySizeBytes":"2119"}},
+    {"eventId":"10","eventTime":{"seconds":"4946587779","nanos":392000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"8","identity":"space-synthetic-tests","binaryChecksum":"@temporalio/worker@1.24.0+1bda3f8171c686154b9ae83f1228126364d43fe31da1622f07aff7a2bdb6b26a","sdkMetadata":{},"meteringMetadata":{}}},
+    {"eventId":"11","eventTime":{"seconds":"4946587779","nanos":392000000},"eventType":"EVENT_TYPE_MARKER_RECORDED","markerRecordedEventAttributes":{"markerName":"core_patch","details":{"patch-data":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"eyJpZCI6InJvb20tYmFja2dyb3VuZC1hY3Rpb24tZmFpbHVyZS12MSIsImRlcHJlY2F0ZWQiOmZhbHNlfQ=="}]}},"workflowTaskCompletedEventId":"9"}},
+    {"eventId":"12","eventTime":{"seconds":"4946587779","nanos":392000000},"eventType":"EVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES","upsertWorkflowSearchAttributesEventAttributes":{"workflowTaskCompletedEventId":"9","searchAttributes":{"indexedFields":{"TemporalChangeVersion":{"metadata":{"type":"S2V5d29yZExpc3Q=","encoding":"anNvbi9wbGFpbg=="},"data":"WyJyb29tLWJhY2tncm91bmQtYWN0aW9uLWZhaWx1cmUtdjEiXQ=="}}}}},
+    {"eventId":"13","eventTime":{"seconds":"4946587779","nanos":392000000},"eventType":"EVENT_TYPE_ACTIVITY_TASK_SCHEDULED","activityTaskScheduledEventAttributes":{"activityId":"2","activityType":{"name":"failRoomAgentBackgroundAction"},"taskQueue":{"name":"sandbox-60-main","kind":"TASK_QUEUE_KIND_NORMAL"},"header":{},"input":{"payloads":[{"metadata":{"encoding":"anNvbi9wbGFpbg=="},"data":"eyJyb29tSWQiOiJyb29tOnN5bnRoZXRpYyIsIm1pc3Npb25JZCI6Im1pc3Npb246YWN0aW9uIiwiYWN0aW9uSWQiOiJhY3Rpb246c3ludGhldGljIiwiY2FuY2VsbGVkIjpmYWxzZX0="}]},"scheduleToCloseTimeout":{"seconds":"315360000"},"scheduleToStartTimeout":{"seconds":"315360000"},"startToCloseTimeout":{"seconds":"60"},"heartbeatTimeout":{},"workflowTaskCompletedEventId":"9","retryPolicy":{"initialInterval":{"seconds":"1"},"backoffCoefficient":2,"maximumInterval":{"seconds":"100"},"maximumAttempts":5}}},
+    {"eventId":"14","eventTime":{"seconds":"4946587779","nanos":392000000},"eventType":"EVENT_TYPE_ACTIVITY_TASK_STARTED","activityTaskStartedEventAttributes":{"scheduledEventId":"13","identity":"space-synthetic-tests","attempt":1}},
+    {"eventId":"15","eventTime":{"seconds":"4946587779","nanos":394000000},"eventType":"EVENT_TYPE_ACTIVITY_TASK_COMPLETED","activityTaskCompletedEventAttributes":{"result":{"payloads":[{"metadata":{"encoding":"YmluYXJ5L251bGw="}}]},"scheduledEventId":"13","startedEventId":"14","identity":"space-synthetic-tests"}},
+    {"eventId":"16","eventTime":{"seconds":"4946587779","nanos":394000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_SCHEDULED","workflowTaskScheduledEventAttributes":{"taskQueue":{"name":"sandbox-60-main","kind":"TASK_QUEUE_KIND_NORMAL"},"startToCloseTimeout":{"seconds":"10"},"attempt":1}},
+    {"eventId":"17","eventTime":{"seconds":"4946587779","nanos":394000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_STARTED","workflowTaskStartedEventAttributes":{"scheduledEventId":"16","identity":"space-synthetic-tests","historySizeBytes":"3021"}},
+    {"eventId":"18","eventTime":{"seconds":"4946587779","nanos":397000000},"eventType":"EVENT_TYPE_WORKFLOW_TASK_COMPLETED","workflowTaskCompletedEventAttributes":{"scheduledEventId":"16","identity":"space-synthetic-tests","binaryChecksum":"@temporalio/worker@1.24.0+1bda3f8171c686154b9ae83f1228126364d43fe31da1622f07aff7a2bdb6b26a","sdkMetadata":{},"meteringMetadata":{}}},
+    {"eventId":"19","eventTime":{"seconds":"4946587779","nanos":397000000},"eventType":"EVENT_TYPE_WORKFLOW_EXECUTION_FAILED","workflowExecutionFailedEventAttributes":{"failure":{"message":"Activity task failed","cause":{"message":"Synthetic action failure","source":"TypeScriptSDK","stackTrace":"Synthetic test failure","applicationFailureInfo":{"type":"Error"}},"activityFailureInfo":{"scheduledEventId":"5","startedEventId":"6","identity":"space-synthetic-tests","activityType":{"name":"runRoomAgentBackgroundAction"},"activityId":"1","retryState":"RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED"}},"workflowTaskCompletedEventId":"17"}}
+  ]
+}