From 133ac05b1901ba8273f6565c3dcf0a3a336bae87 Mon Sep 17 00:00:00 2001 From: New2thisshit <96698527+New2thisshit@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:23:01 +0200 Subject: [PATCH 1/3] fix(observability): connect API and Temporal OTLP traces Export workflow, child-workflow and activity-attempt spans from all three workers through the shared bounded OTLP exporter, parented to the API request span carried in Temporal headers, inputs or memo. Workflow code does no I/O: spans leave the sandbox through a host sink with callDuringReplay: false, and span IDs are deterministic. Room Agent missions keep their own request parent; the long-lived supervisor run roots its own trace and links the request that started it, so monitoring polls and context-free work never join an earlier request's trace. Background continuations and continued runs carry links; a legacy queued item's request context is kept as a link, never as a cross-trace parent. Under Space's default Temporal failure policy, ordinary errors escaping workflow execution, signal or update handlers fail the workflow task and do not export a completed span. Non-cancellation Temporal failures export FAILED; cancellation exports CANCELLED, and continue-as-new exports CONTINUED_AS_NEW. An ApplicationFailure in an update rejects that update while the workflow continues. Exported correlation attributes come from named fields and must match a bounded identifier syntax; out-of-range timestamps and invalid attempts are dropped. Workflow-sandbox tracing modules import small contracts entry points. Fix idle empty flushes that prevented later exports. Tests: real Temporal with a local collector (production Room Agent supervisor through the API coordinator, task-failure retries for workflow, signal and update handlers), replay of a supervisor history recorded by 1.1.6, and allowlist/identifier boundary tests. Docs and changelog updated. AI-assisted implementation and fixes, reviewed by two independent AI reviewers. Refs #61. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 + apps/api/src/dot-goals.ts | 20 +- .../src/memory-consolidation-coordinator.ts | 5 +- apps/api/src/memory-mutation-coordinator.ts | 5 +- apps/api/src/room-agent.ts | 10 +- apps/api/src/trace-context.ts | 4 +- apps/api/tests/dot-goal-tracing.test.ts | 47 + apps/api/tests/dot-goals.test.ts | 7 + .../memory-consolidation-coordinator.test.ts | 6 +- .../tests/memory-mutation-coordinator.test.ts | 6 +- apps/api/tests/trace-correlation.test.ts | 92 +- apps/api/vitest.config.ts | 2 + apps/worker/package.json | 2 + apps/worker/src/activity-observability.ts | 158 +- apps/worker/src/memory-maintenance-worker.ts | 9 +- apps/worker/src/memory-mutation-worker.ts | 4 +- .../src/room-background-continuation.ts | 13 +- apps/worker/src/temporal-trace-context.ts | 216 ++ apps/worker/src/worker-tracing.ts | 87 + apps/worker/src/worker.ts | 4 +- apps/worker/src/workflow-tracing.ts | 289 ++ apps/worker/src/workflows.ts | 185 +- .../tests/activity-observability.test.ts | 164 +- .../room-agent-supervisor-1.1.6.json | 3005 +++++++++++++++++ .../fixtures/otlp-task-failure-workflows.ts | 45 + apps/worker/tests/fixtures/otlp-workflows.ts | 78 + .../tests/otlp-supervisor-integration.test.ts | 156 + .../tests/otlp-supervisor-replay.test.ts | 42 + apps/worker/tests/otlp-task-failure.test.ts | 183 + .../tests/otlp-worker-integration.test.ts | 537 +++ .../room-background-continuation.test.ts | 23 +- .../worker/tests/room-continuation-fixture.ts | 4 +- .../room-supervisor-continuation.test.ts | 4 + apps/worker/tests/workflow-tracing.test.ts | 381 +++ apps/worker/vitest.config.ts | 2 + docs/configuration-reference.md | 21 + docs/quality/observability.md | 144 +- package-lock.json | 21 + packages/contracts/package.json | 8 + packages/contracts/src/schemas.ts | 2 + .../contracts/tests/observability.test.ts | 7 + packages/runtime/src/otlp-span-exporter.ts | 43 +- .../runtime/tests/otlp-span-exporter.test.ts | 76 + 43 files changed, 5911 insertions(+), 220 deletions(-) create mode 100644 apps/api/tests/dot-goal-tracing.test.ts create mode 100644 apps/worker/src/temporal-trace-context.ts create mode 100644 apps/worker/src/worker-tracing.ts create mode 100644 apps/worker/src/workflow-tracing.ts create mode 100644 apps/worker/tests/fixtures/histories/room-agent-supervisor-1.1.6.json create mode 100644 apps/worker/tests/fixtures/otlp-task-failure-workflows.ts create mode 100644 apps/worker/tests/fixtures/otlp-workflows.ts create mode 100644 apps/worker/tests/otlp-supervisor-integration.test.ts create mode 100644 apps/worker/tests/otlp-supervisor-replay.test.ts create mode 100644 apps/worker/tests/otlp-task-failure.test.ts create mode 100644 apps/worker/tests/otlp-worker-integration.test.ts create mode 100644 apps/worker/tests/workflow-tracing.test.ts create mode 100644 packages/runtime/tests/otlp-span-exporter.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b8cb4bef9..1a3a7efbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ Semantic Versioning. ## [Unreleased] +### Observability + +- Optional OTLP tracing connects API requests to workflow executions, child + workflows and activity attempts in the main worker and both memory workers. + Each Room Agent mission stays under its own request; the long-lived supervisor + run has its own trace, so its background polling never joins an earlier + request's trace. Continued runs and background missions link to the work they + continue. Exported correlation attributes accept only values with a bounded + identifier syntax. +- Room Agent queues preserve the originating trace across retries and recovery. + Workflow replay emits no spans, and a slow or failing collector does not block + business execution. Export remains off unless an endpoint is configured. +- Fixed an idle exporter flush that could prevent later spans from being sent. + ## [1.1.6] - 2026-10-18 Source: sanitized Space app @ f8803d5d96 (4595 files). Adds install telemetry diff --git a/apps/api/src/dot-goals.ts b/apps/api/src/dot-goals.ts index 30dd69dee..ca66c4a20 100644 --- a/apps/api/src/dot-goals.ts +++ b/apps/api/src/dot-goals.ts @@ -16,6 +16,7 @@ import { } from "@space/contracts"; import { SpaceConflictError, SpaceNotFoundError, type SpaceStore } from "@space/runtime"; import type { DotGoalsRepository, DotsRepository } from "@space/db"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; /** * Space Dots Stage 4: Goals. @@ -26,10 +27,10 @@ import type { DotGoalsRepository, DotsRepository } from "@space/db"; * blocking question, cancellation, three consecutive failures or the budget. */ export interface DotGoalWorkflowCoordinator { - start(goalId: string): Promise; + start(goalId: string, traceId?: string): Promise; wake(goalId: string): Promise; /** Starts the Goal's workflow when none is running; true when it had to start one. */ - ensure?(goalId: string): Promise; + ensure?(goalId: string, traceId?: string): Promise; } export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordinator { constructor(private readonly options: { address: string; namespace: string; taskQueue: string }) {} @@ -38,11 +39,12 @@ export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordi try { return await action(new Client({ connection, namespace: this.options.namespace })); } finally { await connection.close(); } } - start(goalId: string) { + start(goalId: string, traceId?: string) { return this.client(async (client) => { try { await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue, - args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE" }); + args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE", + memo: workflowCorrelationMemo({ traceId, traceparent: traceparentForTraceId(traceId) }) }); } catch (error) { if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error; } }); } @@ -52,11 +54,12 @@ export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordi catch (error) { if (!(error instanceof WorkflowNotFoundError)) throw error; } }); } - ensure(goalId: string) { + ensure(goalId: string, traceId?: string) { return this.client(async (client) => { try { await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue, - args: [{ goalId }], workflowIdReusePolicy: "ALLOW_DUPLICATE" }); + args: [{ goalId }], workflowIdReusePolicy: "ALLOW_DUPLICATE", + memo: workflowCorrelationMemo({ traceId, traceparent: traceparentForTraceId(traceId) }) }); return true; } catch (error) { if (error instanceof WorkflowExecutionAlreadyStartedError) return false; @@ -125,7 +128,10 @@ export function createDotGoalService(deps: DotGoalServiceDeps) { if ((error as { code?: string }).code === "DOT_GOAL_OPEN") throw new DotGoalError(409, "DOT_GOAL_OPEN", "This conversation already has an open Goal."); throw error; } - try { await deps.workflow.start(goal.goalId); } + try { + if (request?.requestIdForSpace) await deps.workflow.start(goal.goalId, request.requestIdForSpace); + else await deps.workflow.start(goal.goalId); + } catch { goal = await finish(goal, "FAILED", "The Goal runtime could not start. Try again."); throw new DotGoalError(503, "DOT_GOAL_RUNTIME", "The Goal runtime could not start. Try again."); diff --git a/apps/api/src/memory-consolidation-coordinator.ts b/apps/api/src/memory-consolidation-coordinator.ts index df237c648..96d1ae042 100644 --- a/apps/api/src/memory-consolidation-coordinator.ts +++ b/apps/api/src/memory-consolidation-coordinator.ts @@ -6,6 +6,7 @@ import { type MemoryConsolidationWorkflowInput } from "@space/contracts"; import { SpaceFeatureDisabledError } from "@space/runtime"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; export { MEMORY_CONSOLIDATION_TASK_QUEUE, MEMORY_CONSOLIDATION_WORKFLOW_TYPE }; @@ -21,6 +22,7 @@ interface WorkflowStartOptions { workflowId: string; workflowIdReusePolicy: "REJECT_DUPLICATE"; workflowIdConflictPolicy: "USE_EXISTING"; + memo: Record; } interface WorkflowStartHandle { @@ -49,7 +51,8 @@ export async function scheduleMemoryConsolidation( taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", - workflowIdConflictPolicy: "USE_EXISTING" + workflowIdConflictPolicy: "USE_EXISTING", + memo: workflowCorrelationMemo({ traceId: input.traceId, traceparent: traceparentForTraceId(input.traceId) }) }); return { status: "SCHEDULED", diff --git a/apps/api/src/memory-mutation-coordinator.ts b/apps/api/src/memory-mutation-coordinator.ts index f7d3d48bb..44a72ee4f 100644 --- a/apps/api/src/memory-mutation-coordinator.ts +++ b/apps/api/src/memory-mutation-coordinator.ts @@ -7,6 +7,7 @@ import { type MemoryMutationWorkflowInput } from "@space/contracts"; import { SpaceFeatureDisabledError } from "@space/runtime"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; export { MEMORY_MUTATION_TASK_QUEUE, MEMORY_MUTATION_WORKFLOW_TYPE, buildMemoryMutationWorkflowId }; @@ -22,6 +23,7 @@ interface WorkflowStartOptions { workflowId: string; workflowIdReusePolicy: "REJECT_DUPLICATE"; workflowIdConflictPolicy: "USE_EXISTING"; + memo: Record; } interface WorkflowStartHandle { @@ -51,7 +53,8 @@ export async function scheduleMemoryMutation( taskQueue: MEMORY_MUTATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", - workflowIdConflictPolicy: "USE_EXISTING" + workflowIdConflictPolicy: "USE_EXISTING", + memo: workflowCorrelationMemo({ traceId: input.traceId, traceparent: traceparentForTraceId(input.traceId) }) }); return { status: "SCHEDULED", workflowId: handle.workflowId, runId: handle.firstExecutionRunId ?? null }; } catch (error) { diff --git a/apps/api/src/room-agent.ts b/apps/api/src/room-agent.ts index 3af0cf9aa..bc1dfda96 100644 --- a/apps/api/src/room-agent.ts +++ b/apps/api/src/room-agent.ts @@ -225,6 +225,11 @@ export class TemporalRoomAgentWorkflowCoordinator implements RoomAgentWorkflowCo workflowId, taskQueue: this.options.taskQueue, args: [{ roomId: item.turn.roomId, pending: [], processedCount: 0 }], + memo: workflowCorrelationMemo({ + traceId: item.turn.traceId, + traceparent: item.turn.traceparent ?? null, + missionId: item.missionId + }), signal: ROOM_AGENT_ENQUEUE_SIGNAL, signalArgs: [item] }); @@ -659,7 +664,10 @@ export function createRoomAgentService(options: { selectedToolIds: [...roomAgentSelectedToolIds], permissionMode: "full_access", collaborationMode: "default", - traceId + traceId, + // Persist the originating span before enqueue so recovery and retries + // retain it after the API request's in-memory context is evicted. + traceparent: traceparentForTraceId(traceId) ?? undefined } }); const childWorkflowId = buildCodexAppServerTurnWorkflowId(turn.turn); diff --git a/apps/api/src/trace-context.ts b/apps/api/src/trace-context.ts index af9e55efd..7c28eb013 100644 --- a/apps/api/src/trace-context.ts +++ b/apps/api/src/trace-context.ts @@ -96,7 +96,9 @@ export const requestTraceContexts = new TraceContextRegistry(); /** traceparent for work started on behalf of this Space request id, if known. */ export function traceparentForTraceId(traceId: string | null | undefined, registry = requestTraceContexts): string | null { - return childTraceparent(registry.lookup(traceId)?.traceparent ?? null); + // Propagate the span we actually export. Minting an unrecorded child here + // leaves every downstream workflow attached to a missing parent. + return registry.lookup(traceId)?.traceparent ?? null; } /** Adds the request's traceparent to a turn input unless it already has a valid one. */ diff --git a/apps/api/tests/dot-goal-tracing.test.ts b/apps/api/tests/dot-goal-tracing.test.ts new file mode 100644 index 000000000..c4b76f052 --- /dev/null +++ b/apps/api/tests/dot-goal-tracing.test.ts @@ -0,0 +1,47 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { parseTraceparent } from "@space/contracts"; +import { TemporalDotGoalWorkflowCoordinator } from "../src/dot-goals.js"; +import { createRequestTraceContext, requestTraceContexts } from "../src/trace-context.js"; + +const temporal = vi.hoisted(() => ({ start: vi.fn(), close: vi.fn() })); + +vi.mock("@temporalio/client", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + Connection: { connect: async () => ({ close: temporal.close }) }, + Client: class { workflow = { start: temporal.start }; } + }; +}); + +beforeEach(() => { + temporal.start.mockReset().mockResolvedValue({}); + temporal.close.mockReset().mockResolvedValue(undefined); +}); + +describe("Dot Goal workflow trace propagation", () => { + it("parents both explicit starts and requested restarts to their recorded API spans", async () => { + const coordinator = new TemporalDotGoalWorkflowCoordinator({ address: "unused", namespace: "default", taskQueue: "agent-turns" }); + const first = createRequestTraceContext("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"); + const restart = createRequestTraceContext("00-11111111111111111111111111111111-2222222222222222-01"); + requestTraceContexts.remember("req:goal-create", first); + requestTraceContexts.remember("req:goal-restart", restart); + + await coordinator.start("dot_goal:abcdefgh1", "req:goal-create"); + expect(await coordinator.ensure("dot_goal:abcdefgh1", "req:goal-restart")).toBe(true); + + const start = temporal.start.mock.calls[0]![1]; + const resumed = temporal.start.mock.calls[1]![1]; + expect(start).toMatchObject({ args: [{ goalId: "dot_goal:abcdefgh1" }], workflowIdReusePolicy: "REJECT_DUPLICATE" }); + expect(resumed).toMatchObject({ args: [{ goalId: "dot_goal:abcdefgh1" }], workflowIdReusePolicy: "ALLOW_DUPLICATE" }); + expect(parseTraceparent(start.memo.traceparent)).toMatchObject({ traceId: first.w3cTraceId, parentId: first.spanId }); + expect(parseTraceparent(resumed.memo.traceparent)).toMatchObject({ traceId: restart.w3cTraceId, parentId: restart.spanId }); + expect(temporal.close).toHaveBeenCalledTimes(2); + }); + + it("does not fabricate a parent for autonomous reconciliation with no request", async () => { + const coordinator = new TemporalDotGoalWorkflowCoordinator({ address: "unused", namespace: "default", taskQueue: "agent-turns" }); + await coordinator.ensure("dot_goal:abcdefgh1"); + expect(temporal.start.mock.calls[0]![1].memo).toEqual({}); + }); +}); diff --git a/apps/api/tests/dot-goals.test.ts b/apps/api/tests/dot-goals.test.ts index 0c4a38e11..82ca126ec 100644 --- a/apps/api/tests/dot-goals.test.ts +++ b/apps/api/tests/dot-goals.test.ts @@ -69,6 +69,13 @@ function fixture(overrides: { runStatus?: string; pendingApproval?: boolean; sen const input = { sessionId: "agent_session:1", objective: "Ship the report", milestones: ["Draft", "Review"], criteria: ["Report published"], budget: { maxSteps: 2 } }; describe("Dot Goals", () => { + it("passes the creating API request's correlation id to the workflow starter", async () => { + const { service, deps } = fixture(); + const request = { requestIdForSpace: "req:dot-goal-tracing" }; + const goal = await service.create("user:1", dotId, input, request as never); + expect(deps.workflow!.start).toHaveBeenCalledWith(goal.goalId, request.requestIdForSpace); + }); + it("creates one open Goal per conversation for the owner's Dot and starts its workflow", async () => { const { service, deps } = fixture(); const goal = await service.create("user:1", dotId, input, null); diff --git a/apps/api/tests/memory-consolidation-coordinator.test.ts b/apps/api/tests/memory-consolidation-coordinator.test.ts index 3ff07aecb..baa6bc022 100644 --- a/apps/api/tests/memory-consolidation-coordinator.test.ts +++ b/apps/api/tests/memory-consolidation-coordinator.test.ts @@ -4,11 +4,14 @@ import { MEMORY_CONSOLIDATION_WORKFLOW_TYPE, scheduleMemoryConsolidation } from "../src/memory-consolidation-coordinator.js"; +import { createRequestTraceContext, requestTraceContexts } from "../src/trace-context.js"; describe("Memory consolidation coordinator", () => { it("starts a retry-safe workflow on the exclusive maintenance queue", async () => { const calls: Array<{ workflowType: string; options: unknown }> = []; const input = { runId: "memory_consolidation:run-1", traceId: "trace:memory-consolidation" }; + const request = createRequestTraceContext("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"); + requestTraceContexts.remember(input.traceId, request); const workflowId = `space-memory-consolidation:${"a".repeat(64)}`; const result = await scheduleMemoryConsolidation(input, workflowId, async (workflowType, options) => { @@ -24,7 +27,8 @@ describe("Memory consolidation coordinator", () => { taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", - workflowIdConflictPolicy: "USE_EXISTING" + workflowIdConflictPolicy: "USE_EXISTING", + memo: { spaceTraceId: input.traceId, traceparent: request.traceparent, w3cTraceId: request.w3cTraceId } } }]); }); diff --git a/apps/api/tests/memory-mutation-coordinator.test.ts b/apps/api/tests/memory-mutation-coordinator.test.ts index e07db2b56..b3126cc71 100644 --- a/apps/api/tests/memory-mutation-coordinator.test.ts +++ b/apps/api/tests/memory-mutation-coordinator.test.ts @@ -5,11 +5,14 @@ import { buildMemoryMutationWorkflowId, scheduleMemoryMutation } from "../src/memory-mutation-coordinator.js"; +import { createRequestTraceContext, requestTraceContexts } from "../src/trace-context.js"; describe("Memory mutation coordinator", () => { it("starts the dedicated workflow with deterministic duplicate guards", async () => { const calls: Array<{ workflowType: string; options: unknown }> = []; const input = { changeSetId: "memory_change:approved", traceId: "req:mutation-start" }; + const request = createRequestTraceContext("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"); + requestTraceContexts.remember(input.traceId, request); const result = await scheduleMemoryMutation(input, async (workflowType, options) => { calls.push({ workflowType, options }); @@ -30,7 +33,8 @@ describe("Memory mutation coordinator", () => { taskQueue: MEMORY_MUTATION_TASK_QUEUE, workflowId: result.workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", - workflowIdConflictPolicy: "USE_EXISTING" + workflowIdConflictPolicy: "USE_EXISTING", + memo: { spaceTraceId: input.traceId, traceparent: request.traceparent, w3cTraceId: request.w3cTraceId } } } ]); diff --git a/apps/api/tests/trace-correlation.test.ts b/apps/api/tests/trace-correlation.test.ts index ff8a47758..c4ab8ce25 100644 --- a/apps/api/tests/trace-correlation.test.ts +++ b/apps/api/tests/trace-correlation.test.ts @@ -5,7 +5,8 @@ import { createApp } from "../src/app.js"; import { getApiConfig } from "../src/config.js"; import { apiDomainMetrics } from "../src/api-metrics.js"; import { TemporalCodexAppServerTurnStarter } from "../src/turns.js"; -import { TemporalRoomAgentWorkflowCoordinator } from "../src/room-agent.js"; +import { createRoomAgentService, TemporalRoomAgentWorkflowCoordinator } from "../src/room-agent.js"; +import * as roomAgentModels from "../src/room-agent-model-resolver.js"; import type { TemporalClientFactory } from "../src/temporal-client.js"; import { TraceContextRegistry, @@ -47,9 +48,11 @@ describe("request trace context", () => { expect(registry.size).toBe(2); expect(registry.lookup("req:aaaaaaaaaaaa")).toBeNull(); - registry.remember("req:dddddddddddd", createRequestTraceContext(inbound)); + const request = createRequestTraceContext(inbound); + registry.remember("req:dddddddddddd", request); const filled = withTurnTraceparent({ traceId: "req:dddddddddddd" }, registry); - expect(parseTraceparent(filled.traceparent)?.traceId).toBe(inboundTraceId); + expect(filled.traceparent).toBe(request.traceparent); + expect(parseTraceparent(filled.traceparent)).toMatchObject({ traceId: inboundTraceId, parentId: request.spanId }); const kept = withTurnTraceparent({ traceId: "req:dddddddddddd", traceparent: inbound }, registry); expect(kept.traceparent).toBe(inbound); expect(withTurnTraceparent({ traceId: "req:unknown00000" }, registry)).toEqual({ traceId: "req:unknown00000" }); @@ -173,8 +176,68 @@ describe("workflow starters", () => { artifactIds: [] }; + it("persists the original mission parent before dispatch and keeps it through registry eviction, recovery and request retry", async () => { + const selection: roomAgentModels.AdvancedMissionModelSelection = { + providerId: null, modelId: "gpt-5.6-sol", reasoningEffort: "high", providerName: null, modelName: "GPT-5.6 Sol", tier: "CODEX" + }; + const model = vi.spyOn(roomAgentModels, "resolveAdvancedMissionModel").mockResolvedValue(selection); + try { + const store = new InMemorySpaceStore(); + const room = store.createRoom({ name: "Mission tracing recovery", initialPaneCount: 0 }); + const requestId = "req:persisted-mission-parent"; + const request = createRequestTraceContext(inbound); + requestTraceContexts.remember(requestId, request); + const unavailable = new Error("Temporal dispatch unavailable in this fixture"); + const enqueue = vi.fn(async (_item: RoomAgentSupervisorQueueItem) => ({ workflowId: "workflow:mission-tracing", runId: null })) + .mockRejectedValueOnce(unavailable); + const options: Parameters[0] = { + store, + workflow: { enqueue, stop: async () => {} }, + missionStopper: { pauseMission: async () => {}, resumeMission: async () => {}, stopMission: async () => {} }, + roomPlanInventoryProvider: { inspect: async () => ({ + totalPanes: 0, plannedPanes: 0, pendingPlans: 0, readyPlans: 0, pausedPlans: 0, runningPlans: 0, + checkedAt: new Date().toISOString(), plans: [] + }) }, + modelSelection: async () => selection + }; + const actor = { holderType: "OPERATOR", holderId: "user:operator" } as const; + const objective = "Investigate the test fixture's queued task."; + const clientRequestId = "mission:trace-persistence"; + + await expect(createRoomAgentService(options).startMission(room.id, objective, clientRequestId, requestId, actor)).rejects.toBe(unavailable); + const unsignaled = store.listUnsignaledRoomAgentEnqueues(); + expect(unsignaled).toHaveLength(1); + const original = unsignaled[0]!; + // Inspect the persisted queue item, not the outgoing coordinator copy. + expect(original.queueItem.turn.traceparent).toBe(request.traceparent); + expect(parseTraceparent(original.queueItem.turn.traceparent)?.parentId).toBe(request.spanId); + + // Recovery must work after the request context has left the bounded map. + for (let index = 0; index < 4096; index += 1) requestTraceContexts.remember(`req:mission-eviction-${index}`, request); + expect(requestTraceContexts.lookup(requestId)).toBeNull(); + const recoveredService = createRoomAgentService(options); + expect(await recoveredService.recoverPending()).toEqual({ scanned: 1, recovered: 1, failed: 0 }); + expect(enqueue.mock.calls[1]![0].turn.traceparent).toBe(request.traceparent); + expect(store.listUnsignaledRoomAgentEnqueues()).toHaveLength(0); + + const retriedRequestId = "req:retried-mission-parent"; + const retriedRequest = createRequestTraceContext(undefined); + requestTraceContexts.remember(retriedRequestId, retriedRequest); + await recoveredService.startMission(room.id, objective, clientRequestId, retriedRequestId, actor); + const persisted = store.getRoomAgentTurn(original.mission.id, original.run.runId)!; + expect(persisted.queueItem.turn.traceparent).toBe(request.traceparent); + expect(persisted.queueItem.turn.traceparent).not.toBe(retriedRequest.traceparent); + expect(store.listRoomAgentMissions(room.id)).toHaveLength(1); + expect(enqueue).toHaveBeenCalledTimes(2); + } finally { + model.mockRestore(); + } + }); + + it("passes the request id and traceparent in workflow input and memo", async () => { - requestTraceContexts.remember(turn.traceId, createRequestTraceContext(inbound)); + const request = createRequestTraceContext(inbound); + requestTraceContexts.remember(turn.traceId, request); const start = vi.fn(async (_type: string, options: { workflowId: string }) => ({ workflowId: options.workflowId, firstExecutionRunId: "run-1" })); const close = vi.fn(async () => undefined); const clientFactory = vi.fn(async () => ({ client: { workflow: { start } }, close })) as unknown as TemporalClientFactory; @@ -188,8 +251,9 @@ describe("workflow starters", () => { expect(workflowType).toBe("codexAppServerTurnWorkflow"); expect(options.workflowId).toContain(turn.traceId); expect(options.args[0]?.traceId).toBe(turn.traceId); - expect(parseTraceparent(options.args[0]?.traceparent)?.traceId).toBe(inboundTraceId); - expect(options.memo).toMatchObject({ spaceTraceId: turn.traceId, w3cTraceId: inboundTraceId }); + expect(options.args[0]?.traceparent).toBe(request.traceparent); + expect(parseTraceparent(options.args[0]?.traceparent)).toMatchObject({ traceId: inboundTraceId, parentId: request.spanId }); + expect(options.memo).toMatchObject({ spaceTraceId: turn.traceId, w3cTraceId: inboundTraceId, traceparent: request.traceparent }); expect(JSON.stringify(options.memo)).not.toContain("PRIVATE PROMPT"); expect(close).toHaveBeenCalledOnce(); expect(metricValue(/workflow_type="codexAppServerTurnWorkflow",outcome="STARTED",category="NONE"/)).toBe(before + 1); @@ -211,7 +275,8 @@ describe("workflow starters", () => { }); it("adds the traceparent to queued room-agent turns and links background actions to their origin", async () => { - requestTraceContexts.remember("req:roomagent0001", createRequestTraceContext(inbound)); + const request = createRequestTraceContext(inbound); + requestTraceContexts.remember("req:roomagent0001", request); const signalWithStart = vi.fn(async () => ({ signaledRunId: "run-2" })); const start = vi.fn(async () => ({})); const clientFactory = vi.fn(async () => ({ client: { workflow: { signalWithStart, start } }, close: vi.fn(async () => undefined) })) as unknown as TemporalClientFactory; @@ -222,10 +287,13 @@ describe("workflow starters", () => { }; await coordinator.enqueue(item); - const signalOptions = (signalWithStart.mock.calls[0] as unknown as [string, { signalArgs: RoomAgentSupervisorQueueItem[] }])[1]; - expect(parseTraceparent(signalOptions.signalArgs[0]?.turn.traceparent)?.traceId).toBe(inboundTraceId); + const signalOptions = (signalWithStart.mock.calls[0] as unknown as [string, { signalArgs: RoomAgentSupervisorQueueItem[]; memo: Record }])[1]; + expect(signalOptions.signalArgs[0]?.turn.traceparent).toBe(request.traceparent); + expect(parseTraceparent(signalOptions.signalArgs[0]?.turn.traceparent)).toMatchObject({ traceId: inboundTraceId, parentId: request.spanId }); + expect(signalOptions.memo).toMatchObject({ spaceTraceId: "req:roomagent0001", w3cTraceId: inboundTraceId, traceparent: request.traceparent, missionId: item.missionId }); - requestTraceContexts.remember("req:internal0001", createRequestTraceContext(inbound)); + const internalRequest = createRequestTraceContext(inbound); + requestTraceContexts.remember("req:internal0001", internalRequest); await coordinator.enqueueAction("room_agent_action:1", { roomId: turn.roomId, missionId: "room_agent_mission:trace", agentPaneId: turn.paneId, agentSessionId: "agent_session:1", requestId: "req:roomagent0001", selectedToolIds: [], actions: [] @@ -235,7 +303,9 @@ describe("workflow starters", () => { spaceTraceId: "req:internal0001", parentSpaceTraceId: "req:roomagent0001", missionId: "room_agent_mission:trace", - w3cTraceId: inboundTraceId + w3cTraceId: inboundTraceId, + traceparent: internalRequest.traceparent }); + expect(parseTraceparent(actionOptions.memo.traceparent)?.parentId).toBe(internalRequest.spanId); }); }); diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts index 1917bbed9..50c7f10e9 100644 --- a/apps/api/vitest.config.ts +++ b/apps/api/vitest.config.ts @@ -11,6 +11,8 @@ export default defineConfig({ "@space/cli-host": resolve(__dirname, "../../packages/cli-host/src/index.ts"), "@space/db": resolve(__dirname, "../../packages/db/src/index.ts"), "@space/gateway": resolve(__dirname, "../../packages/gateway/src/index.ts"), + "@space/contracts/observability": resolve(__dirname, "../../packages/contracts/src/observability.ts"), + "@space/contracts/workflows": resolve(__dirname, "../../packages/contracts/src/workflows.ts"), "@space/contracts": resolve(__dirname, "../../packages/contracts/src/index.ts"), "@space/opencode-control": resolve(__dirname, "../../packages/opencode-control/src/index.ts"), "@space/runtime": resolve(__dirname, "../../packages/runtime/src/index.ts"), diff --git a/apps/worker/package.json b/apps/worker/package.json index abd012313..0def1e33c 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -26,6 +26,8 @@ "undici": "8.10.2" }, "devDependencies": { + "@temporalio/proto": "1.24.0", + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.11" }, diff --git a/apps/worker/src/activity-observability.ts b/apps/worker/src/activity-observability.ts index ea4d021d1..a71603e9d 100644 --- a/apps/worker/src/activity-observability.ts +++ b/apps/worker/src/activity-observability.ts @@ -8,9 +8,11 @@ import { isSafeErrorCode, parseTraceparent, traceIdFromTraceparent, - type FailureCategory + type FailureCategory, + type ParsedTraceparent } from "@space/contracts"; import type { SpanExporter } from "@space/runtime"; +import { correlationIdentifier, decodeActivityTraceContext } from "./temporal-trace-context.js"; import { logIdentifier, workerLog, type WorkerLogValue } from "./worker-log.js"; import { workerMetrics, type WorkerMetrics } from "./worker-metrics.js"; @@ -30,6 +32,7 @@ export type ActivityOutcome = "COMPLETED" | "FAILED_RESULT" | "FAILED" | "CANCEL interface ActivityObservation { reasonCode: string | null; + traceparent: string | null; } const observations = new AsyncLocalStorage(); @@ -51,17 +54,23 @@ export function noteActivityFailureReason(reasonCode: string): void { export function failureReasonScope(execute: () => Promise): { promise: Promise; reasonCode: () => string | null } { const existing = observations.getStore(); if (existing) return { promise: execute(), reasonCode: () => existing.reasonCode }; - const observation: ActivityObservation = { reasonCode: null }; + const observation: ActivityObservation = { reasonCode: null, traceparent: null }; return { promise: observations.run(observation, execute), reasonCode: () => observation.reasonCode }; } -/** traceparent for an outbound call: same W3C trace, new span id; null if none. */ +function randomSpanId(): string { + let spanId = randomBytes(8).toString("hex"); + while (/^0+$/.test(spanId)) spanId = randomBytes(8).toString("hex"); + return spanId; +} + +/** Continue the exported activity attempt on outbound calls, or use the legacy input fallback. */ export function childTraceparent(traceparent: string | null | undefined): string | null { + const active = observations.getStore()?.traceparent; + if (active) return active; const parsed = parseTraceparent(traceparent); if (!parsed) return null; - let spanId = randomBytes(8).toString("hex"); - while (/^0+$/.test(spanId)) spanId = randomBytes(8).toString("hex"); - return formatTraceparent(parsed.traceId, spanId, parsed.flags); + return formatTraceparent(parsed.traceId, randomSpanId(), parsed.flags); } // Keys whose values are identifiers. Prompts, content and messages are never read. @@ -135,8 +144,10 @@ export interface ObserveActivityDependencies { metrics?: WorkerMetrics; now?: () => number; cancelled?: () => boolean; - /** OTLP export (SPACE_OTEL_EXPORTER_OTLP_ENDPOINT): one span per activity whose input carries a traceparent. */ + /** OTLP export (SPACE_OTEL_EXPORTER_OTLP_ENDPOINT): one span per activity attempt. */ spans?: SpanExporter | null; + /** Propagated workflow span from Temporal headers; legacy inputs remain a fallback. */ + traceContext?: ParsedTraceparent | null; } function activityTraceparent(args: readonly unknown[]) { @@ -145,6 +156,36 @@ function activityTraceparent(args: readonly unknown[]) { return parseTraceparent(found.traceparent); } +// Observability must never turn a successful activity into a retry, or replace +// the original business failure. In particular, exporting is never awaited. +function bestEffort(observe: () => T): T | undefined { + try { return observe(); } catch { return undefined; } +} + +// Temporal-assigned identities; the Space correlation IDs below must also have the generated `:` shape. +const temporalSpanKeys = { + activityId: "space.activity_id", + workflowType: "space.workflow_type", + workflowId: "space.workflow_id", + temporalRunId: "space.temporal_run_id" +} as const; + +const spanCorrelationKeys = { + traceId: "space.trace_id", + parentTraceId: "space.request_id", + missionId: "space.mission_id", + roomId: "space.room_id", + paneId: "space.pane_id", + agentRunId: "space.agent_run_id", + agentSessionId: "space.agent_session_id", + actionId: "space.action_id", + goalId: "space.goal_id", + dotId: "space.dot_id", + turnId: "space.turn_id", + cliTaskId: "space.cli_task_id", + revisionId: "space.revision_id" +} as const; + export async function observeActivityExecution( info: ActivityInfo, args: readonly unknown[], @@ -165,60 +206,94 @@ export async function observeActivityExecution( workflowId: info.workflowExecution?.workflowId ?? null, temporalRunId: info.workflowExecution?.runId ?? null, attempt: info.attempt, - ...activityCorrelation(args) + ...(bestEffort(() => activityCorrelation(args)) ?? {}), + ...(dependencies.traceContext ? { w3cTraceId: dependencies.traceContext.traceId } : {}) }; const labels = { activity_type: activityType }; - if (info.attempt > 1) metrics.activityRetries.inc(labels); - if (queueMs !== null) metrics.activityScheduleToStart.observe(labels, queueMs / 1000); - metrics.activitiesInFlight.inc(labels); - workerLog("info", "activity.started", { ...base, queueMs }); - const observation: ActivityObservation = { reasonCode: null }; + bestEffort(() => { + if (info.attempt > 1) metrics.activityRetries.inc(labels); + if (queueMs !== null) metrics.activityScheduleToStart.observe(labels, queueMs / 1000); + metrics.activitiesInFlight.inc(labels); + workerLog("info", "activity.started", { ...base, queueMs }); + }); + const activitySpan = bestEffort(() => { + if (!dependencies.spans) return null; + const parent = dependencies.traceContext ?? activityTraceparent(args); + return parent ? { parent, spanId: randomSpanId() } : null; + }); + const observation: ActivityObservation = { + reasonCode: null, + traceparent: activitySpan ? formatTraceparent(activitySpan.parent.traceId, activitySpan.spanId, activitySpan.parent.flags) : null + }; - const finish = (outcome: ActivityOutcome, reasonCode: string | null, errorName: string | null) => { + const finish = (outcome: ActivityOutcome, reasonCode: string | null, errorName: string | null) => bestEffort(() => { const durationMs = Math.max(0, now() - startedAt); - metrics.activitiesInFlight.inc(labels, -1); - metrics.activityExecutions.inc({ activity_type: activityType, outcome }); - metrics.activityDuration.observe(labels, durationMs / 1000); - let category: FailureCategory | null = null; - if (outcome !== "COMPLETED") { - category = outcome === "CANCELLED" ? "CANCELLED" : categorizeFailureCode(reasonCode); - metrics.activityFailures.inc({ activity_type: activityType, category, code: reasonCode ?? "UNSPECIFIED" }); - } - workerLog(outcome === "COMPLETED" ? "info" : outcome === "CANCELLED" ? "warn" : "error", "activity.finished", { - ...base, - outcome, - reasonCode, - category, - errorName, - durationMs, - queueMs, - rssBytes: process.memoryUsage().rss + const category: FailureCategory | null = outcome === "COMPLETED" ? null + : outcome === "CANCELLED" ? "CANCELLED" : categorizeFailureCode(reasonCode); + bestEffort(() => { + metrics.activitiesInFlight.inc(labels, -1); + metrics.activityExecutions.inc({ activity_type: activityType, outcome }); + metrics.activityDuration.observe(labels, durationMs / 1000); + if (category !== null) { + metrics.activityFailures.inc({ activity_type: activityType, category, code: reasonCode ?? "UNSPECIFIED" }); + } + workerLog(outcome === "COMPLETED" ? "info" : outcome === "CANCELLED" ? "warn" : "error", "activity.finished", { + ...base, + outcome, + reasonCode, + category, + errorName, + durationMs, + queueMs, + rssBytes: process.memoryUsage().rss + }); }); - const parent = dependencies.spans ? activityTraceparent(args) : null; - if (parent) { + if (activitySpan) { + const { parent, spanId } = activitySpan; + const attributes: Record = { + "space.activity_type": activityType, + "space.outcome": outcome, + "space.duration_ms": durationMs + }; + if (Number.isSafeInteger(info.attempt) && info.attempt >= 1) attributes["space.attempt"] = info.attempt; + if (queueMs !== null) attributes["space.queue_ms"] = queueMs; + for (const [key, attribute] of Object.entries(temporalSpanKeys)) { + const identifier = logIdentifier(base[key]); + if (identifier !== null) attributes[attribute] = identifier; + } + for (const [key, attribute] of Object.entries(spanCorrelationKeys)) { + const identifier = correlationIdentifier(base[key]); + if (identifier !== null) attributes[attribute] = identifier; + } + if (isSafeErrorCode(reasonCode)) attributes["space.reason_code"] = reasonCode; + if (category !== null) attributes["space.failure_category"] = category; dependencies.spans!.record({ traceId: parent.traceId, - spanId: randomBytes(8).toString("hex"), + spanId, parentSpanId: parent.parentId, name: `activity ${activityType}`, startTimeUnixNano: BigInt(startedAt) * 1_000_000n, endTimeUnixNano: BigInt(startedAt + durationMs) * 1_000_000n, - attributes: { "space.activity_type": activityType, "space.outcome": outcome, "space.attempt": info.attempt }, + attributes, error: outcome === "FAILED" || outcome === "FAILED_RESULT", kind: 1 }); } - }; + }); try { const result = await observations.run(observation, execute); - if (isFailedResult(result)) finish("FAILED_RESULT", observation.reasonCode ?? "ACTIVITY_RESULT_FAILED", null); - else finish("COMPLETED", null, null); + bestEffort(() => { + if (isFailedResult(result)) finish("FAILED_RESULT", observation.reasonCode ?? "ACTIVITY_RESULT_FAILED", null); + else finish("COMPLETED", null, null); + }); return result; } catch (error) { - const cancelled = dependencies.cancelled?.() === true || /Cancel/i.test((error as { name?: string } | null)?.name ?? ""); - const errorName = logIdentifier((error as { name?: unknown } | null)?.name); - finish(cancelled ? "CANCELLED" : "FAILED", observation.reasonCode ?? errorCodeOf(error), errorName); + bestEffort(() => { + const cancelled = dependencies.cancelled?.() === true || /Cancel/i.test((error as { name?: string } | null)?.name ?? ""); + const errorName = logIdentifier((error as { name?: unknown } | null)?.name); + finish(cancelled ? "CANCELLED" : "FAILED", observation.reasonCode ?? errorCodeOf(error), errorName); + }); throw error; } } @@ -229,6 +304,7 @@ export function createActivityObservabilityInterceptors(dependencies: Omit observeActivityExecution(context.info, input.args, () => next(input), { ...dependencies, + traceContext: decodeActivityTraceContext(input.headers), cancelled: () => context.cancellationSignal.aborted }) } diff --git a/apps/worker/src/memory-maintenance-worker.ts b/apps/worker/src/memory-maintenance-worker.ts index 28122f546..119a819dd 100644 --- a/apps/worker/src/memory-maintenance-worker.ts +++ b/apps/worker/src/memory-maintenance-worker.ts @@ -8,7 +8,7 @@ import { } from "./memory-maintenance-config.js"; import { DEFAULT_TEMPORAL_ADDRESS, DEFAULT_TEMPORAL_NAMESPACE } from "./constants.js"; import { otlpSpanExporterFromEnv } from "@space/runtime"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { createWorkerTracing } from "./worker-tracing.js"; const service = "space-memory-maintenance-worker"; @@ -19,9 +19,9 @@ if (!memoryMaintenanceEnabled(process.env)) { const namespace = process.env.SPACE_TEMPORAL_NAMESPACE ?? DEFAULT_TEMPORAL_NAMESPACE; const workflowsPath = fileURLToPath(new URL("./memory-maintenance-workflows.js", import.meta.url)); + const spans = otlpSpanExporterFromEnv(process.env, service); try { const connection = await NativeConnection.connect({ address }); - const spans = otlpSpanExporterFromEnv(process.env, service); const worker = await Worker.create({ connection, namespace, @@ -29,7 +29,7 @@ if (!memoryMaintenanceEnabled(process.env)) { workflowsPath, activities, ...memoryMaintenanceWorkerOptions, - interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] } + ...createWorkerTracing(spans) }); console.log(JSON.stringify({ service, @@ -40,7 +40,6 @@ if (!memoryMaintenanceEnabled(process.env)) { activityConcurrency: memoryMaintenanceWorkerOptions.maxConcurrentActivityTaskExecutions })); await worker.run(); - await spans?.close(); } catch (error) { console.error(JSON.stringify({ service, @@ -48,5 +47,7 @@ if (!memoryMaintenanceEnabled(process.env)) { error: error instanceof Error ? error.message : "Unknown maintenance worker failure." })); process.exitCode = 1; + } finally { + await spans?.close(); } } diff --git a/apps/worker/src/memory-mutation-worker.ts b/apps/worker/src/memory-mutation-worker.ts index adc069daf..e4c8bfc80 100644 --- a/apps/worker/src/memory-mutation-worker.ts +++ b/apps/worker/src/memory-mutation-worker.ts @@ -4,7 +4,7 @@ import { executePersistedMemoryChangeSet } from "./memory-mutation-activity.js"; import { MEMORY_MUTATION_TASK_QUEUE, memoryMutationsEnabled, memoryMutationWorkerOptions } from "./memory-mutation-config.js"; import { DEFAULT_TEMPORAL_ADDRESS, DEFAULT_TEMPORAL_NAMESPACE } from "./constants.js"; import { otlpSpanExporterFromEnv } from "@space/runtime"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { createWorkerTracing } from "./worker-tracing.js"; const service = "space-memory-mutation-worker"; @@ -22,7 +22,7 @@ if (!memoryMutationsEnabled(process.env)) { workflowsPath: fileURLToPath(new URL("./memory-mutation-workflows.js", import.meta.url)), activities: { executePersistedMemoryChangeSet }, ...memoryMutationWorkerOptions, - interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] } + ...createWorkerTracing(spans) }); console.log(JSON.stringify({ service, status: "running", address, namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE })); try { diff --git a/apps/worker/src/room-background-continuation.ts b/apps/worker/src/room-background-continuation.ts index e15c29a21..0a4560c4c 100644 --- a/apps/worker/src/room-background-continuation.ts +++ b/apps/worker/src/room-background-continuation.ts @@ -1,6 +1,7 @@ import { buildCodexAppServerTurnWorkflowId, buildRoomAgentSupervisorWorkflowId, + formatTraceparent, roomAgentSupervisorQueueItemSchema, spaceAgentRoomActionRequestSchema, type RoomAgentActionRecord, @@ -8,6 +9,7 @@ import { } from "@space/contracts"; import { SpaceConflictError, redactMemoryText, type SpaceStore } from "@space/runtime"; import { roomActionSummary } from "./room-action-summary.js"; +import { backgroundMissionTraceContext } from "./temporal-trace-context.js"; import { DEFAULT_ROOM_AGENT_MAX_CONTINUATIONS, DEFAULT_ROOM_AGENT_MAX_MISSION_TOKENS, ROOM_AGENT_CONTINUATION_LIMIT, ROOM_AGENT_TOKEN_LIMIT } from "./room-agent-limits.js"; export function continuationEvidence(actions: RoomAgentActionRecord[], budget: number): string { @@ -122,6 +124,10 @@ maxMissionTokens: number | null = DEFAULT_ROOM_AGENT_MAX_MISSION_TOKENS): Promis ].join("\n"); const prefix = `${instructions}\n\nOperator request:\n${request}\n\nSettled action results (bounded; inspect for details):\n`; const prompt = prefix + continuationEvidence(actions, Math.max(0, 8000 - prefix.length)); + const previousBackground = backgroundMissionTraceContext(source.queueItem.turn); + const continuedTraceparent = previousBackground + ? formatTraceparent(previousBackground.traceId, previousBackground.parentId, previousBackground.flags) + : source.queueItem.turn.traceparent ?? source.queueItem.turn.continuedTraceparent; const item = roomAgentSupervisorQueueItemSchema.parse({ missionId: input.missionId, turn: { ...source.queueItem.turn, prompt, roomAgentOperatorRequest: request, @@ -130,7 +136,12 @@ maxMissionTokens: number | null = DEFAULT_ROOM_AGENT_MAX_MISSION_TOKENS): Promis agentUserMessageId: `agent_msg:bg-context-${input.continuationId}`, agentAssistantMessageId: `agent_msg:bg-response-${input.continuationId}`, // Resolve the current native thread from the durable session when executing. - agentThreadId: null, traceId: `trace:bg-${input.continuationId}` + agentThreadId: null, traceId: `trace:bg-${input.continuationId}`, + // Automatic work gets its own trace. The first continuation links to the + // operator request span it continues; later ones link to the preceding + // background mission span. Persist the link so a lost activity reply cannot change it. + traceparent: undefined, + continuedTraceparent } }); try { const enqueued = await store.enqueueRoomAgentMission({ diff --git a/apps/worker/src/temporal-trace-context.ts b/apps/worker/src/temporal-trace-context.ts new file mode 100644 index 000000000..382f83d94 --- /dev/null +++ b/apps/worker/src/temporal-trace-context.ts @@ -0,0 +1,216 @@ +// Leaf entry points keep the workflow sandbox from evaluating every contract schema per workflow. +import { formatTraceparent, parseTraceparent, type ParsedTraceparent } from "@space/contracts/observability"; +import type { Headers } from "@temporalio/workflow"; + +/** Private Temporal headers contain only W3C identifiers, never workflow inputs. */ +export const TRACEPARENT_HEADER = "space-traceparent"; +export const TRACE_LINK_HEADER = "space-trace-link"; + +export interface TraceSpanLink { + traceId: string; + spanId: string; +} + +export type WorkflowTraceOutcome = "COMPLETED" | "FAILED_RESULT" | "FAILED" | "CANCELLED" | "CONTINUED_AS_NEW"; + +/** Plain, bounded data copied from the workflow isolate to the worker sink. */ +export interface WorkflowSpanRecord { + traceId: string; + spanId: string; + parentSpanId: string | null; + startTimeMs: number; + endTimeMs: number; + outcome: WorkflowTraceOutcome; + operation?: "mission" | "signal" | "update"; + attributes: Readonly>; + links?: readonly TraceSpanLink[]; + reasonCode?: string; +} + +function ascii(value: string): Uint8Array { + return Uint8Array.from(value, (character) => character.charCodeAt(0)); +} + +function decodeHeader(headers: Headers | null | undefined, key: string): ParsedTraceparent | null { + try { + const data = headers?.[key]?.data; + if (!data || data.length > 512) return null; + let value = ""; + for (const byte of data) { + if (byte > 127) return null; + value += String.fromCharCode(byte); + } + // Accept both our binary/plain payload and clients using the SDK JSON converter. + if (value.startsWith('"')) return parseTraceparent(JSON.parse(value)); + return parseTraceparent(value); + } catch { + return null; + } +} + +/** Header propagation also reaches activities with positional or no arguments. */ +export function decodeActivityTraceContext(headers: Headers | null | undefined): ParsedTraceparent | null { + return decodeHeader(headers, TRACEPARENT_HEADER) ?? decodeHeader(headers, "traceparent"); +} + +export function decodeTraceLink(headers: Headers | null | undefined): ParsedTraceparent | null { + return decodeHeader(headers, TRACE_LINK_HEADER); +} + +/** Preserve other interceptors' headers; callers pass the span that actually exists. */ +export function encodeTraceHeaders(headers: Headers, parent: ParsedTraceparent, link?: ParsedTraceparent | null): Headers { + const encoded: Headers = { + ...headers, + [TRACEPARENT_HEADER]: { + metadata: { encoding: ascii("binary/plain") }, + data: ascii(formatTraceparent(parent.traceId, parent.parentId, parent.flags)) + } + }; + // An incoming continue-as-new link describes this run, not every future child. + delete encoded[TRACE_LINK_HEADER]; + if (link) { + encoded[TRACE_LINK_HEADER] = { + metadata: { encoding: ascii("binary/plain") }, + data: ascii(formatTraceparent(link.traceId, link.parentId, link.flags)) + }; + } + return encoded; +} + +const nestedKeys = ["turn", "item", "bridge", "input", "turnInput"] as const; + +function readTraceparent(value: unknown, key: "traceparent" | "continuedTraceparent", depth: number): ParsedTraceparent | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const record = value as Record; + const direct = parseTraceparent(record[key]); + if (direct) return direct; + if (depth === 0) return null; + for (const nested of nestedKeys) { + const found = readTraceparent(record[nested], key, depth - 1); + if (found) return found; + } + return null; +} + +export function traceparentFromArgs(args: readonly unknown[]): ParsedTraceparent | null { + for (const arg of args.slice(0, 8)) { + const found = readTraceparent(arg, "traceparent", 2); + if (found) return found; + } + return null; +} + +export function continuedTraceparentFromArgs(args: readonly unknown[]): ParsedTraceparent | null { + for (const arg of args.slice(0, 8)) { + const found = readTraceparent(arg, "continuedTraceparent", 2); + if (found) return found; + } + return null; +} + +/** Hash only stable identifiers; do not advance Temporal's application random stream. */ +function stableId(value: string): string { + let hash = 0xcbf29ce484222325n; + for (let index = 0; index < value.length; index += 1) { + hash = BigInt.asUintN(64, (hash ^ BigInt(value.charCodeAt(index))) * 0x100000001b3n); + } + return (hash === 0n ? 1n : hash).toString(16).padStart(16, "0"); +} + +export function deterministicWorkflowSpanId(runId: string, scope = "run"): string { + return stableId(`space.workflow:${runId}:${scope}`); +} + +/** A root or background trace without an API parent still has replay-stable identity. */ +export function deterministicWorkflowTraceId(identity: string): string { + return stableId(`space.trace:0:${identity}`) + stableId(`space.trace:1:${identity}`); +} + +const identifierPattern = /^[A-Za-z0-9:._-]{1,200}$/; + +export function traceIdentifier(value: unknown): string | null { + return typeof value === "string" && identifierPattern.test(value) ? value : null; +} + +const correlationKeys = { + traceId: "space.trace_id", + spaceTraceId: "space.trace_id", + requestId: "space.request_id", + parentSpaceTraceId: "space.request_id", + roomId: "space.room_id", + paneId: "space.pane_id", + missionId: "space.mission_id", + roomAgentMissionId: "space.mission_id", + agentRunId: "space.agent_run_id", + agentSessionId: "space.agent_session_id", + actionId: "space.action_id", + goalId: "space.goal_id", + dotId: "space.dot_id", + turnId: "space.turn_id", + cliTaskId: "space.cli_task_id", + revisionId: "space.revision_id" +} as const; + +function readAttributes(value: unknown, into: Record, depth: number): void { + if (!value || typeof value !== "object" || Array.isArray(value)) return; + const record = value as Record; + for (const [source, target] of Object.entries(correlationKeys)) { + if (into[target] !== undefined) continue; + const identifier = traceIdentifier(record[source]); + if (identifier) into[target] = identifier; + } + if (depth === 0) return; + for (const key of nestedKeys) readAttributes(record[key], into, depth - 1); +} + +/** Only named identifier fields are examined; free text is never serialized. */ +export function workflowTraceAttributes(args: readonly unknown[]): Record { + const attributes: Record = {}; + for (const arg of args.slice(0, 8)) readAttributes(arg, attributes, 2); + return attributes; +} + +/** + * Background mission identity survives supervisor run rotation. The host can + * reference this exact span when enqueueing the next background continuation. + */ +export function backgroundMissionTraceContext(input: unknown): ParsedTraceparent | null { + const attributes = workflowTraceAttributes([input]); + const traceId = attributes["space.trace_id"]; + if (!traceId?.startsWith("trace:bg-")) return null; + const identity = attributes["space.agent_run_id"] ?? traceId; + return { + traceId: deterministicWorkflowTraceId(`background:${identity}`), + parentId: deterministicWorkflowSpanId(identity, "background-mission"), + flags: continuedTraceparentFromArgs([input])?.flags ?? "01" + }; +} + +/** + * Space correlation values are generated identifiers shaped `:` + * (for example `room:V1StGXR8_Z5j` or `trace:bg-`). Exported spans accept + * only that shape, so text, file names, paths and URLs placed in an identifier + * field are dropped. This narrows, but cannot prove, where a value came from. + */ +const correlationIdentifierPattern = /^[a-z][a-z0-9_]{0,63}:[A-Za-z0-9_-]{1,128}(?::[A-Za-z0-9_-]{1,128}){0,3}$/; + +export function correlationIdentifier(value: unknown): string | null { + return typeof value === "string" && value.length <= 200 && correlationIdentifierPattern.test(value) ? value : null; +} + +const correlationAttributeKeys = new Set(Object.values(correlationKeys)); +const operationNameKeys = ["space.signal_name", "space.update_name"] as const; + +/** Apply the allowlist again, with the stricter identifier shape, at the host-side export boundary. */ +export function sanitizeWorkflowTraceAttributes(attributes: Readonly>): Record { + const safe: Record = {}; + for (const key of correlationAttributeKeys) { + const identifier = correlationIdentifier(attributes[key]); + if (identifier) safe[key] = identifier; + } + for (const key of operationNameKeys) { + const name = traceIdentifier(attributes[key]); + if (name) safe[key] = name; + } + return safe; +} diff --git a/apps/worker/src/worker-tracing.ts b/apps/worker/src/worker-tracing.ts new file mode 100644 index 000000000..ba3e5a486 --- /dev/null +++ b/apps/worker/src/worker-tracing.ts @@ -0,0 +1,87 @@ +import { fileURLToPath } from "node:url"; +import { categorizeFailureCode, isSafeErrorCode, parseTraceparent } from "@space/contracts"; +import type { SpanExporter } from "@space/runtime"; +import type { InjectedSinks, WorkerOptions } from "@temporalio/worker"; +import type { WorkflowInfo } from "@temporalio/workflow"; +import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { + sanitizeWorkflowTraceAttributes, + traceIdentifier, + type WorkflowSpanRecord +} from "./temporal-trace-context.js"; +import type { WorkflowTracingSinks } from "./workflow-tracing.js"; + +// OTLP timestamps are fixed64 nanoseconds: (2^64 - 1) ns, in whole milliseconds. +const MAX_OTLP_TIME_MS = 18_446_744_073_709; + +function recordWorkflowSpan(spans: SpanExporter, info: WorkflowInfo, span: WorkflowSpanRecord): void { + try { + if (!["COMPLETED", "FAILED_RESULT", "FAILED", "CANCELLED", "CONTINUED_AS_NEW"].includes(span.outcome)) return; + if (!parseTraceparent(`00-${span.traceId}-${span.spanId}-01`)) return; + if (span.parentSpanId !== null && !parseTraceparent(`00-${span.traceId}-${span.parentSpanId}-01`)) return; + if (!Number.isSafeInteger(span.startTimeMs) || span.startTimeMs < 0 || !Number.isSafeInteger(span.endTimeMs)) return; + if (span.startTimeMs > MAX_OTLP_TIME_MS || span.endTimeMs > MAX_OTLP_TIME_MS) return; + const endTimeMs = Math.max(span.startTimeMs, span.endTimeMs); + const workflowType = traceIdentifier(info.workflowType) ?? "unknown"; + const attributes: Record = { + ...sanitizeWorkflowTraceAttributes(span.attributes), + "space.workflow_type": workflowType, + "space.outcome": span.outcome, + "space.duration_ms": endTimeMs - span.startTimeMs + }; + if (Number.isSafeInteger(info.attempt) && info.attempt >= 1) attributes["space.attempt"] = info.attempt; + const workflowId = traceIdentifier(info.workflowId); + const runId = traceIdentifier(info.runId); + if (workflowId) attributes["space.workflow_id"] = workflowId; + if (runId) attributes["space.temporal_run_id"] = runId; + const operation = span.operation === "mission" || span.operation === "signal" || span.operation === "update" ? span.operation : null; + if (operation) attributes["space.workflow_operation"] = operation; + const error = span.outcome === "FAILED" || span.outcome === "FAILED_RESULT"; + if (isSafeErrorCode(span.reasonCode)) attributes["space.reason_code"] = span.reasonCode!; + if (error || span.outcome === "CANCELLED") { + attributes["space.failure_category"] = span.outcome === "CANCELLED" ? "CANCELLED" : categorizeFailureCode(span.reasonCode); + } + const links = (span.links ?? []).slice(0, 16).filter((link) => parseTraceparent(`00-${link.traceId}-${link.spanId}-01`)); + spans.record({ + traceId: span.traceId, + spanId: span.spanId, + parentSpanId: span.parentSpanId, + name: `workflow ${workflowType}${operation ? ` ${operation}` : ""}`, + startTimeUnixNano: BigInt(span.startTimeMs) * 1_000_000n, + endTimeUnixNano: BigInt(endTimeMs) * 1_000_000n, + kind: 1, + attributes, + error, + ...(links.length ? { links } : {}) + }); + } catch { + // The host sink never lets an exporter or malformed telemetry fail a workflow task. + } +} + +/** + * Common tracing composition for every worker. The workflow sandbox only + * copies span data into this host sink; the shared exporter owns all I/O. + */ +export function createWorkerTracing( + spans: SpanExporter | null, + options: { workflowInterceptorModule?: string } = {} +): Pick { + const activity = [createActivityObservabilityInterceptors({ spans })]; + if (!spans) return { interceptors: { activity } }; + const sinks: InjectedSinks = { + spaceTracing: { + record: { + fn: (info, span) => recordWorkflowSpan(spans, info, span), + callDuringReplay: false + } + } + }; + return { + interceptors: { + activity, + workflowModules: [options.workflowInterceptorModule ?? fileURLToPath(new URL("./workflow-tracing.js", import.meta.url))] + }, + sinks + }; +} diff --git a/apps/worker/src/worker.ts b/apps/worker/src/worker.ts index 896269754..1d3b99dd5 100644 --- a/apps/worker/src/worker.ts +++ b/apps/worker/src/worker.ts @@ -6,7 +6,7 @@ import { StreamingCredentialStore } from "@space/streaming"; import { TelegramDeliveryWorker, TelegramSecretStore, otlpSpanExporterFromEnv } from "@space/runtime"; import { NativeConnection, Runtime, Worker } from "@temporalio/worker"; import * as activities from "./activities.js"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { createWorkerTracing } from "./worker-tracing.js"; import { resolveTemporalCoreMetricsBind, resolveWorkerMetricsListen, startWorkerMetricsServer } from "./worker-metrics.js"; import { workerLog } from "./worker-log.js"; import { sweepCodexCliCompletions } from "./codex-cli-completion.js"; @@ -45,7 +45,7 @@ const worker = await Worker.create({ activities, // Structured, correlated activity records and worker metrics. Activity-side // only: the workflow sandbox and its replay behaviour are unchanged. - interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] } + ...createWorkerTracing(spans) }); // Optional loopback-only Prometheus endpoint for worker metrics. diff --git a/apps/worker/src/workflow-tracing.ts b/apps/worker/src/workflow-tracing.ts new file mode 100644 index 000000000..7edb84e9a --- /dev/null +++ b/apps/worker/src/workflow-tracing.ts @@ -0,0 +1,289 @@ +import { + AsyncLocalStorage, + ContinueAsNew, + isCancellation, + proxySinks, + TemporalFailure, + workflowInfo, + type Headers, + type Sinks, + type WorkflowInfo, + type WorkflowInterceptors +} from "@temporalio/workflow"; +import { isSafeErrorCode, type ParsedTraceparent } from "@space/contracts/observability"; +import { ROOM_AGENT_SUPERVISOR_WORKFLOW_TYPE } from "@space/contracts/workflows"; +import { + backgroundMissionTraceContext, + continuedTraceparentFromArgs, + decodeActivityTraceContext, + decodeTraceLink, + deterministicWorkflowSpanId, + deterministicWorkflowTraceId, + encodeTraceHeaders, + traceIdentifier, + traceparentFromArgs, + workflowTraceAttributes, + type TraceSpanLink, + type WorkflowSpanRecord, + type WorkflowTraceOutcome +} from "./temporal-trace-context.js"; + +export interface WorkflowTracingSinks extends Sinks { + spaceTracing: { + record(span: WorkflowSpanRecord): void; + }; +} + +interface TraceScope { + context: ParsedTraceparent; + parentSpanId: string | null; + startTimeMs: number; + operation?: WorkflowSpanRecord["operation"]; + attributes: Record; + links: TraceSpanLink[]; +} + +const scopes = new AsyncLocalStorage(); +const { spaceTracing } = proxySinks(); +let runScope: TraceScope | null = null; +let runInfo: WorkflowInfo | null = null; +let scopeSequence = 0; + +/** + * A supervisor serves many requests. Its run roots its own trace and only + * links to the request that started it, so run-level work (monitoring polls, + * signals and missions without context) never joins that first request's + * trace. Each mission keeps its own request parent; continued runs (which + * carry the link header) stay children of the preceding run. + */ +const requestIndependentWorkflowTypes = new Set([ROOM_AGENT_SUPERVISOR_WORKFLOW_TYPE]); + +function addLink(scope: TraceScope, context: ParsedTraceparent | null | undefined): void { + if (!context || scope.links.length >= 16) return; + if (context.traceId === scope.context.traceId && context.parentId === scope.context.parentId) return; + if (scope.links.some((link) => link.traceId === context.traceId && link.spanId === context.parentId)) return; + scope.links.push({ traceId: context.traceId, spanId: context.parentId }); +} + +function currentScope(): TraceScope | null { + return scopes.getStore() ?? runScope; +} + +function failureCode(error: unknown): string { + let cause = error; + for (let depth = 0; depth < 5 && cause && typeof cause === "object"; depth += 1) { + const record = cause as { code?: unknown; type?: unknown; name?: unknown; cause?: unknown }; + if (isSafeErrorCode(record.code)) return record.code as string; + if (isSafeErrorCode(record.type)) return record.type as string; + if (record.name === "TimeoutFailure") return "WORKFLOW_TIMEOUT"; + cause = record.cause; + } + return "WORKFLOW_ERROR"; +} + +function record(scope: TraceScope, outcome: WorkflowTraceOutcome, reasonCode?: string): void { + try { + spaceTracing.record({ + traceId: scope.context.traceId, + spanId: scope.context.parentId, + parentSpanId: scope.parentSpanId, + startTimeMs: scope.startTimeMs, + endTimeMs: Math.max(scope.startTimeMs, Date.now()), + outcome, + operation: scope.operation, + attributes: scope.attributes, + links: scope.links, + reasonCode + }); + } catch { + // Sink/serialization failures must not replace a workflow's result or error. + } +} + +/** + * `sdkEntry` marks the workflow function and signal/update handlers: a + * non-Temporal error thrown there fails the workflow task (Temporal retries it), + * so no completed span exists yet. Mission callbacks may have their errors caught. + */ +async function observe(scope: TraceScope, execute: () => Promise, sdkEntry = false): Promise { + let outcome: WorkflowTraceOutcome = "COMPLETED"; + let reasonCode: string | undefined; + let invoked = false; + let taskFailure = false; + try { + let result: T; + try { + result = await scopes.run(scope, () => { + invoked = true; + return execute(); + }); + } catch (error) { + // An instrumentation setup failure still executes application code exactly once. + if (invoked) throw error; + result = await execute(); + } + try { + if (result && typeof result === "object") { + const status = (result as { status?: unknown }).status; + if (status === "FAILED") { + outcome = "FAILED_RESULT"; + reasonCode = "WORKFLOW_FAILED_RESULT"; + } else if (status === "CANCELLED" || status === "INTERRUPTED") { + outcome = "CANCELLED"; + reasonCode = "WORKFLOW_CANCELLED"; + } + } + } catch {} + return result; + } catch (error) { + try { + if (error instanceof ContinueAsNew) outcome = "CONTINUED_AS_NEW"; + else if (isCancellation(error)) { + outcome = "CANCELLED"; + reasonCode = "WORKFLOW_CANCELLED"; + } else if (sdkEntry && !(error instanceof TemporalFailure)) { + // A non-Temporal error fails the workflow task, which Temporal retries; + // the execution is still running, so no completed span exists yet. + taskFailure = true; + } else { + outcome = "FAILED"; + reasonCode = failureCode(error); + } + } catch { + outcome = "FAILED"; + reasonCode = "WORKFLOW_ERROR"; + } + throw error; + } finally { + if (!taskFailure) record(scope, outcome, reasonCode); + } +} + +function makeOperationScope( + args: readonly unknown[], + operation: NonNullable, + headers?: Headers +): TraceScope | null { + if (!runInfo || !runScope) return null; + const enclosing = currentScope() ?? runScope; + const incoming = decodeActivityTraceContext(headers) ?? traceparentFromArgs(args); + const continued = decodeTraceLink(headers) ?? continuedTraceparentFromArgs(args); + const background = operation === "mission" ? backgroundMissionTraceContext(args[0]) : null; + // A background mission always roots its own trace. A legacy queued item may + // still carry its source request's traceparent: keep that as a link only. + const parent = background ? null : incoming ?? (continued ? null : enclosing.context); + scopeSequence += 1; + const context = background ?? { + traceId: parent?.traceId ?? deterministicWorkflowTraceId(`${runInfo.runId}:${operation}:${scopeSequence}`), + parentId: deterministicWorkflowSpanId(runInfo.runId, `${operation}:${scopeSequence}`), + flags: parent?.flags ?? continued?.flags ?? "01" + }; + const scope: TraceScope = { + context, + parentSpanId: parent?.parentId ?? null, + startTimeMs: Date.now(), + operation, + attributes: workflowTraceAttributes(args), + links: [] + }; + addLink(scope, continued); + if (background) addLink(scope, incoming); + if (parent?.parentId !== enclosing.context.parentId || parent?.traceId !== enclosing.context.traceId) { + addLink(scope, enclosing.context); + } + return scope; +} + +/** + * A supervisor outlives the requests delivered through its signals. Give each + * queued mission its own completed span and asynchronous propagation scope. + * Importing this helper with tracing disabled adds no headers or sink calls. + */ +export async function withWorkflowTraceContext(input: unknown, execute: () => Promise): Promise { + let scope: TraceScope | null = null; + try { scope = makeOperationScope([input], "mission"); } catch {} + return scope ? observe(scope, execute) : execute(); +} + +function propagate(input: T, continueRun = false): T { + try { + const scope = currentScope(); + if (scope) { + return { ...input, headers: encodeTraceHeaders(input.headers, scope.context, continueRun ? runScope?.context : null) }; + } + } catch {} + return input; +} + +/** Loaded only by an enabled worker's workflowModules; contains no host I/O. */ +export function interceptors(): WorkflowInterceptors { + runScope = null; + runInfo = null; + scopeSequence = 0; + return { + inbound: [{ + async execute(input, next) { + let scope: TraceScope | null = null; + try { + const info = workflowInfo(); + const incoming = decodeActivityTraceContext(input.headers) + ?? traceparentFromArgs(input.args) + ?? traceparentFromArgs([info.memo]); + const continued = decodeTraceLink(input.headers) ?? continuedTraceparentFromArgs(input.args); + const detached = requestIndependentWorkflowTypes.has(info.workflowType) && !decodeTraceLink(input.headers); + const parent = detached ? null : incoming; + scope = { + context: { + traceId: parent?.traceId ?? deterministicWorkflowTraceId(info.firstExecutionRunId), + parentId: deterministicWorkflowSpanId(info.runId), + flags: parent?.flags ?? incoming?.flags ?? continued?.flags ?? "01" + }, + parentSpanId: parent?.parentId ?? null, + startTimeMs: info.runStartTime.getTime(), + // The starting request's memo ids describe one mission, not the supervisor. + attributes: detached ? workflowTraceAttributes(input.args) + : { ...workflowTraceAttributes([info.memo]), ...workflowTraceAttributes(input.args) }, + links: [] + }; + addLink(scope, continued); + if (detached) addLink(scope, incoming); + if (!continued && info.continuedFromExecutionRunId) { + addLink(scope, { + traceId: scope.context.traceId, + parentId: deterministicWorkflowSpanId(info.continuedFromExecutionRunId), + flags: scope.context.flags + }); + } + runInfo = info; + runScope = scope; + } catch {} + return scope ? observe(scope, () => next(input), true) : next(input); + }, + async handleSignal(input, next) { + let scope: TraceScope | null = null; + try { + scope = makeOperationScope(input.args, "signal", input.headers); + const name = traceIdentifier(input.signalName); + if (scope && name) scope.attributes["space.signal_name"] = name; + } catch {} + return scope ? observe(scope, () => next(input), true) : next(input); + }, + async handleUpdate(input, next) { + let scope: TraceScope | null = null; + try { + scope = makeOperationScope(input.args, "update", input.headers); + const name = traceIdentifier(input.name); + if (scope && name) scope.attributes["space.update_name"] = name; + } catch {} + return scope ? observe(scope, () => next(input), true) : next(input); + } + }], + outbound: [{ + scheduleActivity: (input, next) => next(propagate(input)), + scheduleLocalActivity: (input, next) => next(propagate(input)), + startChildWorkflowExecution: (input, next) => next(propagate(input)), + signalWorkflow: (input, next) => next(propagate(input)), + continueAsNew: (input, next) => next(propagate(input, true)) + }] + }; +} diff --git a/apps/worker/src/workflows.ts b/apps/worker/src/workflows.ts index 5f3a7d4cd..b668e8c0a 100644 --- a/apps/worker/src/workflows.ts +++ b/apps/worker/src/workflows.ts @@ -39,6 +39,7 @@ import { } from "./room-supervisor-state.js"; import { isCliChatTurnProviderId, isNativeChatTurn, NATIVE_CHAT_TURN_ACTIVITY_TIMEOUT } from "./turn-runtime-policy.js"; import { runDotGoalLoop } from "./dot-goal-loop.js"; +import { withWorkflowTraceContext } from "./workflow-tracing.js"; const { recordDummyTurnStarted, recordDummyTurnCompleted } = proxyActivities({ startToCloseTimeout: "1 minute", @@ -187,61 +188,69 @@ async function legacyRoomAgentSupervisorWorkflow(rawInput: RoomAgentSupervisorIn const stoppedBeforeStart = interrupted.shift(); if (stoppedBeforeStart) { - await markRoomAgentMissionFinished({ - missionId: stoppedBeforeStart.item.missionId, - roomId: stoppedBeforeStart.item.turn.roomId, - status: "INTERRUPTED", - statusReason: stoppedBeforeStart.reason + await withWorkflowTraceContext(stoppedBeforeStart.item, async () => { + await markRoomAgentMissionFinished({ + missionId: stoppedBeforeStart.item.missionId, + roomId: stoppedBeforeStart.item.turn.roomId, + status: "INTERRUPTED", + statusReason: stoppedBeforeStart.reason + }); + return { status: "INTERRUPTED" as const }; }); processedCount += 1; } else { const item = pending.shift()!; activeMissionId = item.missionId; activeStopReason = "Stopped by operator."; - activeScope = new CancellationScope({ cancellable: true }); - await markRoomAgentMissionStarted({ missionId: item.missionId, roomId: item.turn.roomId, paneId: item.turn.paneId }); - try { - const result = await activeScope.run(() => - executeChild(codexAppServerTurnWorkflow, { - args: [item.turn], - workflowId: buildCodexAppServerTurnWorkflowId(item.turn), - cancellationType: ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED - }) - ); - const completion = roomAgentMissionCompletion(result); - if (result.roomAgentOutcome?.status === "VERIFIED" && result.roomAgentOutcome.pendingActionIds?.length) { - monitoring.add(item.missionId); - await markRoomAgentMissionContinued({ missionId: item.missionId, roomId: item.turn.roomId, - statusReason: "Pane work is running independently. Room Agent is ready for further commands." }); - } else if (monitoring.has(item.missionId) || (completion.status === "COMPLETED" && roomAgentMissionHasContinuation(pending, item.missionId))) { - await markRoomAgentMissionContinued({ - missionId: item.missionId, - roomId: item.turn.roomId, - statusReason: "The current turn completed; continuing the same goal with the next operator follow-up." - }); - } else { - await markRoomAgentMissionFinished({ - missionId: item.missionId, - roomId: item.turn.roomId, - status: completion.status, - statusReason: completion.statusReason - }); + const turnScope = new CancellationScope({ cancellable: true }); + activeScope = turnScope; + await withWorkflowTraceContext(item, async () => { + await markRoomAgentMissionStarted({ missionId: item.missionId, roomId: item.turn.roomId, paneId: item.turn.paneId }); + try { + const result = await turnScope.run(() => + executeChild(codexAppServerTurnWorkflow, { + args: [item.turn], + workflowId: buildCodexAppServerTurnWorkflowId(item.turn), + cancellationType: ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED + }) + ); + const completion = roomAgentMissionCompletion(result); + if (result.roomAgentOutcome?.status === "VERIFIED" && result.roomAgentOutcome.pendingActionIds?.length) { + monitoring.add(item.missionId); + await markRoomAgentMissionContinued({ missionId: item.missionId, roomId: item.turn.roomId, + statusReason: "Pane work is running independently. Room Agent is ready for further commands." }); + } else if (monitoring.has(item.missionId) || (completion.status === "COMPLETED" && roomAgentMissionHasContinuation(pending, item.missionId))) { + await markRoomAgentMissionContinued({ + missionId: item.missionId, + roomId: item.turn.roomId, + statusReason: "The current turn completed; continuing the same goal with the next operator follow-up." + }); + } else { + await markRoomAgentMissionFinished({ + missionId: item.missionId, + roomId: item.turn.roomId, + status: completion.status, + statusReason: completion.statusReason + }); + } + return completion; + } catch (error) { + await CancellationScope.nonCancellable(() => + markRoomAgentMissionFinished({ + missionId: item.missionId, + roomId: item.turn.roomId, + status: isCancellation(error) ? "INTERRUPTED" : "FAILED", + statusReason: isCancellation(error) ? activeStopReason : "Room agent child workflow failed.", + ...(isCancellation(error) ? {} : { errorCode: supervisorFailureCode(error) }) + }) + ); + return { status: isCancellation(error) ? "INTERRUPTED" as const : "FAILED" as const }; + } finally { + activeMissionId = null; + activeScope = null; + processedCount += 1; } - } catch (error) { - await CancellationScope.nonCancellable(() => - markRoomAgentMissionFinished({ - missionId: item.missionId, - roomId: item.turn.roomId, - status: isCancellation(error) ? "INTERRUPTED" : "FAILED", - statusReason: isCancellation(error) ? activeStopReason : "Room agent child workflow failed.", - ...(isCancellation(error) ? {} : { errorCode: supervisorFailureCode(error) }) - }) - ); - } finally { - activeMissionId = null; - activeScope = null; - processedCount += 1; - } + }); } if (processedCount >= 50) { @@ -307,49 +316,57 @@ async function continuingRoomAgentSupervisorWorkflow(rawInput: RoomAgentSupervis for (;;) { if (interrupted.length) { const stopped = interrupted.shift()!; - await supervisorActivities.discardRoomAgentSupervisorTurn(stopped); - // A discarded automatic turn must never terminate its newer operator turn. - await supervisorActivities.finishRoomAgentSupervisorTurn({ item: stopped.item, status: "INTERRUPTED", statusReason: stopped.reason }); + await withWorkflowTraceContext(stopped.item, async () => { + await supervisorActivities.discardRoomAgentSupervisorTurn(stopped); + // A discarded automatic turn must never terminate its newer operator turn. + await supervisorActivities.finishRoomAgentSupervisorTurn({ item: stopped.item, status: "INTERRUPTED", statusReason: stopped.reason }); + return { status: "INTERRUPTED" as const }; + }); processedCount++; } else if (pending.length) { const item = pending.shift()!; active = item; - scope = new CancellationScope({ cancellable: true }); + const turnScope = new CancellationScope({ cancellable: true }); + scope = turnScope; stopReason = "Stopped by operator."; let paused = false; - try { - await scope.run(async () => { - const state = await supervisorActivities.checkRoomAgentSupervisorTurn(item); - if (state === "PAUSED") { paused = true; return; } - if (state === "SKIP") { - await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason: "Turn is no longer active or was superseded." }); - return; - } - const result = await executeChild(codexAppServerTurnWorkflow, { - args: [item.turn], workflowId: buildCodexAppServerTurnWorkflowId(item.turn), - cancellationType: ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED + await withWorkflowTraceContext(item, async () => { + try { + return await turnScope.run(async () => { + const state = await supervisorActivities.checkRoomAgentSupervisorTurn(item); + if (state === "PAUSED") { paused = true; return; } + if (state === "SKIP") { + await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason: "Turn is no longer active or was superseded." }); + return; + } + const result = await executeChild(codexAppServerTurnWorkflow, { + args: [item.turn], workflowId: buildCodexAppServerTurnWorkflowId(item.turn), + cancellationType: ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED + }); + const completion = roomAgentMissionCompletion(result); + if (result.roomAgentOutcome?.status === "VERIFIED" && result.roomAgentOutcome.pendingActionIds?.length) { + monitoring.add(item.missionId); + } else if (completion.status === "COMPLETED" && monitoring.has(item.missionId) && !item.turn.roomAgentContinuation) { + // A FIND/layout follow-up can finish while earlier pane tasks continue. + } else if (!roomAgentMissionHasContinuation(pending, item.missionId)) { + monitoring.delete(item.missionId); + await supervisorActivities.finishRoomAgentSupervisorTurn({ item, ...completion }); + } + return completion; }); - const completion = roomAgentMissionCompletion(result); - if (result.roomAgentOutcome?.status === "VERIFIED" && result.roomAgentOutcome.pendingActionIds?.length) { - monitoring.add(item.missionId); - } else if (completion.status === "COMPLETED" && monitoring.has(item.missionId) && !item.turn.roomAgentContinuation) { - // A FIND/layout follow-up can finish while earlier pane tasks continue. - } else if (!roomAgentMissionHasContinuation(pending, item.missionId)) { - monitoring.delete(item.missionId); - await supervisorActivities.finishRoomAgentSupervisorTurn({ item, ...completion }); - } - }); - } catch (error) { - await CancellationScope.nonCancellable(async () => { - const reason = isCancellation(error) ? stopReason : "Room Agent supervisor turn failed."; - const errorCode = isCancellation(error) ? undefined : supervisorFailureCode(error); - await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason, errorCode }); - if (!roomAgentMissionHasContinuation(pending, item.missionId)) { - monitoring.delete(item.missionId); - await supervisorActivities.finishRoomAgentSupervisorTurn({ item, status: errorCode ? "FAILED" : "INTERRUPTED", statusReason: reason, errorCode }); - } - }); - } finally { active = null; scope = null; processedCount++; } + } catch (error) { + await CancellationScope.nonCancellable(async () => { + const reason = isCancellation(error) ? stopReason : "Room Agent supervisor turn failed."; + const errorCode = isCancellation(error) ? undefined : supervisorFailureCode(error); + await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason, errorCode }); + if (!roomAgentMissionHasContinuation(pending, item.missionId)) { + monitoring.delete(item.missionId); + await supervisorActivities.finishRoomAgentSupervisorTurn({ item, status: errorCode ? "FAILED" : "INTERRUPTED", statusReason: reason, errorCode }); + } + }); + return { status: isCancellation(error) ? "INTERRUPTED" as const : "FAILED" as const }; + } finally { active = null; scope = null; processedCount++; } + }); if (paused) { pending.push(item); const revision = signalRevision; diff --git a/apps/worker/tests/activity-observability.test.ts b/apps/worker/tests/activity-observability.test.ts index 654bf7962..9ae3884f9 100644 --- a/apps/worker/tests/activity-observability.test.ts +++ b/apps/worker/tests/activity-observability.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { request } from "node:http"; -import { MetricsRegistry } from "@space/runtime"; +import { MetricsRegistry, type ServerSpan, type SpanExporter } from "@space/runtime"; import { activityCorrelation, childTraceparent, @@ -37,6 +37,23 @@ function capture() { return { records, lines, restore }; } +function captureSpans() { + const recorded: ServerSpan[] = []; + const spans: SpanExporter = { + record: span => void recorded.push(span), + flush: async () => undefined, + close: async () => undefined, + stats: () => ({ exported: recorded.length, dropped: 0, failed: 0 }) + }; + return { recorded, spans }; +} + +// A Temporal binary/plain header, constructed independently of the encoder. +function traceHeaders(value: string) { + const encoder = new TextEncoder(); + return { "space-traceparent": { metadata: { encoding: encoder.encode("binary/plain") }, data: encoder.encode(value) } }; +} + let restoreLogs: (() => void) | null = null; afterEach(() => { restoreLogs?.(); restoreLogs = null; }); @@ -135,6 +152,127 @@ describe("observeActivityExecution", () => { expect(value).toBe("done"); expect(logs.records.map((record) => record.event)).toEqual(["activity.started", "activity.finished"]); }); + + it.each([ + { name: "no arguments", args: [] }, + { name: "legacy arguments", args: [{ traceparent, prompt: secretPrompt }] } + ])("uses the workflow header for activities with $name", async ({ args }) => { + const logs = capture(); restoreLogs = logs.restore; + const { recorded, spans } = captureSpans(); + const metrics = createWorkerMetrics(new MetricsRegistry()); + const workflowParent = "00-11111111111111111111111111111111-2222222222222222-01"; + const interceptors = createActivityObservabilityInterceptors({ metrics, spans })({ + info: info(), cancellationSignal: new AbortController().signal + } as never); + const input = { args, headers: traceHeaders(workflowParent) }; + const result = { value: secretPrompt }; + const returned = await interceptors.inbound!.execute!(input, async nextInput => { + expect(nextInput).toBe(input); + return result; + }); + expect(returned).toBe(result); + expect(recorded).toHaveLength(1); + expect(recorded[0]).toMatchObject({ traceId: "11111111111111111111111111111111", parentSpanId: "2222222222222222", kind: 1 }); + expect(logs.records[0]).toMatchObject({ w3cTraceId: "11111111111111111111111111111111" }); + expect(JSON.stringify(recorded[0]!.attributes)).not.toContain(secretPrompt); + }); + + it("falls back to the legacy argument context when a workflow header is malformed", async () => { + const logs = capture(); restoreLogs = logs.restore; + const { recorded, spans } = captureSpans(); + const interceptors = createActivityObservabilityInterceptors({ metrics: createWorkerMetrics(new MetricsRegistry()), spans })({ + info: info(), cancellationSignal: new AbortController().signal + } as never); + await interceptors.inbound!.execute!({ args: [{ turn: { traceparent } }], headers: traceHeaders("invalid") }, async () => "ok"); + await interceptors.inbound!.execute!({ args: [], headers: traceHeaders("invalid") }, async () => "ok"); + expect(recorded).toHaveLength(1); + expect(recorded[0]).toMatchObject({ traceId: "4bf92f3577b34da6a3ce929d0e0e4736", parentSpanId: "00f067aa0ba902b7" }); + }); + + it("gives retry attempts distinct spans with safe failure and correlation metadata", async () => { + const logs = capture(); restoreLogs = logs.restore; + const { recorded, spans } = captureSpans(); + const metrics = createWorkerMetrics(new MetricsRegistry()); + let clock = 1_000; + const args = [{ turn: { traceparent, traceId: "req:abcdefghijkl", requestId: "req:parent00001", + roomId: "room:1", paneId: "pane:1", roomAgentMissionId: "mission:1", agentRunId: "agent_run:1", + prompt: secretPrompt, output: secretPrompt, path: "/private/transcript", url: "https://private.example/?token=secret", + headers: { authorization: secretPrompt } } }]; + const failedResult = { status: "FAILED", message: secretPrompt }; + expect(await observeActivityExecution(info(), args, async () => { + noteActivityFailureReason("CODEX_APP_SERVER_TURN_TIMEOUT"); + clock += 125; + return failedResult; + }, { metrics, spans, now: () => clock })).toBe(failedResult); + expect(await observeActivityExecution(info({ attempt: 2 }), args, async () => { + clock += 75; + return "ok"; + }, { metrics, spans, now: () => clock })).toBe("ok"); + + expect(recorded).toHaveLength(2); + expect(recorded[0]!.spanId).not.toBe(recorded[1]!.spanId); + for (const exported of recorded) { + expect(exported.spanId).toMatch(/^[0-9a-f]{16}$/); + expect(exported.spanId).not.toBe("0000000000000000"); + expect(exported).toMatchObject({ traceId: "4bf92f3577b34da6a3ce929d0e0e4736", parentSpanId: "00f067aa0ba902b7" }); + expect(exported.attributes).toMatchObject({ "space.activity_id": "3", "space.workflow_type": "codexAppServerTurnWorkflow", + "space.workflow_id": info().workflowExecution.workflowId, "space.temporal_run_id": "run-uuid-1", + "space.trace_id": "req:abcdefghijkl", "space.request_id": "req:parent00001", "space.room_id": "room:1", + "space.pane_id": "pane:1", "space.mission_id": "mission:1", "space.agent_run_id": "agent_run:1" }); + } + expect(recorded[0]).toMatchObject({ error: true, attributes: { "space.attempt": 1, "space.outcome": "FAILED_RESULT", + "space.reason_code": "CODEX_APP_SERVER_TURN_TIMEOUT", "space.failure_category": "TIMEOUT", "space.duration_ms": 125 } }); + expect(recorded[1]).toMatchObject({ error: false, attributes: { "space.attempt": 2, "space.outcome": "COMPLETED", "space.duration_ms": 75 } }); + expect(recorded[1]!.attributes).not.toHaveProperty("space.reason_code"); + const exportedText = JSON.stringify(recorded.map(({ name, attributes }) => ({ name, attributes }))); + expect(exportedText).not.toMatch(/PRIVATE PROMPT|sk-live|private\.example|transcript|authorization/); + }); + + it("parents outbound calls to the exported attempt and keeps concurrent activity scopes separate", async () => { + const logs = capture(); restoreLogs = logs.restore; + const { recorded, spans } = captureSpans(); + const metrics = createWorkerMetrics(new MetricsRegistry()); + const outgoing = await Promise.all(["1", "2"].map(activityId => observeActivityExecution(info({ activityId }), [{ traceparent }], async () => { + await Promise.resolve(); + return { activityId, traceparent: childTraceparent(null) }; + }, { metrics, spans }))); + expect(recorded).toHaveLength(2); + for (const call of outgoing) { + const activity = recorded.find(span => span.attributes["space.activity_id"] === call.activityId)!; + expect(call.traceparent).toBe(`00-${activity.traceId}-${activity.spanId}-01`); + } + expect(outgoing[0]!.traceparent).not.toBe(outgoing[1]!.traceparent); + expect(childTraceparent(null)).toBeNull(); + }); + + it("never lets a throwing span exporter replace a result or an original activity failure", async () => { + const logs = capture(); restoreLogs = logs.restore; + const { recorded, spans } = captureSpans(); + let flushes = 0; + spans.record = span => { recorded.push(span); throw new Error("collector is unavailable"); }; + spans.flush = async () => { flushes += 1; }; + const dependencies = { metrics: createWorkerMetrics(new MetricsRegistry()), spans }; + const result = { status: "COMPLETED", value: secretPrompt }; + expect(await observeActivityExecution(info(), [{ traceparent }], async () => result, dependencies)).toBe(result); + const failure = Object.assign(new Error(secretPrompt), { code: "TOOL_UNAVAILABLE" }); + await expect(observeActivityExecution(info(), [{ traceparent }], async () => { throw failure; }, dependencies)).rejects.toBe(failure); + expect(recorded.map(span => span.attributes["space.outcome"])).toEqual(["COMPLETED", "FAILED"]); + expect(recorded[1]).toMatchObject({ error: true, attributes: { "space.reason_code": "TOOL_UNAVAILABLE", "space.failure_category": "TOOL_UNAVAILABLE" } }); + expect(flushes).toBe(0); + }); + + it("keeps telemetry failures and cancellation classification from changing business outcomes", async () => { + const logs = capture(); restoreLogs = logs.restore; + const { recorded, spans } = captureSpans(); + const metrics = createWorkerMetrics(new MetricsRegistry()); + metrics.activitiesInFlight.inc = () => { throw new Error("broken metrics sink"); }; + expect(await observeActivityExecution(info(), [{ traceparent }], async () => "ok", { metrics, spans })).toBe("ok"); + const failure = new Error(secretPrompt); + await expect(observeActivityExecution(info(), [{ traceparent }], async () => { throw failure; }, { metrics, spans, cancelled: () => true })) + .rejects.toBe(failure); + expect(recorded.map(span => span.attributes["space.outcome"])).toEqual(["COMPLETED", "CANCELLED"]); + expect(recorded[1]).toMatchObject({ error: false, attributes: { "space.failure_category": "CANCELLED" } }); + }); }); describe("worker log", () => { @@ -184,9 +322,18 @@ describe("worker metrics endpoint", () => { it("registers the activity interceptor on every Temporal worker, including both memory workers", async () => { const { readFile } = await import("node:fs/promises"); + const { createWorkerTracing } = await import("../src/worker-tracing.js"); + const logs = capture(); restoreLogs = logs.restore; + const tracing = createWorkerTracing(null); + expect(tracing.interceptors?.activity).toHaveLength(1); + const interceptors = tracing.interceptors!.activity![0]!({ + info: info(), cancellationSignal: new AbortController().signal + } as never); + await interceptors.inbound!.execute!({ args: [], headers: {} }, async () => "ok"); + expect(logs.records.map(record => record.event)).toEqual(["activity.started", "activity.finished"]); for (const entry of ["worker.ts", "memory-maintenance-worker.ts", "memory-mutation-worker.ts"]) { const source = await readFile(new URL(`../src/${entry}`, import.meta.url), "utf8"); - expect(source, entry).toMatch(/interceptors: \{ activity: \[createActivityObservabilityInterceptors\(\{ spans \}\)\] \}/); + expect(source, entry).toMatch(/\.\.\.createWorkerTracing\(spans\)/); } }); @@ -203,3 +350,16 @@ it("exports one INTERNAL span per activity under the caller's traceparent", asyn expect(recorded[0]).toMatchObject({ traceId: "4bf92f3577b34da6a3ce929d0e0e4736", parentSpanId: "00f067aa0ba902b7", name: "activity runTurn", kind: 1, error: false, startTimeUnixNano: 1_000_000_000n, endTimeUnixNano: 1_250_000_000n }); }); + +it("exports only generated correlation identifiers and a valid attempt on activity spans", async () => { + const { recorded, spans } = captureSpans(); + const args = [{ turn: { traceparent, traceId: "req:abcdefghijkl", roomId: "room:1", + roomAgentMissionId: "PRIVATE-PROMPT-and-tool-output-must-stay-local", paneId: "private-notes.txt" } }]; + await observeActivityExecution(info(), args, async () => "ok", { spans }); + await observeActivityExecution(info({ attempt: -1 }), args, async () => "ok", { spans }); + expect(recorded[0]!.attributes).toMatchObject({ "space.room_id": "room:1", "space.trace_id": "req:abcdefghijkl", "space.attempt": 1 }); + expect(recorded[0]!.attributes).not.toHaveProperty("space.mission_id"); + expect(recorded[0]!.attributes).not.toHaveProperty("space.pane_id"); + expect(JSON.stringify(recorded.map(({ name, attributes }) => ({ name, attributes })))).not.toMatch(/PRIVATE-PROMPT|private-notes/); + expect(recorded[1]!.attributes).not.toHaveProperty("space.attempt"); +}); diff --git a/apps/worker/tests/fixtures/histories/room-agent-supervisor-1.1.6.json b/apps/worker/tests/fixtures/histories/room-agent-supervisor-1.1.6.json new file mode 100644 index 000000000..2e122f0b6 --- /dev/null +++ b/apps/worker/tests/fixtures/histories/room-agent-supervisor-1.1.6.json @@ -0,0 +1,3005 @@ +{ + "events": [ + { + "eventId": "1", + "eventTime": { + "seconds": "1792404557", + "nanos": 997000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_STARTED", + "workflowExecutionStartedEventAttributes": { + "workflowType": { + "name": "roomAgentSupervisorWorkflow" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGVuZGluZyI6W10sInByb2Nlc3NlZENvdW50IjowfQ==" + } + ] + }, + "workflowExecutionTimeout": { + "seconds": "315360000" + }, + "workflowRunTimeout": { + "seconds": "315360000" + }, + "workflowTaskTimeout": { + "seconds": "10" + }, + "originalExecutionRunId": "c332edcf-4108-439a-8bcd-1ed7d3e96c9c", + "identity": "replay-fixture-worker", + "firstExecutionRunId": "c332edcf-4108-439a-8bcd-1ed7d3e96c9c", + "attempt": 1, + "firstWorkflowTaskBackoff": {}, + "memo": { + "fields": { + "spaceTraceId": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "InJlcTpyZXBsYXktZml4dHVyZS1hIg==" + }, + "missionId": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "InJvb21fYWdlbnRfbWlzc2lvbjphIg==" + } + } + }, + "header": {} + } + }, + { + "eventId": "2", + "eventTime": { + "seconds": "1792404557", + "nanos": 997000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "3", + "eventTime": { + "seconds": "1792404557", + "nanos": 999000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "enqueueRoomAgentMission", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YSIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjphIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmEiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWEiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWEiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1hIn19" + } + ] + }, + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "4", + "eventTime": { + "seconds": "1792404558" + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "2", + "identity": "replay-fixture-worker", + "historySizeBytes": "1096" + } + }, + { + "eventId": "5", + "eventTime": { + "seconds": "1792404561", + "nanos": 169000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "2", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": { + "coreUsedFlags": [ + 2, + 1, + 3 + ], + "langUsedFlags": [ + 2 + ], + "sdkName": "temporal-typescript", + "sdkVersion": "1.24.0" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "6", + "eventTime": { + "seconds": "1792404561", + "nanos": 169000000 + }, + "eventType": "EVENT_TYPE_MARKER_RECORDED", + "markerRecordedEventAttributes": { + "markerName": "core_patch", + "details": { + "patch-data": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJpZCI6InJvb20tYmFja2dyb3VuZC1jb250aW51YXRpb24tdjEiLCJkZXByZWNhdGVkIjpmYWxzZX0=" + } + ] + } + }, + "workflowTaskCompletedEventId": "4" + } + }, + { + "eventId": "7", + "eventTime": { + "seconds": "1792404561", + "nanos": 169000000 + }, + "eventType": "EVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES", + "upsertWorkflowSearchAttributesEventAttributes": { + "workflowTaskCompletedEventId": "4", + "searchAttributes": { + "indexedFields": { + "TemporalChangeVersion": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==", + "type": "S2V5d29yZExpc3Q=" + }, + "data": "WyJyb29tLWJhY2tncm91bmQtY29udGludWF0aW9uLXYxIl0=" + } + } + } + } + }, + { + "eventId": "8", + "eventTime": { + "seconds": "1792404561", + "nanos": 169000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "1", + "activityType": { + "name": "checkRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YSIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjphIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmEiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWEiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWEiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1hIn19" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "4", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "9", + "eventTime": { + "seconds": "1792404561", + "nanos": 169000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "enqueueRoomAgentMission", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YiIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjpiIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmIiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWIiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWIiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1iIn19" + } + ] + }, + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "10", + "eventTime": { + "seconds": "1792404561", + "nanos": 169000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 2 + } + }, + { + "eventId": "11", + "eventTime": { + "seconds": "1792404561", + "nanos": 170000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "10", + "identity": "replay-fixture-worker", + "historySizeBytes": "2897" + } + }, + { + "eventId": "12", + "eventTime": { + "seconds": "1792404561", + "nanos": 260000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "10", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "13", + "eventTime": { + "seconds": "1792404561", + "nanos": 170000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "8", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "14", + "eventTime": { + "seconds": "1792404561", + "nanos": 260000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "IlJFQURZIg==" + } + ] + }, + "scheduledEventId": "8", + "startedEventId": "12", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "15", + "eventTime": { + "seconds": "1792404561", + "nanos": 260000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "16", + "eventTime": { + "seconds": "1792404561", + "nanos": 260000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "15", + "identity": "replay-fixture-worker", + "historySizeBytes": "3278" + } + }, + { + "eventId": "17", + "eventTime": { + "seconds": "1792404561", + "nanos": 332000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "15", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "18", + "eventTime": { + "seconds": "1792404561", + "nanos": 332000000 + }, + "eventType": "EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-a", + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmEiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YSIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtYSIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtYSIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWEifQ==" + } + ] + }, + "workflowExecutionTimeout": {}, + "workflowRunTimeout": {}, + "workflowTaskTimeout": { + "seconds": "10" + }, + "workflowTaskCompletedEventId": "16", + "header": {}, + "memo": {} + } + }, + { + "eventId": "19", + "eventTime": { + "seconds": "1792404561", + "nanos": 340000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "18", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-a", + "runId": "4ea6b62b-d548-4d02-9ef7-d884bd5d842b" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + } + } + }, + { + "eventId": "20", + "eventTime": { + "seconds": "1792404561", + "nanos": 340000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "21", + "eventTime": { + "seconds": "1792404561", + "nanos": 341000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "20", + "identity": "replay-fixture-worker", + "historySizeBytes": "4408" + } + }, + { + "eventId": "22", + "eventTime": { + "seconds": "1792404563", + "nanos": 991000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "20", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "23", + "eventTime": { + "seconds": "1792404564", + "nanos": 16000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED", + "childWorkflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ3b3JrZmxvd0lkIjoid29ya2Zsb3c6Y29kZXgtYXBwLXNlcnZlcjpyb29tOnJlcGxheS1maXh0dXJlOnBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQ6cmVxOnJlcGxheS1maXh0dXJlLWEiLCJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInRyYWNlSWQiOiJyZXE6cmVwbGF5LWZpeHR1cmUtYSIsInN0YXR1cyI6IkNPTVBMRVRFRCIsIm1lc3NhZ2UiOiJzeW50aGV0aWMiLCJyb29tQWdlbnRPdXRjb21lIjp7InN0YXR1cyI6IlZFUklGSUVEIiwic3RhdHVzUmVhc29uIjoib2sifX0=" + } + ] + }, + "namespace": "default", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-a", + "runId": "4ea6b62b-d548-4d02-9ef7-d884bd5d842b" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "initiatedEventId": "18", + "startedEventId": "19" + } + }, + { + "eventId": "24", + "eventTime": { + "seconds": "1792404564", + "nanos": 16000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "25", + "eventTime": { + "seconds": "1792404564", + "nanos": 16000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "24", + "identity": "replay-fixture-worker", + "historySizeBytes": "5204" + } + }, + { + "eventId": "26", + "eventTime": { + "seconds": "1792404564", + "nanos": 54000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "24", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "27", + "eventTime": { + "seconds": "1792404564", + "nanos": 54000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "2", + "activityType": { + "name": "finishRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJpdGVtIjp7Im1pc3Npb25JZCI6InJvb21fYWdlbnRfbWlzc2lvbjphIiwidHVybiI6eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmEiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YSIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtYSIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtYSIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWEifX0sInN0YXR1cyI6IkNPTVBMRVRFRCIsInN0YXR1c1JlYXNvbiI6Im9rIn0=" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "25", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "28", + "eventTime": { + "seconds": "1792404564", + "nanos": 54000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "27", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "29", + "eventTime": { + "seconds": "1792404564", + "nanos": 63000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "27", + "startedEventId": "28", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "30", + "eventTime": { + "seconds": "1792404564", + "nanos": 63000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "31", + "eventTime": { + "seconds": "1792404564", + "nanos": 64000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "30", + "identity": "replay-fixture-worker", + "historySizeBytes": "6286" + } + }, + { + "eventId": "32", + "eventTime": { + "seconds": "1792404564", + "nanos": 80000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "30", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "33", + "eventTime": { + "seconds": "1792404564", + "nanos": 80000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "3", + "activityType": { + "name": "checkRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YiIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjpiIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmIiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWIiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWIiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1iIn19" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "31", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "34", + "eventTime": { + "seconds": "1792404564", + "nanos": 81000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "33", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "35", + "eventTime": { + "seconds": "1792404564", + "nanos": 89000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "IlJFQURZIg==" + } + ] + }, + "scheduledEventId": "33", + "startedEventId": "34", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "36", + "eventTime": { + "seconds": "1792404564", + "nanos": 89000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "37", + "eventTime": { + "seconds": "1792404564", + "nanos": 89000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "36", + "identity": "replay-fixture-worker", + "historySizeBytes": "7325" + } + }, + { + "eventId": "38", + "eventTime": { + "seconds": "1792404564", + "nanos": 113000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "36", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "39", + "eventTime": { + "seconds": "1792404564", + "nanos": 113000000 + }, + "eventType": "EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-b", + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmIiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YiIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtYiIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtYiIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWIifQ==" + } + ] + }, + "workflowExecutionTimeout": {}, + "workflowRunTimeout": {}, + "workflowTaskTimeout": { + "seconds": "10" + }, + "workflowTaskCompletedEventId": "37", + "header": {}, + "memo": {} + } + }, + { + "eventId": "40", + "eventTime": { + "seconds": "1792404564", + "nanos": 114000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "39", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-b", + "runId": "341a3238-9272-4090-9190-049841ea18d2" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + } + } + }, + { + "eventId": "41", + "eventTime": { + "seconds": "1792404564", + "nanos": 114000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "42", + "eventTime": { + "seconds": "1792404564", + "nanos": 115000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "41", + "identity": "replay-fixture-worker", + "historySizeBytes": "8451" + } + }, + { + "eventId": "43", + "eventTime": { + "seconds": "1792404567", + "nanos": 319000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "41", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "44", + "eventTime": { + "seconds": "1792404567", + "nanos": 394000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "enqueueRoomAgentMission", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246ZCIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjpkIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmQiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWQiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWQiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1kIn19" + } + ] + }, + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "45", + "eventTime": { + "seconds": "1792404567", + "nanos": 394000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "46", + "eventTime": { + "seconds": "1792404567", + "nanos": 395000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "45", + "identity": "replay-fixture-worker", + "historySizeBytes": "9294" + } + }, + { + "eventId": "47", + "eventTime": { + "seconds": "1792404567", + "nanos": 460000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "45", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "48", + "eventTime": { + "seconds": "1792404567", + "nanos": 460000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "enqueueRoomAgentMission", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YyIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjpjIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmMiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWMiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWMiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1jIn19" + } + ] + }, + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "49", + "eventTime": { + "seconds": "1792404567", + "nanos": 460000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "workflowExecutionSignaledEventAttributes": { + "signalName": "stopRoomAgentMission", + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YiIsInJlYXNvbiI6IlN0b3BwZWQgYnkgb3BlcmF0b3IuIn0=" + } + ] + }, + "identity": "replay-fixture-worker", + "header": {} + } + }, + { + "eventId": "50", + "eventTime": { + "seconds": "1792404567", + "nanos": 460000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 2 + } + }, + { + "eventId": "51", + "eventTime": { + "seconds": "1792404567", + "nanos": 460000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "50", + "identity": "replay-fixture-worker", + "historySizeBytes": "10306" + } + }, + { + "eventId": "52", + "eventTime": { + "seconds": "1792404567", + "nanos": 521000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "50", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "53", + "eventTime": { + "seconds": "1792404567", + "nanos": 521000000 + }, + "eventType": "EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED", + "requestCancelExternalWorkflowExecutionInitiatedEventAttributes": { + "workflowTaskCompletedEventId": "51", + "namespace": "default", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-b", + "runId": "341a3238-9272-4090-9190-049841ea18d2" + }, + "childWorkflowOnly": true + } + }, + { + "eventId": "54", + "eventTime": { + "seconds": "1792404567", + "nanos": 524000000 + }, + "eventType": "EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_CANCEL_REQUESTED", + "externalWorkflowExecutionCancelRequestedEventAttributes": { + "initiatedEventId": "53", + "namespace": "default", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-b", + "runId": "341a3238-9272-4090-9190-049841ea18d2" + } + } + }, + { + "eventId": "55", + "eventTime": { + "seconds": "1792404567", + "nanos": 524000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "56", + "eventTime": { + "seconds": "1792404567", + "nanos": 525000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "55", + "identity": "replay-fixture-worker", + "historySizeBytes": "10895" + } + }, + { + "eventId": "57", + "eventTime": { + "seconds": "1792404567", + "nanos": 532000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "55", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "58", + "eventTime": { + "seconds": "1792404567", + "nanos": 615000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED", + "childWorkflowExecutionCanceledEventAttributes": { + "details": {}, + "namespace": "default", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-b", + "runId": "341a3238-9272-4090-9190-049841ea18d2" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "initiatedEventId": "39", + "startedEventId": "40" + } + }, + { + "eventId": "59", + "eventTime": { + "seconds": "1792404567", + "nanos": 615000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "60", + "eventTime": { + "seconds": "1792404567", + "nanos": 616000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "59", + "identity": "replay-fixture-worker", + "historySizeBytes": "11348" + } + }, + { + "eventId": "61", + "eventTime": { + "seconds": "1792404567", + "nanos": 673000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "59", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "62", + "eventTime": { + "seconds": "1792404567", + "nanos": 673000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "4", + "activityType": { + "name": "discardRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJpdGVtIjp7Im1pc3Npb25JZCI6InJvb21fYWdlbnRfbWlzc2lvbjpiIiwidHVybiI6eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmIiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YiIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtYiIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtYiIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWIifX0sInJlYXNvbiI6IlN0b3BwZWQgYnkgb3BlcmF0b3IuIn0=" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "60", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "63", + "eventTime": { + "seconds": "1792404567", + "nanos": 673000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "62", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "64", + "eventTime": { + "seconds": "1792404567", + "nanos": 682000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "62", + "startedEventId": "63", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "65", + "eventTime": { + "seconds": "1792404567", + "nanos": 682000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "66", + "eventTime": { + "seconds": "1792404567", + "nanos": 682000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "65", + "identity": "replay-fixture-worker", + "historySizeBytes": "12428" + } + }, + { + "eventId": "67", + "eventTime": { + "seconds": "1792404567", + "nanos": 699000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "65", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "68", + "eventTime": { + "seconds": "1792404567", + "nanos": 699000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "5", + "activityType": { + "name": "finishRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJpdGVtIjp7Im1pc3Npb25JZCI6InJvb21fYWdlbnRfbWlzc2lvbjpiIiwidHVybiI6eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmIiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YiIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtYiIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtYiIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWIifX0sInN0YXR1cyI6IklOVEVSUlVQVEVEIiwic3RhdHVzUmVhc29uIjoiU3RvcHBlZCBieSBvcGVyYXRvci4ifQ==" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "66", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "69", + "eventTime": { + "seconds": "1792404567", + "nanos": 701000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "68", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "70", + "eventTime": { + "seconds": "1792404567", + "nanos": 713000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "68", + "startedEventId": "69", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "71", + "eventTime": { + "seconds": "1792404567", + "nanos": 713000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "72", + "eventTime": { + "seconds": "1792404567", + "nanos": 713000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "71", + "identity": "replay-fixture-worker", + "historySizeBytes": "13536" + } + }, + { + "eventId": "73", + "eventTime": { + "seconds": "1792404567", + "nanos": 733000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "71", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "74", + "eventTime": { + "seconds": "1792404567", + "nanos": 733000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "6", + "activityType": { + "name": "checkRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246ZCIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjpkIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmQiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWQiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWQiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1kIn19" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "72", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "75", + "eventTime": { + "seconds": "1792404567", + "nanos": 733000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "74", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "76", + "eventTime": { + "seconds": "1792404567", + "nanos": 739000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "IlJFQURZIg==" + } + ] + }, + "scheduledEventId": "74", + "startedEventId": "75", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "77", + "eventTime": { + "seconds": "1792404567", + "nanos": 739000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "78", + "eventTime": { + "seconds": "1792404567", + "nanos": 740000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "77", + "identity": "replay-fixture-worker", + "historySizeBytes": "14581" + } + }, + { + "eventId": "79", + "eventTime": { + "seconds": "1792404567", + "nanos": 758000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "77", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "80", + "eventTime": { + "seconds": "1792404567", + "nanos": 758000000 + }, + "eventType": "EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-d", + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmQiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246ZCIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtZCIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtZCIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWQifQ==" + } + ] + }, + "workflowExecutionTimeout": {}, + "workflowRunTimeout": {}, + "workflowTaskTimeout": { + "seconds": "10" + }, + "workflowTaskCompletedEventId": "78", + "header": {}, + "memo": {} + } + }, + { + "eventId": "81", + "eventTime": { + "seconds": "1792404567", + "nanos": 758000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "80", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-d", + "runId": "e1223dfa-8978-482d-8ef1-24dfbd06343a" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + } + } + }, + { + "eventId": "82", + "eventTime": { + "seconds": "1792404567", + "nanos": 758000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "83", + "eventTime": { + "seconds": "1792404567", + "nanos": 759000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "82", + "identity": "replay-fixture-worker", + "historySizeBytes": "15712" + } + }, + { + "eventId": "84", + "eventTime": { + "seconds": "1792404570", + "nanos": 121000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "82", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "85", + "eventTime": { + "seconds": "1792404570", + "nanos": 167000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_FAILED", + "childWorkflowExecutionFailedEventAttributes": { + "failure": { + "message": "Activity task failed", + "cause": { + "message": "synthetic turn failure", + "source": "TypeScriptSDK", + "stackTrace": "", + "applicationFailureInfo": { + "type": "TOOL_UNAVAILABLE", + "nonRetryable": true + } + }, + "activityFailureInfo": { + "scheduledEventId": "5", + "startedEventId": "6", + "identity": "replay-fixture-worker", + "activityType": { + "name": "runCodexAppServerTurn" + }, + "activityId": "1", + "retryState": "RETRY_STATE_NON_RETRYABLE_FAILURE" + } + }, + "namespace": "default", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-d", + "runId": "e1223dfa-8978-482d-8ef1-24dfbd06343a" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "initiatedEventId": "80", + "startedEventId": "81" + } + }, + { + "eventId": "86", + "eventTime": { + "seconds": "1792404570", + "nanos": 167000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "87", + "eventTime": { + "seconds": "1792404570", + "nanos": 167000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "86", + "identity": "replay-fixture-worker", + "historySizeBytes": "16586" + } + }, + { + "eventId": "88", + "eventTime": { + "seconds": "1792404570", + "nanos": 185000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "86", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "89", + "eventTime": { + "seconds": "1792404570", + "nanos": 185000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "7", + "activityType": { + "name": "discardRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJpdGVtIjp7Im1pc3Npb25JZCI6InJvb21fYWdlbnRfbWlzc2lvbjpkIiwidHVybiI6eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmQiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246ZCIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtZCIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtZCIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWQifX0sInJlYXNvbiI6IlJvb20gQWdlbnQgc3VwZXJ2aXNvciB0dXJuIGZhaWxlZC4iLCJlcnJvckNvZGUiOiJUT09MX1VOQVZBSUxBQkxFIn0=" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "87", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "90", + "eventTime": { + "seconds": "1792404570", + "nanos": 185000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "89", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "91", + "eventTime": { + "seconds": "1792404570", + "nanos": 191000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "89", + "startedEventId": "90", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "92", + "eventTime": { + "seconds": "1792404570", + "nanos": 191000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "93", + "eventTime": { + "seconds": "1792404570", + "nanos": 191000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "92", + "identity": "replay-fixture-worker", + "historySizeBytes": "17706" + } + }, + { + "eventId": "94", + "eventTime": { + "seconds": "1792404570", + "nanos": 206000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "92", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "95", + "eventTime": { + "seconds": "1792404570", + "nanos": 206000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "8", + "activityType": { + "name": "finishRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJpdGVtIjp7Im1pc3Npb25JZCI6InJvb21fYWdlbnRfbWlzc2lvbjpkIiwidHVybiI6eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmQiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246ZCIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtZCIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtZCIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWQifX0sInN0YXR1cyI6IkZBSUxFRCIsInN0YXR1c1JlYXNvbiI6IlJvb20gQWdlbnQgc3VwZXJ2aXNvciB0dXJuIGZhaWxlZC4iLCJlcnJvckNvZGUiOiJUT09MX1VOQVZBSUxBQkxFIn0=" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "93", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "96", + "eventTime": { + "seconds": "1792404570", + "nanos": 207000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "95", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "97", + "eventTime": { + "seconds": "1792404570", + "nanos": 213000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "95", + "startedEventId": "96", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "98", + "eventTime": { + "seconds": "1792404570", + "nanos": 213000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "99", + "eventTime": { + "seconds": "1792404570", + "nanos": 213000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "98", + "identity": "replay-fixture-worker", + "historySizeBytes": "18849" + } + }, + { + "eventId": "100", + "eventTime": { + "seconds": "1792404570", + "nanos": 226000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "98", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "101", + "eventTime": { + "seconds": "1792404570", + "nanos": 226000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "9", + "activityType": { + "name": "checkRoomAgentSupervisorTurn" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJtaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YyIsInR1cm4iOnsicm9vbUlkIjoicm9vbTpyZXBsYXktZml4dHVyZSIsInBhbmVJZCI6InBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQiLCJwcm9tcHQiOiJzeW50aGV0aWMgcHJvbXB0IiwiYXJ0aWZhY3RJZHMiOltdLCJhZ2VudFNlc3Npb25JZCI6ImFnZW50X3Nlc3Npb246cmVwbGF5IiwiYWdlbnRSdW5JZCI6ImFnZW50X3J1bjpjIiwicm9vbUFnZW50TWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmMiLCJyb29tQWdlbnRPcGVyYXRvclJlcXVlc3QiOiJzeW50aGV0aWMgcmVxdWVzdCIsImFnZW50VXNlck1lc3NhZ2VJZCI6ImFnZW50X21zZzp1LWMiLCJhZ2VudEFzc2lzdGFudE1lc3NhZ2VJZCI6ImFnZW50X21zZzphLWMiLCJwZXJtaXNzaW9uTW9kZSI6ImZ1bGxfYWNjZXNzIiwiY29sbGFib3JhdGlvbk1vZGUiOiJkZWZhdWx0IiwidHJhY2VJZCI6InJlcTpyZXBsYXktZml4dHVyZS1jIn19" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "99", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "102", + "eventTime": { + "seconds": "1792404570", + "nanos": 226000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "101", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "103", + "eventTime": { + "seconds": "1792404570", + "nanos": 231000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "IlJFQURZIg==" + } + ] + }, + "scheduledEventId": "101", + "startedEventId": "102", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "104", + "eventTime": { + "seconds": "1792404570", + "nanos": 231000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "105", + "eventTime": { + "seconds": "1792404570", + "nanos": 232000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "104", + "identity": "replay-fixture-worker", + "historySizeBytes": "19889" + } + }, + { + "eventId": "106", + "eventTime": { + "seconds": "1792404570", + "nanos": 245000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "104", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "107", + "eventTime": { + "seconds": "1792404570", + "nanos": 245000000 + }, + "eventType": "EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-c", + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInByb21wdCI6InN5bnRoZXRpYyBwcm9tcHQiLCJhcnRpZmFjdElkcyI6W10sImFnZW50U2Vzc2lvbklkIjoiYWdlbnRfc2Vzc2lvbjpyZXBsYXkiLCJhZ2VudFJ1bklkIjoiYWdlbnRfcnVuOmMiLCJyb29tQWdlbnRNaXNzaW9uSWQiOiJyb29tX2FnZW50X21pc3Npb246YyIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InN5bnRoZXRpYyByZXF1ZXN0IiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnUtYyIsImFnZW50QXNzaXN0YW50TWVzc2FnZUlkIjoiYWdlbnRfbXNnOmEtYyIsInBlcm1pc3Npb25Nb2RlIjoiZnVsbF9hY2Nlc3MiLCJjb2xsYWJvcmF0aW9uTW9kZSI6ImRlZmF1bHQiLCJ0cmFjZUlkIjoicmVxOnJlcGxheS1maXh0dXJlLWMifQ==" + } + ] + }, + "workflowExecutionTimeout": {}, + "workflowRunTimeout": {}, + "workflowTaskTimeout": { + "seconds": "10" + }, + "workflowTaskCompletedEventId": "105", + "header": {}, + "memo": {} + } + }, + { + "eventId": "108", + "eventTime": { + "seconds": "1792404570", + "nanos": 247000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "107", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-c", + "runId": "aa0ef35e-fd35-444b-b3de-9c045f6e52c2" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + } + } + }, + { + "eventId": "109", + "eventTime": { + "seconds": "1792404570", + "nanos": 247000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "110", + "eventTime": { + "seconds": "1792404570", + "nanos": 247000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "109", + "identity": "replay-fixture-worker", + "historySizeBytes": "21016" + } + }, + { + "eventId": "111", + "eventTime": { + "seconds": "1792404573", + "nanos": 221000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "109", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "112", + "eventTime": { + "seconds": "1792404573", + "nanos": 284000000 + }, + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED", + "childWorkflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ3b3JrZmxvd0lkIjoid29ya2Zsb3c6Y29kZXgtYXBwLXNlcnZlcjpyb29tOnJlcGxheS1maXh0dXJlOnBhbmU6cmVwbGF5LWZpeHR1cmUtYWdlbnQ6cmVxOnJlcGxheS1maXh0dXJlLWMiLCJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwicGFuZUlkIjoicGFuZTpyZXBsYXktZml4dHVyZS1hZ2VudCIsInRyYWNlSWQiOiJyZXE6cmVwbGF5LWZpeHR1cmUtYyIsInN0YXR1cyI6IkNPTVBMRVRFRCIsIm1lc3NhZ2UiOiJzeW50aGV0aWMiLCJyb29tQWdlbnRPdXRjb21lIjp7InN0YXR1cyI6IlZFUklGSUVEIiwic3RhdHVzUmVhc29uIjoib2siLCJwZW5kaW5nQWN0aW9uSWRzIjpbInJvb21fYWN0aW9uOnJlcGxheS1vbmUiXX19" + } + ] + }, + "namespace": "default", + "workflowExecution": { + "workflowId": "workflow:codex-app-server:room:replay-fixture:pane:replay-fixture-agent:req:replay-fixture-c", + "runId": "aa0ef35e-fd35-444b-b3de-9c045f6e52c2" + }, + "workflowType": { + "name": "codexAppServerTurnWorkflow" + }, + "initiatedEventId": "107", + "startedEventId": "108" + } + }, + { + "eventId": "113", + "eventTime": { + "seconds": "1792404573", + "nanos": 284000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "114", + "eventTime": { + "seconds": "1792404573", + "nanos": 284000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "113", + "identity": "replay-fixture-worker", + "historySizeBytes": "21859" + } + }, + { + "eventId": "115", + "eventTime": { + "seconds": "1792404573", + "nanos": 345000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "113", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "116", + "eventTime": { + "seconds": "1792404573", + "nanos": 345000000 + }, + "eventType": "EVENT_TYPE_TIMER_STARTED", + "timerStartedEventAttributes": { + "timerId": "1", + "startToFireTimeout": { + "seconds": "1" + }, + "workflowTaskCompletedEventId": "114" + } + }, + { + "eventId": "117", + "eventTime": { + "seconds": "1792404574", + "nanos": 345000000 + }, + "eventType": "EVENT_TYPE_TIMER_FIRED", + "timerFiredEventAttributes": { + "timerId": "1", + "startedEventId": "116" + } + }, + { + "eventId": "118", + "eventTime": { + "seconds": "1792404574", + "nanos": 345000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "119", + "eventTime": { + "seconds": "1792404574", + "nanos": 346000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "118", + "identity": "replay-fixture-worker", + "historySizeBytes": "22167" + } + }, + { + "eventId": "120", + "eventTime": { + "seconds": "1792404574", + "nanos": 371000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "118", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "121", + "eventTime": { + "seconds": "1792404574", + "nanos": 371000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "10", + "activityType": { + "name": "prepareRoomAgentBackgroundContinuation" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwibWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmMiLCJjb250aW51YXRpb25JZCI6IjI0OWY3MDVhLTBjN2QtNDZiNi05MjM1LTYwYjE1ZGNlYzdmZiJ9" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "119", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "122", + "eventTime": { + "seconds": "1792404574", + "nanos": 371000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "121", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "123", + "eventTime": { + "seconds": "1792404574", + "nanos": 381000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJzdGF0ZSI6IldBSVRJTkcifQ==" + } + ] + }, + "scheduledEventId": "121", + "startedEventId": "122", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "124", + "eventTime": { + "seconds": "1792404574", + "nanos": 381000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "125", + "eventTime": { + "seconds": "1792404574", + "nanos": 381000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "124", + "identity": "replay-fixture-worker", + "historySizeBytes": "22869" + } + }, + { + "eventId": "126", + "eventTime": { + "seconds": "1792404574", + "nanos": 414000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "124", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "127", + "eventTime": { + "seconds": "1792404574", + "nanos": 414000000 + }, + "eventType": "EVENT_TYPE_TIMER_STARTED", + "timerStartedEventAttributes": { + "timerId": "2", + "startToFireTimeout": { + "seconds": "1" + }, + "workflowTaskCompletedEventId": "125" + } + }, + { + "eventId": "128", + "eventTime": { + "seconds": "1792404575", + "nanos": 415000000 + }, + "eventType": "EVENT_TYPE_TIMER_FIRED", + "timerFiredEventAttributes": { + "timerId": "2", + "startedEventId": "127" + } + }, + { + "eventId": "129", + "eventTime": { + "seconds": "1792404575", + "nanos": 415000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "130", + "eventTime": { + "seconds": "1792404575", + "nanos": 415000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "129", + "identity": "replay-fixture-worker", + "historySizeBytes": "23179" + } + }, + { + "eventId": "131", + "eventTime": { + "seconds": "1792404575", + "nanos": 427000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "129", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "132", + "eventTime": { + "seconds": "1792404575", + "nanos": 427000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "11", + "activityType": { + "name": "prepareRoomAgentBackgroundContinuation" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwibWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmMiLCJjb250aW51YXRpb25JZCI6ImQ2Mjg0YjczLWI2YTItNGFmMi05NWQ5LTRkNGE4NGFhZTI2OSJ9" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "130", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "133", + "eventTime": { + "seconds": "1792404575", + "nanos": 427000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "132", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "134", + "eventTime": { + "seconds": "1792404575", + "nanos": 434000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJzdGF0ZSI6IldBSVRJTkcifQ==" + } + ] + }, + "scheduledEventId": "132", + "startedEventId": "133", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "135", + "eventTime": { + "seconds": "1792404575", + "nanos": 434000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "136", + "eventTime": { + "seconds": "1792404575", + "nanos": 434000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "135", + "identity": "replay-fixture-worker", + "historySizeBytes": "23893" + } + }, + { + "eventId": "137", + "eventTime": { + "seconds": "1792404575", + "nanos": 455000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "135", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "138", + "eventTime": { + "seconds": "1792404575", + "nanos": 455000000 + }, + "eventType": "EVENT_TYPE_TIMER_STARTED", + "timerStartedEventAttributes": { + "timerId": "3", + "startToFireTimeout": { + "seconds": "1" + }, + "workflowTaskCompletedEventId": "136" + } + }, + { + "eventId": "139", + "eventTime": { + "seconds": "1792404576", + "nanos": 455000000 + }, + "eventType": "EVENT_TYPE_TIMER_FIRED", + "timerFiredEventAttributes": { + "timerId": "3", + "startedEventId": "138" + } + }, + { + "eventId": "140", + "eventTime": { + "seconds": "1792404576", + "nanos": 455000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "141", + "eventTime": { + "seconds": "1792404576", + "nanos": 455000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "140", + "identity": "replay-fixture-worker", + "historySizeBytes": "24210" + } + }, + { + "eventId": "142", + "eventTime": { + "seconds": "1792404576", + "nanos": 468000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "140", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "143", + "eventTime": { + "seconds": "1792404576", + "nanos": 468000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "activityTaskScheduledEventAttributes": { + "activityId": "12", + "activityType": { + "name": "prepareRoomAgentBackgroundContinuation" + }, + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": {}, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJyb29tSWQiOiJyb29tOnJlcGxheS1maXh0dXJlIiwibWlzc2lvbklkIjoicm9vbV9hZ2VudF9taXNzaW9uOmMiLCJjb250aW51YXRpb25JZCI6IjUxNWFiZDgzLTY0YjUtNDY1Mi04OTM0LWUxZmJiNTU1NjAxOCJ9" + } + ] + }, + "scheduleToCloseTimeout": { + "seconds": "315360000" + }, + "scheduleToStartTimeout": { + "seconds": "315360000" + }, + "startToCloseTimeout": { + "seconds": "60" + }, + "heartbeatTimeout": {}, + "workflowTaskCompletedEventId": "141", + "retryPolicy": { + "initialInterval": { + "seconds": "1" + }, + "backoffCoefficient": 2, + "maximumInterval": { + "seconds": "100" + }, + "maximumAttempts": 5 + } + } + }, + { + "eventId": "144", + "eventTime": { + "seconds": "1792404576", + "nanos": 469000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "143", + "identity": "replay-fixture-worker", + "attempt": 1 + } + }, + { + "eventId": "145", + "eventTime": { + "seconds": "1792404576", + "nanos": 474000000 + }, + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJzdGF0ZSI6IkRPTkUifQ==" + } + ] + }, + "scheduledEventId": "143", + "startedEventId": "144", + "identity": "replay-fixture-worker" + } + }, + { + "eventId": "146", + "eventTime": { + "seconds": "1792404576", + "nanos": 474000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "replay-fixture-1792404555766", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": { + "seconds": "10" + }, + "attempt": 1 + } + }, + { + "eventId": "147", + "eventTime": { + "seconds": "1792404576", + "nanos": 474000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "146", + "identity": "replay-fixture-worker", + "historySizeBytes": "24921" + } + }, + { + "eventId": "148", + "eventTime": { + "seconds": "1792404576", + "nanos": 488000000 + }, + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "146", + "identity": "replay-fixture-worker", + "binaryChecksum": "@temporalio/worker@1.24.0+61b1ed58d29677e2000d84b95127c5ea89cfdf9a596e1df00e9e4af3f83e3a70", + "sdkMetadata": {}, + "meteringMetadata": {} + } + } + ] +} diff --git a/apps/worker/tests/fixtures/otlp-task-failure-workflows.ts b/apps/worker/tests/fixtures/otlp-task-failure-workflows.ts new file mode 100644 index 000000000..c066a87c3 --- /dev/null +++ b/apps/worker/tests/fixtures/otlp-task-failure-workflows.ts @@ -0,0 +1,45 @@ +import { ApplicationFailure, condition, defineSignal, defineUpdate, proxySinks, setHandler, type Sinks } from "@temporalio/workflow"; + +// Counts every execution, including Temporal's retries of a failed workflow task. +export interface TaskFailureProbeSinks extends Sinks { + taskFailureProbe: { executed(entry: string): void }; +} +const { taskFailureProbe } = proxySinks(); + +// A non-Temporal error fails the workflow task, which Temporal retries; the +// workflow execution itself keeps running. +export async function traceTaskFailureWorkflow(): Promise { + taskFailureProbe.executed("workflow"); + throw new Error("synthetic non-Temporal workflow error"); +} + +// A Temporal failure is a terminal workflow outcome. +export async function traceTerminalFailureWorkflow(): Promise { + throw ApplicationFailure.nonRetryable("synthetic terminal failure", "TOOL_UNAVAILABLE"); +} + +export const plainErrorSignal = defineSignal("plainErrorSignal"); +export const applicationFailureSignal = defineSignal("applicationFailureSignal"); +export const plainErrorUpdate = defineUpdate("plainErrorUpdate"); +export const applicationFailureUpdate = defineUpdate("applicationFailureUpdate"); + +// Signal and update handlers follow the same rule: a non-Temporal error fails +// the task; a Temporal failure fails the workflow (signal) or the update. +export async function traceHandlerFailureWorkflow(): Promise { + setHandler(plainErrorSignal, () => { + taskFailureProbe.executed("signal"); + throw new Error("synthetic non-Temporal signal error"); + }); + setHandler(applicationFailureSignal, () => { + throw ApplicationFailure.nonRetryable("synthetic signal failure", "TOOL_UNAVAILABLE"); + }); + setHandler(plainErrorUpdate, () => { + taskFailureProbe.executed("update"); + throw new Error("synthetic non-Temporal update error"); + }); + setHandler(applicationFailureUpdate, () => { + throw ApplicationFailure.nonRetryable("synthetic update failure", "TOOL_UNAVAILABLE"); + }); + await condition(() => false); + return "unreachable"; +} diff --git a/apps/worker/tests/fixtures/otlp-workflows.ts b/apps/worker/tests/fixtures/otlp-workflows.ts new file mode 100644 index 000000000..c4c521929 --- /dev/null +++ b/apps/worker/tests/fixtures/otlp-workflows.ts @@ -0,0 +1,78 @@ +import { condition, continueAsNew, defineQuery, defineSignal, executeChild, proxyActivities, proxyLocalActivities, setHandler, uuid4, workflowInfo } from "@temporalio/workflow"; +import { withWorkflowTraceContext } from "../../src/workflow-tracing.js"; + +export interface TraceTestInput { + traceId: string; + traceparent?: string; + roomId?: string; + missionId?: string; + agentRunId?: string; + continuedTraceparent?: string; + prompt?: string; + toolOutput?: string; + path?: string; + url?: string; +} + +export interface TraceTestActivities { + noArgumentActivity(): Promise; + retryOnceActivity(): Promise; + privateActivity(input: TraceTestInput): Promise<{ status: string; message: string }>; +} + +const activities = proxyActivities({ + startToCloseTimeout: "10 seconds", + retry: { initialInterval: "10 milliseconds", maximumInterval: "10 milliseconds", maximumAttempts: 2 } +}); +const localActivities = proxyLocalActivities>({ + startToCloseTimeout: "10 seconds" +}); + +// These fixtures execute through the real SDK sandbox and activity executor. +// They deliberately include activities with no arguments: those can only +// inherit tracing through the Temporal headers, not application payloads. +export async function traceTreeWorkflow(input: TraceTestInput): Promise { + const first = await activities.noArgumentActivity(); + const retried = await activities.retryOnceActivity(); + const local = await localActivities.noArgumentActivity(); + await activities.privateActivity(input); + const child = await executeChild(traceChildWorkflow, { + workflowId: `${workflowInfo().workflowId}:child`, + args: [] + }); + return [first, retried, local, child]; +} + +export async function traceChildWorkflow(): Promise { + return activities.noArgumentActivity(); +} + +export async function traceContinueWorkflow(run = 0): Promise { + await activities.noArgumentActivity(); + if (run === 0) return continueAsNew(1); + return "continued"; +} + +export async function traceSingleActivityWorkflow(): Promise { + return activities.noArgumentActivity(); +} + +export async function traceRandomActivityIdWorkflow(): Promise { + // Temporal validates this command identity on replay. Tracing must not + // consume the deterministic random stream that application code uses. + return activities.noArgumentActivity.executeWithOptions({ activityId: uuid4() }, []); +} + +export async function traceMissionWorkflow(missionCount: number): Promise { + const queue: TraceTestInput[] = []; + let completed = 0; + setHandler(defineSignal<[TraceTestInput]>("traceMission"), (input) => { queue.push(input); }); + setHandler(defineQuery("completedTraceMissions"), () => completed); + while (completed < missionCount) { + await condition(() => queue.length > 0); + const input = queue.shift()!; + await withWorkflowTraceContext(input, () => activities.noArgumentActivity()); + completed += 1; + } + return completed; +} diff --git a/apps/worker/tests/otlp-supervisor-integration.test.ts b/apps/worker/tests/otlp-supervisor-integration.test.ts new file mode 100644 index 000000000..a33b64fe0 --- /dev/null +++ b/apps/worker/tests/otlp-supervisor-integration.test.ts @@ -0,0 +1,156 @@ +// The production Room Agent supervisor in a real Temporal test server, driven +// through the production API coordinator (signalWithStart) and API trace hooks, +// with the production worker tracing composition. Only activities are stubbed. +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { fileURLToPath } from "node:url"; +import { Context } from "@temporalio/activity"; +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { bundleWorkflowCode, DefaultLogger, Runtime, Worker } from "@temporalio/worker"; +import Fastify from "fastify"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { buildRoomAgentSupervisorWorkflowId, type RoomAgentSupervisorQueueItem } from "@space/contracts"; +import { createOtlpSpanExporter } from "@space/runtime"; +import { createHttpObservability } from "../../api/src/observability.js"; +import { TemporalRoomAgentWorkflowCoordinator } from "../../api/src/room-agent.js"; +import { requestTraceContexts, traceContextForRawRequest } from "../../api/src/trace-context.js"; +import { deterministicWorkflowSpanId } from "../src/temporal-trace-context.js"; +import { createWorkerTracing } from "../src/worker-tracing.js"; + +const workflowInterceptorModule = fileURLToPath(new URL("../src/workflow-tracing.ts", import.meta.url)); +const workflowsPath = fileURLToPath(new URL("../src/workflows.ts", import.meta.url)); +const roomId = "room:otlp-supervisor"; + +interface OtlpSpan { + traceId: string; spanId: string; parentSpanId?: string; name: string; + attributes: Array<{ key: string; value: { stringValue?: string; intValue?: string } }>; +} +const attrs = (span: OtlpSpan) => Object.fromEntries(span.attributes.map(({ key, value }) => [key, value.stringValue ?? value.intValue ?? ""])); + +async function startCollector() { + const bodies: Array<{ resourceSpans: Array<{ scopeSpans: Array<{ spans: OtlpSpan[] }> }> }> = []; + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += String(chunk); + bodies.push(JSON.parse(body)); + response.writeHead(200, { "content-type": "application/json" }).end("{}"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return { + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + spans: () => bodies.flatMap((b) => b.resourceSpans.flatMap((r) => r.scopeSpans.flatMap((s) => s.spans))), + close: () => new Promise((resolve) => { server.closeAllConnections(); server.close(() => resolve()); }) + }; +} + +function queueItem(missionId: string, traceId: string): RoomAgentSupervisorQueueItem { + const key = missionId.split(":")[1]; + return { + missionId, + turn: { + roomId, paneId: "pane:otlp-supervisor-agent", prompt: "synthetic prompt", artifactIds: [], + agentSessionId: "agent_session:otlp", agentRunId: `agent_run:${key}`, + roomAgentMissionId: missionId, roomAgentOperatorRequest: "synthetic request", + agentUserMessageId: `agent_msg:u-${key}`, agentAssistantMessageId: `agent_msg:a-${key}`, + permissionMode: "full_access", collaborationMode: "default", traceId + } + }; +} + +describe("OTLP through the production Room Agent supervisor", () => { + let env: TestWorkflowEnvironment; + beforeAll(async () => { + Runtime.install({ logger: new DefaultLogger("ERROR") }); + env = await TestWorkflowEnvironment.createTimeSkipping(); + }, 120_000); + afterAll(async () => { await env?.teardown(); }); + + it("parents each mission to its own API span and keeps run-level polling out of the first request's trace", async () => { + const collector = await startCollector(); + const apiSpans = createOtlpSpanExporter({ endpoint: collector.endpoint, serviceName: "space-api", flushIntervalMs: 60_000 }); + const spans = createOtlpSpanExporter({ endpoint: collector.endpoint, serviceName: "space-worker", flushIntervalMs: 60_000 }); + const taskQueue = `otlp-supervisor-${randomUUID()}`; + const coordinator = new TemporalRoomAgentWorkflowCoordinator({ + address: "unused", namespace: env.namespace, taskQueue, + clientFactory: async () => ({ client: env.client, close: async () => undefined }) + }); + const observability = createHttpObservability({ serviceName: "space-api", spanExporter: apiSpans }); + const app = Fastify({ genReqId: () => `req:${randomUUID()}` }); + // Same calls as apps/api/src/app.ts onRequest/onResponse. + app.addHook("onRequest", async (request) => { + observability.onRequest(request); + requestTraceContexts.remember(request.id, traceContextForRawRequest(request.raw)); + }); + app.addHook("onResponse", async (request, reply) => observability.onResponse(request, reply)); + app.post<{ Params: { missionId: string } }>("/api/rooms/otlp/agent/:missionId", async (request) => { + await coordinator.enqueue(queueItem(request.params.missionId, request.id)); + return { ok: true }; + }); + let polls = 0; + const workflowBundle = await bundleWorkflowCode({ workflowsPath, workflowInterceptorModules: [workflowInterceptorModule], logger: new DefaultLogger("ERROR") }); + const tracing = createWorkerTracing(spans, { workflowInterceptorModule }); + const worker = await Worker.create({ + connection: env.nativeConnection, namespace: env.namespace, taskQueue, workflowBundle, + ...tracing, interceptors: { ...tracing.interceptors, workflowModules: [] }, + activities: { + checkRoomAgentSupervisorTurn: async () => "READY" as const, + discardRoomAgentSupervisorTurn: async () => undefined, + finishRoomAgentSupervisorTurn: async () => undefined, + prepareRoomAgentBackgroundContinuation: async () => (++polls >= 3 ? { state: "DONE" } : { state: "WAITING" }), + runCodexAppServerTurn: async (input: RoomAgentSupervisorQueueItem["turn"]) => ({ + workflowId: Context.current().info.workflowExecution.workflowId, roomId, paneId: input.paneId, traceId: input.traceId, + status: "COMPLETED", message: "synthetic", + // Mission C leaves background pane work, so the supervisor polls for it. + roomAgentOutcome: input.roomAgentMissionId === "room_agent_mission:c" + ? { status: "VERIFIED", statusReason: "ok", pendingActionIds: ["room_action:otlp-one"] } + : { status: "VERIFIED", statusReason: "ok" } + }) + } + }); + const traceA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1"; + const traceC = "ccccccccccccccccccccccccccccccc3"; + const caller = "1111111111111111"; + const flushed = async () => { await Promise.all([apiSpans.flush(), spans.flush()]); return collector.spans(); }; + const missionSpan = (all: OtlpSpan[], missionId: string) => + all.find((s) => s.name === "workflow roomAgentSupervisorWorkflow mission" && attrs(s)["space.mission_id"] === missionId); + try { + await worker.runUntil(async () => { + // Requests A and B share ONE W3C trace but have different API spans. + await app.inject({ method: "POST", url: "/api/rooms/otlp/agent/room_agent_mission:a", headers: { traceparent: `00-${traceA}-${caller}-01` } }); + await expect.poll(async () => missionSpan(await flushed(), "room_agent_mission:a"), { timeout: 30_000 }).toBeDefined(); + await app.inject({ method: "POST", url: "/api/rooms/otlp/agent/room_agent_mission:b", headers: { traceparent: `00-${traceA}-${caller}-01` } }); + await expect.poll(async () => missionSpan(await flushed(), "room_agent_mission:b"), { timeout: 30_000 }).toBeDefined(); + await app.inject({ method: "POST", url: "/api/rooms/otlp/agent/room_agent_mission:c", headers: { traceparent: `00-${traceC}-${caller}-01` } }); + await expect.poll(async () => (await flushed()).filter((s) => attrs(s)["space.activity_type"] === "prepareRoomAgentBackgroundContinuation").length, + { timeout: 30_000 }).toBeGreaterThanOrEqual(3); + }); + const all = await flushed(); + const api = all.filter((s) => s.name === "POST /api/rooms/otlp/agent/:missionId"); + const apiFor = (missionId: string) => api.find((s) => attrs(s)["space.request_id"] === attrs(missionSpan(all, missionId)!)["space.trace_id"])!; + const [missionA, missionB, missionC] = ["a", "b", "c"].map((m) => missionSpan(all, `room_agent_mission:${m}`)!); + expect(apiFor("room_agent_mission:a").traceId).toBe(traceA); + expect(apiFor("room_agent_mission:b").traceId).toBe(traceA); + expect(apiFor("room_agent_mission:a").spanId).not.toBe(apiFor("room_agent_mission:b").spanId); + expect(missionA).toMatchObject({ traceId: traceA, parentSpanId: apiFor("room_agent_mission:a").spanId }); + expect(missionB).toMatchObject({ traceId: traceA, parentSpanId: apiFor("room_agent_mission:b").spanId }); + expect(missionC).toMatchObject({ traceId: traceC, parentSpanId: apiFor("room_agent_mission:c").spanId }); + const childB = all.find((s) => s.name === "workflow codexAppServerTurnWorkflow" && s.parentSpanId === missionB!.spanId); + expect(all.find((s) => attrs(s)["space.activity_type"] === "runCodexAppServerTurn" && s.parentSpanId === childB?.spanId)).toMatchObject({ traceId: traceA }); + + // Polls for mission C belong to the supervisor's own trace, under its run span. + const pollSpans = all.filter((s) => attrs(s)["space.activity_type"] === "prepareRoomAgentBackgroundContinuation"); + for (const poll of pollSpans) { + expect(attrs(poll)["space.mission_id"]).toBe("room_agent_mission:c"); + expect([traceA, traceC]).not.toContain(poll.traceId); + expect(poll.parentSpanId).toBe(deterministicWorkflowSpanId(attrs(poll)["space.temporal_run_id"]!)); + } + } finally { + await app.close(); + observability.close(); + await Promise.all([apiSpans.close(), spans.close()]); + await collector.close(); + await env.client.workflow.getHandle(buildRoomAgentSupervisorWorkflowId(roomId)).terminate("test cleanup").catch(() => undefined); + } + }, 120_000); +}); diff --git a/apps/worker/tests/otlp-supervisor-replay.test.ts b/apps/worker/tests/otlp-supervisor-replay.test.ts new file mode 100644 index 000000000..6703ac450 --- /dev/null +++ b/apps/worker/tests/otlp-supervisor-replay.test.ts @@ -0,0 +1,42 @@ +// A Room Agent supervisor history recorded by the 1.1.6 workflow code (no +// tracing interceptor) through the production API coordinator: one completed +// mission, one stopped while running, one failed child and background polls. +// It must replay on the current workflow code, traced and untraced, and replay +// must export nothing. The JSON is the protobufjs object form of the history. +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { temporal } from "@temporalio/proto"; +import { bundleWorkflowCode, DefaultLogger, Runtime, Worker } from "@temporalio/worker"; +import { describe, expect, it } from "vitest"; +import type { ServerSpan, SpanExporter } from "@space/runtime"; +import { createWorkerTracing } from "../src/worker-tracing.js"; + +const workflowInterceptorModule = fileURLToPath(new URL("../src/workflow-tracing.ts", import.meta.url)); +const workflowsPath = fileURLToPath(new URL("../src/workflows.ts", import.meta.url)); +const historyPath = new URL("./fixtures/histories/room-agent-supervisor-1.1.6.json", import.meta.url); +const workflowId = "room-supervisor:room:replay-fixture"; + +describe("Room Agent supervisor replay", () => { + it("replays a 1.1.6 supervisor history on the current code, traced and untraced, without exporting", async () => { + Runtime.install({ logger: new DefaultLogger("ERROR") }); + const history = temporal.api.history.v1.History.fromObject(JSON.parse(await readFile(historyPath, "utf8"))); + const events = history.events ?? []; + expect(events.filter((event) => event.childWorkflowExecutionFailedEventAttributes)).toHaveLength(1); + expect(events.filter((event) => event.childWorkflowExecutionCanceledEventAttributes)).toHaveLength(1); + const recorded: ServerSpan[] = []; + const spans: SpanExporter = { + record: (span) => { recorded.push(span); }, + flush: async () => undefined, + close: async () => undefined, + stats: () => ({ exported: 0, dropped: 0, failed: 0 }) + }; + const tracing = createWorkerTracing(spans, { workflowInterceptorModule }); + const [traced, untraced] = await Promise.all([ + bundleWorkflowCode({ workflowsPath, workflowInterceptorModules: [workflowInterceptorModule], logger: new DefaultLogger("ERROR") }), + bundleWorkflowCode({ workflowsPath, logger: new DefaultLogger("ERROR") }) + ]); + await Worker.runReplayHistory({ workflowBundle: traced, ...tracing, interceptors: { ...tracing.interceptors, workflowModules: [] } }, history, workflowId); + await Worker.runReplayHistory({ workflowBundle: untraced }, history, workflowId); + expect(recorded).toEqual([]); + }, 180_000); +}); diff --git a/apps/worker/tests/otlp-task-failure.test.ts b/apps/worker/tests/otlp-task-failure.test.ts new file mode 100644 index 000000000..10e5607df --- /dev/null +++ b/apps/worker/tests/otlp-task-failure.test.ts @@ -0,0 +1,183 @@ +// A non-Temporal error from the workflow function or a signal/update handler +// fails the workflow task: Temporal retries it while the execution keeps +// running, so it must not be exported as a completed FAILED span. A Temporal +// failure is a real outcome and is exported. +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { fileURLToPath } from "node:url"; +import { WorkflowFailedError, WorkflowUpdateFailedError, type WorkflowHandle } from "@temporalio/client"; +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { bundleWorkflowCode, DefaultLogger, Runtime, Worker, type WorkflowBundleWithSourceMap } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createOtlpSpanExporter, type SpanExporter } from "@space/runtime"; +import { deterministicWorkflowSpanId } from "../src/temporal-trace-context.js"; +import { createWorkerTracing } from "../src/worker-tracing.js"; + +const workflowInterceptorModule = fileURLToPath(new URL("../src/workflow-tracing.ts", import.meta.url)); +const workflowsPath = fileURLToPath(new URL("./fixtures/otlp-task-failure-workflows.ts", import.meta.url)); +const traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + +interface OtlpSpan { + traceId: string; spanId: string; parentSpanId?: string; name: string; + attributes: Array<{ key: string; value: { stringValue?: string; intValue?: string } }>; +} +const attrs = (span: OtlpSpan) => Object.fromEntries(span.attributes.map(({ key, value }) => [key, value.stringValue ?? value.intValue ?? ""])); + +interface TracedWorker { + taskQueue: string; + executions: Record; + spans: SpanExporter; + exported: () => OtlpSpan[]; + requests: () => number; +} + +describe("OTLP and workflow-task failures", () => { + let env: TestWorkflowEnvironment; + let workflowBundle: WorkflowBundleWithSourceMap; + beforeAll(async () => { + Runtime.install({ logger: new DefaultLogger("ERROR") }); + env = await TestWorkflowEnvironment.createTimeSkipping(); + workflowBundle = await bundleWorkflowCode({ workflowsPath, workflowInterceptorModules: [workflowInterceptorModule], logger: new DefaultLogger("ERROR") }); + }, 120_000); + afterAll(async () => { await env?.teardown(); }); + + async function withTracedWorker(run: (worker: TracedWorker) => Promise): Promise { + const bodies: Array<{ resourceSpans: Array<{ scopeSpans: Array<{ spans: OtlpSpan[] }> }> }> = []; + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += String(chunk); + bodies.push(JSON.parse(body)); + response.writeHead(200, { "content-type": "application/json" }).end("{}"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const spans = createOtlpSpanExporter({ + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, serviceName: "space-worker", flushIntervalMs: 60_000 + }); + const tracing = createWorkerTracing(spans, { workflowInterceptorModule }); + const executions: Record = {}; + const taskQueue = `otlp-task-failure-${randomUUID()}`; + const worker = await Worker.create({ + connection: env.nativeConnection, namespace: env.namespace, taskQueue, workflowBundle, + ...tracing, interceptors: { ...tracing.interceptors, workflowModules: [] }, activities: {}, + sinks: { + ...tracing.sinks, + taskFailureProbe: { executed: { fn: (_info, entry: string) => { executions[entry] = (executions[entry] ?? 0) + 1; }, callDuringReplay: true } } + } + }); + try { + return await worker.runUntil(run({ + taskQueue, executions, spans, + exported: () => bodies.flatMap((b) => b.resourceSpans.flatMap((r) => r.scopeSpans.flatMap((s) => s.spans))), + requests: () => bodies.length + })); + } finally { + await spans.close(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + } + } + + // History shows the failed task; a second execution shows Temporal retried it. + async function retriedTaskFailure(handle: WorkflowHandle, executions: Record, entry: string): Promise { + await expect.poll(async () => (await handle.fetchHistory()).events!.some((event) => event.workflowTaskFailedEventAttributes), + { timeout: 30_000 }).toBe(true); + await expect.poll(() => executions[entry] ?? 0, { timeout: 30_000 }).toBeGreaterThanOrEqual(2); + return (await handle.describe()).status.name; + } + + async function firstTaskCompleted(handle: WorkflowHandle): Promise { + await expect.poll(async () => (await handle.fetchHistory()).events!.some((event) => event.workflowTaskCompletedEventAttributes), + { timeout: 30_000 }).toBe(true); + } + + // Captured before the execution is terminated for cleanup. + async function capture(worker: TracedWorker, status: string) { + await worker.spans.flush(); + return { status, stats: worker.spans.stats(), requests: worker.requests(), spans: worker.exported() }; + } + + it("exports no span while a workflow function's failed task is retried", async () => { + const result = await withTracedWorker(async (worker) => { + const handle = await env.client.workflow.start("traceTaskFailureWorkflow", { taskQueue: worker.taskQueue, workflowId: worker.taskQueue, memo: { traceparent } }); + const observed = await capture(worker, await retriedTaskFailure(handle, worker.executions, "workflow")); + await handle.terminate("test cleanup"); + return observed; + }); + expect(result).toEqual({ status: "RUNNING", stats: { exported: 0, dropped: 0, failed: 0 }, requests: 0, spans: [] }); + }, 120_000); + + it("exports no span while a signal handler's failed task is retried", async () => { + const result = await withTracedWorker(async (worker) => { + const handle = await env.client.workflow.start("traceHandlerFailureWorkflow", { taskQueue: worker.taskQueue, workflowId: worker.taskQueue, memo: { traceparent } }); + await firstTaskCompleted(handle); + await handle.signal("plainErrorSignal"); + const observed = await capture(worker, await retriedTaskFailure(handle, worker.executions, "signal")); + await handle.terminate("test cleanup"); + return observed; + }); + expect(result).toEqual({ status: "RUNNING", stats: { exported: 0, dropped: 0, failed: 0 }, requests: 0, spans: [] }); + }, 120_000); + + it("exports no span while an update handler's failed task is retried", async () => { + const result = await withTracedWorker(async (worker) => { + const handle = await env.client.workflow.start("traceHandlerFailureWorkflow", { taskQueue: worker.taskQueue, workflowId: worker.taskQueue, memo: { traceparent } }); + await firstTaskCompleted(handle); + const update = handle.executeUpdate("plainErrorUpdate").then(() => "completed", () => "rejected"); + const observed = await capture(worker, await retriedTaskFailure(handle, worker.executions, "update")); + await handle.terminate("test cleanup"); + await Promise.race([update, new Promise((resolve) => setTimeout(resolve, 10_000))]); + return observed; + }); + expect(result).toEqual({ status: "RUNNING", stats: { exported: 0, dropped: 0, failed: 0 }, requests: 0, spans: [] }); + }, 120_000); + + it("rejects an update that throws a Temporal failure and exports a FAILED update span", async () => { + const result = await withTracedWorker(async (worker) => { + const handle = await env.client.workflow.start("traceHandlerFailureWorkflow", { taskQueue: worker.taskQueue, workflowId: worker.taskQueue, memo: { traceparent } }); + const rejection = await handle.executeUpdate("applicationFailureUpdate").then(() => null, (error: unknown) => error); + const observed = await capture(worker, (await handle.describe()).status.name); + await handle.terminate("test cleanup"); + return { ...observed, rejection, runId: handle.firstExecutionRunId }; + }); + expect(result.rejection).toBeInstanceOf(WorkflowUpdateFailedError); + expect(result.status).toBe("RUNNING"); + expect(result.spans).toHaveLength(1); + expect(result.spans[0]).toMatchObject({ traceId: "4bf92f3577b34da6a3ce929d0e0e4736", parentSpanId: deterministicWorkflowSpanId(result.runId!) }); + expect(result.spans[0]!.name).toBe("workflow traceHandlerFailureWorkflow update"); + expect(attrs(result.spans[0]!)).toMatchObject({ + "space.outcome": "FAILED", "space.reason_code": "TOOL_UNAVAILABLE", "space.update_name": "applicationFailureUpdate" + }); + }, 120_000); + + it("exports a FAILED run span when the workflow function throws a Temporal failure", async () => { + const result = await withTracedWorker(async (worker) => { + const handle = await env.client.workflow.start("traceTerminalFailureWorkflow", { taskQueue: worker.taskQueue, workflowId: worker.taskQueue, memo: { traceparent } }); + const failure = await handle.result().then(() => null, (error: unknown) => error); + return { ...(await capture(worker, (await handle.describe()).status.name)), failure }; + }); + expect(result.failure).toBeInstanceOf(WorkflowFailedError); + expect(result.status).toBe("FAILED"); + expect(result.spans).toHaveLength(1); + expect(result.spans[0]).toMatchObject({ traceId: "4bf92f3577b34da6a3ce929d0e0e4736", parentSpanId: "00f067aa0ba902b7" }); + expect(attrs(result.spans[0]!)).toMatchObject({ "space.outcome": "FAILED", "space.reason_code": "TOOL_UNAVAILABLE" }); + }, 120_000); + + it("exports the FAILED signal span when a signal handler's Temporal failure ends the workflow", async () => { + const result = await withTracedWorker(async (worker) => { + const handle = await env.client.workflow.start("traceHandlerFailureWorkflow", { taskQueue: worker.taskQueue, workflowId: worker.taskQueue, memo: { traceparent } }); + await firstTaskCompleted(handle); + await handle.signal("applicationFailureSignal"); + const failure = await handle.result().then(() => null, (error: unknown) => error); + return { ...(await capture(worker, (await handle.describe()).status.name)), failure }; + }); + expect(result.failure).toBeInstanceOf(WorkflowFailedError); + expect(result.status).toBe("FAILED"); + // The workflow function never returns, so the run span is not exported + // (docs/quality/observability.md, Traces). + expect(result.spans.map((span) => span.name)).toEqual(["workflow traceHandlerFailureWorkflow signal"]); + expect(attrs(result.spans[0]!)).toMatchObject({ + "space.outcome": "FAILED", "space.reason_code": "TOOL_UNAVAILABLE", "space.signal_name": "applicationFailureSignal" + }); + }, 120_000); +}); diff --git a/apps/worker/tests/otlp-worker-integration.test.ts b/apps/worker/tests/otlp-worker-integration.test.ts new file mode 100644 index 000000000..ccd450388 --- /dev/null +++ b/apps/worker/tests/otlp-worker-integration.test.ts @@ -0,0 +1,537 @@ +import { randomUUID } from "node:crypto"; +import { createServer, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { fileURLToPath } from "node:url"; +import { ApplicationFailure, Context } from "@temporalio/activity"; +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { bundleWorkflowCode, DefaultLogger, Runtime, Worker, type WorkflowBundleWithSourceMap } from "@temporalio/worker"; +import Fastify from "fastify"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createOtlpSpanExporter, InMemorySpaceStore, otlpSpanExporterFromEnv, type SpanExporter } from "@space/runtime"; +import { createHttpObservability } from "../../api/src/observability.js"; +import { requestTraceContexts, traceContextForRawRequest, withTurnTraceparent, workflowCorrelationMemo } from "../../api/src/trace-context.js"; +import { noteActivityFailureReason } from "../src/activity-observability.js"; +import { createWorkerTracing } from "../src/worker-tracing.js"; +import { MEMORY_MAINTENANCE_TASK_QUEUE } from "../src/memory-maintenance-config.js"; +import { MEMORY_MUTATION_TASK_QUEUE } from "../src/memory-mutation-config.js"; +import type { TraceTestActivities, TraceTestInput } from "./fixtures/otlp-workflows.js"; + +const traceId = "4bf92f3577b34da6a3ce929d0e0e4736"; +const callerSpanId = "00f067aa0ba902b7"; +const traceparent = `00-${traceId}-${callerSpanId}-01`; +const privateText = "PRIVATE-PROMPT-and-tool-output-must-stay-local"; +const privatePath = "/private/synthetic-test-workspace/source.ts"; +const privateUrl = "https://private.invalid/provider?secret=synthetic-test-value"; +const workflowInterceptorModule = fileURLToPath(new URL("../src/workflow-tracing.ts", import.meta.url)); +const workflowsPath = fileURLToPath(new URL("./fixtures/otlp-workflows.ts", import.meta.url)); + +interface OtlpAttribute { + key: string; + value: { stringValue?: string; intValue?: string }; +} +interface OtlpSpan { + traceId: string; + spanId: string; + parentSpanId?: string; + name: string; + kind: number; + attributes: OtlpAttribute[]; + status: { code: number }; + startTimeUnixNano: string; + endTimeUnixNano: string; + links?: Array<{ traceId: string; spanId: string }>; +} +interface OtlpRequest { + resourceSpans: Array<{ + resource: { attributes: OtlpAttribute[] }; + scopeSpans: Array<{ spans: OtlpSpan[] }>; + }>; +} + +function attributes(span: OtlpSpan): Record { + return Object.fromEntries(span.attributes.map(({ key, value }) => [key, value.stringValue ?? value.intValue ?? ""])); +} + +async function startCollector() { + const requests: Array<{ path: string; contentType: string; body: OtlpRequest }> = []; + const heldResponses = new Set(); + let holdResponses = false; + let statusCode = 200; + let notifyRequest: (() => void) | undefined; + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += String(chunk); + requests.push({ path: request.url ?? "", contentType: request.headers["content-type"] ?? "", body: JSON.parse(body) as OtlpRequest }); + notifyRequest?.(); + if (holdResponses) { + heldResponses.add(response); + response.on("close", () => heldResponses.delete(response)); + } else { + response.writeHead(statusCode, { "content-type": "application/json" }).end("{}"); + } + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + return { + endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + requests, + spans: () => requests.flatMap(({ body }) => body.resourceSpans.flatMap((resource) => resource.scopeSpans.flatMap((scope) => scope.spans))), + fail: () => { statusCode = 503; }, + hold: () => { holdResponses = true; }, + get pendingResponses() { return heldResponses.size; }, + nextRequest: () => new Promise((resolve) => { notifyRequest = resolve; }), + release: () => { + holdResponses = false; + for (const response of heldResponses) response.writeHead(200, { "content-type": "application/json" }).end("{}"); + heldResponses.clear(); + }, + close: async () => { + server.closeAllConnections(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } + }; +} + +const testActivities: TraceTestActivities = { + noArgumentActivity: async () => "done", + retryOnceActivity: async () => { + if (Context.current().info.attempt === 1) { + throw ApplicationFailure.create({ message: privateText, type: "TOOL_UNAVAILABLE" }); + } + return "retried"; + }, + privateActivity: async () => { + noteActivityFailureReason("TOOL_UNAVAILABLE"); + return { status: "FAILED", message: privateText }; + } +}; + +describe("OTLP through real Temporal workflows and activities", () => { + let env: TestWorkflowEnvironment; + let workflowBundle: WorkflowBundleWithSourceMap; + let untracedBundle: WorkflowBundleWithSourceMap; + + beforeAll(async () => { + Runtime.install({ logger: new DefaultLogger("ERROR") }); + env = await TestWorkflowEnvironment.createTimeSkipping(); + [workflowBundle, untracedBundle] = await Promise.all([ + bundleWorkflowCode({ workflowsPath, workflowInterceptorModules: [workflowInterceptorModule], logger: new DefaultLogger("ERROR") }), + bundleWorkflowCode({ workflowsPath, logger: new DefaultLogger("ERROR") }) + ]); + }, 90_000); + + afterAll(async () => { await env?.teardown(); }); + + async function workerFor(spans: SpanExporter | null, overrides: Partial = {}) { + const taskQueue = `otlp-test-${randomUUID()}`; + const tracing = createWorkerTracing(spans, { workflowInterceptorModule }); + const worker = await Worker.create({ + connection: env.nativeConnection, + namespace: env.namespace, + taskQueue, + workflowBundle: spans ? workflowBundle : untracedBundle, + ...tracing, + // The same workflow interceptor is already in the prebuilt bundle. + interceptors: { ...tracing.interceptors, workflowModules: [] }, + activities: { ...testActivities, ...overrides }, + shutdownForceTime: "10 seconds" + }); + return { worker, taskQueue }; + } + + function exporter(endpoint: string, serviceName = "space-worker") { + return createOtlpSpanExporter({ endpoint, serviceName, flushIntervalMs: 60_000, batchSize: 512 }); + } + + it("connects a real API span to workflow, child and every activity attempt, then replays without exports", async () => { + const collector = await startCollector(); + const apiSpans = exporter(collector.endpoint, "space-api"); + const workerSpans = exporter(collector.endpoint); + const observability = createHttpObservability({ serviceName: "space-api", spanExporter: apiSpans }); + const app = Fastify({ genReqId: () => "req:otlp-integration" }); + app.addHook("onRequest", async (request) => { + requestTraceContexts.remember(request.id, traceContextForRawRequest(request.raw)); + observability.onRequest(request); + }); + app.addHook("onResponse", async (request, reply) => observability.onResponse(request, reply)); + const workflowId = `otlp-tree-${randomUUID()}`; + const { worker, taskQueue } = await workerFor(workerSpans); + app.post("/api/trace-test/:roomId", async (request) => { + const input = withTurnTraceparent({ + traceId: request.id, roomId: "room:integration", missionId: "mission:integration", + prompt: privateText, toolOutput: privateText, path: privatePath, url: privateUrl + }); + return env.client.workflow.execute("traceTreeWorkflow", { + workflowId, taskQueue, args: [input], memo: workflowCorrelationMemo(input) + }); + }); + try { + const response = await worker.runUntil(app.inject({ + method: "POST", url: "/api/trace-test/private-room?query=private-query-value", headers: { traceparent } + })); + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual(["done", "retried", "done", "done"]); + await Promise.all([apiSpans.flush(), workerSpans.flush()]); + const spans = collector.spans(); + expect(spans).toHaveLength(9); + expect(new Set(spans.map((span) => span.spanId)).size).toBe(spans.length); + expect(new Set(spans.map((span) => span.traceId))).toEqual(new Set([traceId])); + const api = spans.find((span) => span.kind === 2)!; + const workflow = spans.find((span) => span.name === "workflow traceTreeWorkflow")!; + const child = spans.find((span) => span.name === "workflow traceChildWorkflow")!; + expect(api).toMatchObject({ name: "POST /api/trace-test/:roomId", parentSpanId: callerSpanId }); + expect(workflow).toMatchObject({ kind: 1, parentSpanId: api.spanId }); + expect(child).toMatchObject({ kind: 1, parentSpanId: workflow.spanId }); + const activities = spans.filter((span) => attributes(span)["space.activity_type"]); + expect(activities).toHaveLength(6); + for (const activity of activities) { + const values = attributes(activity); + expect(activity.parentSpanId).toBe(values["space.workflow_type"] === "traceChildWorkflow" ? child.spanId : workflow.spanId); + expect(values["space.activity_id"]).toBeTruthy(); + expect(values["space.temporal_run_id"]).toBeTruthy(); + } + const retries = activities.filter((span) => attributes(span)["space.activity_type"] === "retryOnceActivity") + .sort((a, b) => Number(attributes(a)["space.attempt"]) - Number(attributes(b)["space.attempt"])); + expect(retries).toHaveLength(2); + expect(attributes(retries[0]!)).toMatchObject({ + "space.attempt": "1", "space.outcome": "FAILED", "space.reason_code": "TOOL_UNAVAILABLE", "space.failure_category": "TOOL_UNAVAILABLE" + }); + expect(retries[0]!.status.code).toBe(2); + expect(attributes(retries[1]!)).toMatchObject({ "space.attempt": "2", "space.outcome": "COMPLETED" }); + expect(retries[1]!.status.code).toBe(0); + expect(attributes(activities.find((span) => attributes(span)["space.activity_type"] === "privateActivity")!)) + .toMatchObject({ "space.outcome": "FAILED_RESULT", "space.reason_code": "TOOL_UNAVAILABLE" }); + + const allowedAttributes = new Set([ + "http.request.method", "http.route", "http.response.status_code", "space.request_id", + "space.workflow_type", "space.workflow_id", "space.temporal_run_id", "space.activity_type", "space.activity_id", + "space.attempt", "space.outcome", "space.duration_ms", "space.queue_ms", "space.reason_code", "space.failure_category", + "space.trace_id", "space.w3c_trace_id", "space.mission_id", "space.room_id" + ]); + for (const span of spans) { + expect(span.attributes.map(({ key }) => key).filter((key) => !allowedAttributes.has(key))).toEqual([]); + expect(BigInt(span.endTimeUnixNano)).toBeGreaterThanOrEqual(BigInt(span.startTimeUnixNano)); + } + const wire = JSON.stringify(collector.requests); + for (const secret of [privateText, privatePath, privateUrl, "private-room", "private-query-value"]) expect(wire).not.toContain(secret); + expect(collector.requests.every((request) => request.path === "/v1/traces" && request.contentType === "application/json")).toBe(true); + expect(new Set(collector.requests.flatMap(({ body }) => body.resourceSpans.map((resource) => + resource.resource.attributes.find(({ key }) => key === "service.name")?.value.stringValue)))) + .toEqual(new Set(["space-api", "space-worker"])); + + const requestCount = collector.requests.length; + const beforeReplay = workerSpans.stats(); + for (const replayWorkflowId of [workflowId, `${workflowId}:child`]) { + const history = await env.client.workflow.getHandle(replayWorkflowId).fetchHistory(); + expect(history.events!.length).toBeGreaterThan(5); + const tracing = createWorkerTracing(workerSpans, { workflowInterceptorModule }); + await Worker.runReplayHistory({ workflowBundle, sinks: tracing.sinks }, history, replayWorkflowId); + } + await workerSpans.flush(); + expect(collector.requests).toHaveLength(requestCount); + expect(workerSpans.stats()).toEqual(beforeReplay); + } finally { + await app.close(); + observability.close(); + await Promise.all([apiSpans.close(), workerSpans.close()]); + await collector.close(); + } + }, 60_000); + + it("links distinct continue-as-new runs and attaches no-argument activities to their own run", async () => { + const collector = await startCollector(); + const spans = exporter(collector.endpoint); + try { + const { worker, taskQueue } = await workerFor(spans); + const workflowId = `otlp-continue-${randomUUID()}`; + expect(await worker.runUntil(env.client.workflow.execute("traceContinueWorkflow", { + workflowId, taskQueue, args: [0], memo: { traceparent } + }))).toBe("continued"); + await spans.flush(); + const exported = collector.spans(); + const workflows = exported.filter((span) => span.name === "workflow traceContinueWorkflow"); + expect(workflows).toHaveLength(2); + const first = workflows.find((span) => span.parentSpanId === callerSpanId)!; + const next = workflows.find((span) => span.spanId !== first.spanId)!; + expect(next).toMatchObject({ traceId, parentSpanId: first.spanId, links: [{ traceId, spanId: first.spanId }] }); + expect(attributes(first)["space.temporal_run_id"]).not.toBe(attributes(next)["space.temporal_run_id"]); + const activities = exported.filter((span) => attributes(span)["space.activity_type"]); + expect(activities).toHaveLength(2); + for (const activity of activities) { + const run = workflows.find((workflow) => attributes(workflow)["space.temporal_run_id"] === attributes(activity)["space.temporal_run_id"]); + expect(activity.parentSpanId).toBe(run!.spanId); + } + const requestCount = collector.requests.length; + for (const workflow of workflows) { + const history = await env.client.workflow.getHandle(workflowId, attributes(workflow)["space.temporal_run_id"]).fetchHistory(); + await Worker.runReplayHistory({ workflowBundle, sinks: createWorkerTracing(spans).sinks }, history, workflowId); + } + await spans.flush(); + expect(collector.requests).toHaveLength(requestCount); + } finally { + await spans.close(); + await collector.close(); + } + }, 60_000); + + it("separates requests on a long-lived supervisor and links successive background missions to the actual preceding spans", async () => { + const collector = await startCollector(); + const apiSpans = exporter(collector.endpoint, "space-api"); + const spans = exporter(collector.endpoint); + const observability = createHttpObservability({ serviceName: "space-api", spanExporter: apiSpans }); + const app = Fastify({ genReqId: () => `req:${randomUUID()}` }); + app.addHook("onRequest", async (request) => { + requestTraceContexts.remember(request.id, traceContextForRawRequest(request.raw)); + observability.onRequest(request); + }); + app.addHook("onResponse", async (request, reply) => observability.onResponse(request, reply)); + const { worker, taskQueue } = await workerFor(spans); + const workflowId = `otlp-supervisor-${randomUUID()}`; + const handle = env.client.workflow.getHandle(workflowId); + const secondTraceId = "2bf92f3577b34da6a3ce929d0e0e4736"; + app.post("/api/start-trace-supervisor", async (request) => { + const input = withTurnTraceparent({ traceId: request.id, missionId: "mission:first" }); + await env.client.workflow.start("traceMissionWorkflow", { + workflowId, taskQueue, args: [4], memo: workflowCorrelationMemo(input) + }); + await handle.signal("traceMission", input); + return { ok: true }; + }); + app.post("/api/trace-mission", async (request) => { + await handle.signal("traceMission", withTurnTraceparent({ traceId: request.id, missionId: "mission:second" })); + return { ok: true }; + }); + const mission = (missionId: string) => collector.spans().find((span) => + span.name === "workflow traceMissionWorkflow mission" && attributes(span)["space.mission_id"] === missionId)!; + const waitForMission = async (count: number) => { + await expect.poll(() => handle.query("completedTraceMissions"), { timeout: 10_000 }).toBe(count); + await spans.flush(); + }; + try { + expect(await worker.runUntil(async () => { + expect((await app.inject({ method: "POST", url: "/api/start-trace-supervisor", headers: { traceparent } })).statusCode).toBe(200); + await waitForMission(1); + expect((await app.inject({ + method: "POST", url: "/api/trace-mission", headers: { traceparent: `00-${secondTraceId}-${callerSpanId}-01` } + })).statusCode).toBe(200); + await waitForMission(2); + const second = mission("mission:second"); + expect(second).toBeDefined(); + await handle.signal("traceMission", { + traceId: "trace:bg-integration-one", agentRunId: "agentrun:bg-integration-one", missionId: "mission:bg-one", + continuedTraceparent: `00-${second.traceId}-${second.spanId}-01` + }); + await waitForMission(3); + const background = mission("mission:bg-one"); + expect(background).toBeDefined(); + await handle.signal("traceMission", { + traceId: "trace:bg-integration-two", agentRunId: "agentrun:bg-integration-two", missionId: "mission:bg-two", + continuedTraceparent: `00-${background.traceId}-${background.spanId}-01` + }); + return handle.result(); + })).toBe(4); + await Promise.all([apiSpans.flush(), spans.flush()]); + const exported = collector.spans(); + const root = exported.find((span) => span.name === "workflow traceMissionWorkflow")!; + const firstApi = exported.find((span) => span.name === "POST /api/start-trace-supervisor")!; + const secondApi = exported.find((span) => span.name === "POST /api/trace-mission")!; + const first = mission("mission:first"); + const second = mission("mission:second"); + const background = mission("mission:bg-one"); + const nextBackground = mission("mission:bg-two"); + expect(root).toMatchObject({ traceId, parentSpanId: firstApi.spanId }); + expect(first).toMatchObject({ traceId, parentSpanId: firstApi.spanId }); + expect(second).toMatchObject({ traceId: secondTraceId, parentSpanId: secondApi.spanId }); + expect(second.links).toContainEqual({ traceId: root.traceId, spanId: root.spanId }); + expect(new Set([first.traceId, second.traceId, background.traceId, nextBackground.traceId]).size).toBe(4); + for (const [previous, current] of [[second, background], [background, nextBackground]]) { + expect(current!.parentSpanId).toBeUndefined(); + expect(current!.links).toContainEqual({ traceId: previous!.traceId, spanId: previous!.spanId }); + } + const activitySpans = exported.filter((span) => attributes(span)["space.activity_type"]); + expect(activitySpans).toHaveLength(4); + for (const current of [first, second, background, nextBackground]) { + expect(activitySpans.filter((span) => span.traceId === current.traceId && span.parentSpanId === current.spanId)).toHaveLength(1); + } + const count = collector.requests.length; + const history = await handle.fetchHistory(); + await Worker.runReplayHistory({ workflowBundle, sinks: createWorkerTracing(spans).sinks }, history, workflowId); + await spans.flush(); + expect(collector.requests).toHaveLength(count); + } finally { + await app.close(); + observability.close(); + await Promise.all([apiSpans.close(), spans.close()]); + await collector.close(); + } + }, 60_000); + + it("finishes real workflow work while the collector has not responded", async () => { + const collector = await startCollector(); + collector.hold(); + const spans = createOtlpSpanExporter({ endpoint: collector.endpoint, serviceName: "space-worker", batchSize: 1, timeoutMs: 30_000 }); + try { + const requestReceived = collector.nextRequest(); + const { worker, taskQueue } = await workerFor(spans); + const result = await worker.runUntil(env.client.workflow.execute("traceSingleActivityWorkflow", { + workflowId: `otlp-slow-${randomUUID()}`, taskQueue, memo: { traceparent } + })); + await requestReceived; + expect(result).toBe("done"); + // No wall-clock threshold: the collector is still deliberately blocked + // after both the activity and its workflow have completed successfully. + expect(collector.pendingResponses).toBe(1); + expect(spans.stats().exported).toBe(0); + collector.release(); + await spans.flush(); + expect(spans.stats()).toMatchObject({ exported: 2, failed: 0 }); + } finally { + collector.release(); + await spans.close(); + await collector.close(); + } + }, 60_000); + + it("exports production maintenance, consolidation and mutation workflows across both memory worker queues", async () => { + const collector = await startCollector(); + const maintenanceSpans = exporter(collector.endpoint, "space-memory-maintenance-worker"); + const mutationSpans = exporter(collector.endpoint, "space-memory-mutation-worker"); + const sourceHash = "a".repeat(64); + const changeSetId = "changeset:otlp-integration"; + const maintenanceResult = { + status: "REFRESHED", generatedAt: "2026-10-18T00:00:00.000Z", sourceHash, previousSourceHash: null, + summary: { sourceCount: 0, recordCount: 0, nodeCount: 0, edgeCount: 0, issueCount: 0 }, durationMs: 1 + }; + const run = new InMemorySpaceStore().createMemoryConsolidationRun({ + mode: "REPAIR", triggerKind: "SCHEDULED", workflowId: `otlp-consolidation-${randomUUID()}`, + dedupeKey: `otlp-memory-${randomUUID()}`, sourceHash, actorUserId: null + }); + const mutationResult = { status: "APPLIED", changeSetId, resultingSourceHash: sourceHash, reason: null }; + try { + const [maintenanceBundle, mutationBundle] = await Promise.all([ + bundleWorkflowCode({ + workflowsPath: fileURLToPath(new URL("../src/memory-maintenance-workflows.ts", import.meta.url)), + workflowInterceptorModules: [workflowInterceptorModule], logger: new DefaultLogger("ERROR") + }), + bundleWorkflowCode({ + workflowsPath: fileURLToPath(new URL("../src/memory-mutation-workflows.ts", import.meta.url)), + workflowInterceptorModules: [workflowInterceptorModule], logger: new DefaultLogger("ERROR") + }) + ]); + const maintenanceTracing = createWorkerTracing(maintenanceSpans); + const mutationTracing = createWorkerTracing(mutationSpans); + const maintenance = await Worker.create({ + connection: env.nativeConnection, namespace: env.namespace, taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE, + workflowBundle: maintenanceBundle, ...maintenanceTracing, + interceptors: { ...maintenanceTracing.interceptors, workflowModules: [] }, + activities: { + refreshMemoryGraphSnapshot: async () => maintenanceResult, + refreshAllMonthsMemoryGraphSnapshot: async () => maintenanceResult, + reconcileStaleSpaceAgentSessions: async () => 0, + createScheduledMemoryAudit: async () => run, + prepareMemoryConsolidation: async () => ({ run, changeSetId }), + finalizeMemoryRepair: async () => run + } + }); + const mutation = await Worker.create({ + connection: env.nativeConnection, namespace: env.namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE, + workflowBundle: mutationBundle, ...mutationTracing, + interceptors: { ...mutationTracing.interceptors, workflowModules: [] }, + activities: { executePersistedMemoryChangeSet: async () => mutationResult } + }); + const maintenanceId = `otlp-maintenance-${randomUUID()}`; + const mutationId = `otlp-mutation-${randomUUID()}`; + const results = await maintenance.runUntil(mutation.runUntil(async () => { + const maintenanceResult = await env.client.workflow.execute("memoryMaintenanceWorkflow", { + workflowId: maintenanceId, taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE, + args: [{ scheduledAt: "2026-10-18T00:00:00.000Z", traceId: "req:memory-integration" }], memo: { traceparent } + }); + const mutationResult = await env.client.workflow.execute("memoryMutationWorkflow", { + workflowId: mutationId, taskQueue: MEMORY_MUTATION_TASK_QUEUE, + args: [{ changeSetId, traceId: "req:memory-integration" }], memo: { traceparent } + }); + return [maintenanceResult, mutationResult]; + })); + expect(results).toEqual([maintenanceResult, mutationResult]); + await Promise.all([maintenanceSpans.flush(), mutationSpans.flush()]); + const exported = collector.spans(); + expect(exported).toHaveLength(11); + expect(new Set(exported.map((span) => span.traceId))).toEqual(new Set([traceId])); + const maintenanceWorkflow = exported.find((span) => span.name === "workflow memoryMaintenanceWorkflow")!; + const consolidation = exported.find((span) => span.name === "workflow memoryConsolidationWorkflow")!; + const mutationWorkflow = exported.find((span) => span.name === "workflow memoryMutationWorkflow")!; + expect(maintenanceWorkflow.parentSpanId).toBe(callerSpanId); + expect(consolidation.parentSpanId).toBe(maintenanceWorkflow.spanId); + expect(mutationWorkflow.parentSpanId).toBe(callerSpanId); + const workflowsByType = new Map([maintenanceWorkflow, consolidation, mutationWorkflow].map((span) => [attributes(span)["space.workflow_type"], span])); + for (const span of exported.filter((item) => attributes(item)["space.activity_type"])) { + expect(span.parentSpanId).toBe(workflowsByType.get(attributes(span)["space.workflow_type"])!.spanId); + } + const perService = new Map(); + for (const { body } of collector.requests) for (const resource of body.resourceSpans) { + const service = resource.resource.attributes.find(({ key }) => key === "service.name")!.value.stringValue!; + perService.set(service, [...(perService.get(service) ?? []), ...resource.scopeSpans.flatMap((scope) => scope.spans)]); + } + expect(perService.get("space-memory-maintenance-worker")).toHaveLength(8); + expect(perService.get("space-memory-mutation-worker")).toHaveLength(3); + const count = collector.requests.length; + for (const id of [maintenanceId, run.workflowId]) { + await Worker.runReplayHistory({ workflowBundle: maintenanceBundle, sinks: maintenanceTracing.sinks }, + await env.client.workflow.getHandle(id).fetchHistory(), id); + } + await Worker.runReplayHistory({ workflowBundle: mutationBundle, sinks: mutationTracing.sinks }, + await env.client.workflow.getHandle(mutationId).fetchHistory(), mutationId); + await Promise.all([maintenanceSpans.flush(), mutationSpans.flush()]); + expect(collector.requests).toHaveLength(count); + } finally { + await Promise.all([maintenanceSpans.close(), mutationSpans.close()]); + await collector.close(); + } + }, 60_000); + + it("counts refused collector delivery without failing workflow or activity execution", async () => { + const collector = await startCollector(); + collector.fail(); + const spans = exporter(collector.endpoint); + try { + const { worker, taskQueue } = await workerFor(spans); + expect(await worker.runUntil(env.client.workflow.execute("traceSingleActivityWorkflow", { + workflowId: `otlp-failure-${randomUUID()}`, taskQueue, memo: { traceparent } + }))).toBe("done"); + await spans.flush(); + expect(spans.stats()).toEqual({ exported: 0, dropped: 0, failed: 2 }); + } finally { + await spans.close(); + await collector.close(); + } + }, 60_000); + + it("runs normally with tracing disabled when no endpoint is configured", async () => { + const collector = await startCollector(); + try { + const spans = otlpSpanExporterFromEnv({}, "space-worker"); + expect(spans).toBeNull(); + const { worker, taskQueue } = await workerFor(spans); + const workflowId = `otlp-disabled-${randomUUID()}`; + expect(await worker.runUntil(env.client.workflow.execute("traceRandomActivityIdWorkflow", { + workflowId, taskQueue, memo: { traceparent } + }))).toBe("done"); + expect(collector.requests).toHaveLength(0); + // Enabling tracing must also replay histories recorded before this patch, + // without adding export side effects or changing workflow commands. + const enabledSpans = exporter(collector.endpoint); + try { + await Worker.runReplayHistory({ workflowBundle, sinks: createWorkerTracing(enabledSpans).sinks }, + await env.client.workflow.getHandle(workflowId).fetchHistory(), workflowId); + await enabledSpans.flush(); + expect(collector.requests).toHaveLength(0); + } finally { + await enabledSpans.close(); + } + } finally { + await collector.close(); + } + }, 60_000); +}); diff --git a/apps/worker/tests/room-background-continuation.test.ts b/apps/worker/tests/room-background-continuation.test.ts index 127ccb649..58ea5572a 100644 --- a/apps/worker/tests/room-background-continuation.test.ts +++ b/apps/worker/tests/room-background-continuation.test.ts @@ -1,11 +1,32 @@ import { describe, expect, it, vi } from "vitest"; -import { roomAgentSupervisorInputSchema } from "@space/contracts"; +import { formatTraceparent, roomAgentSupervisorInputSchema } from "@space/contracts"; import { continuationEvidence, discardSupervisorTurn, finishSupervisorTurn, prepareBackgroundContinuation, supervisorTurnState } from "../src/room-background-continuation.js"; import { continuationFixture } from "./room-continuation-fixture.js"; import { runCodexAppServerTurn } from "../src/activities.js"; +import { backgroundMissionTraceContext } from "../src/temporal-trace-context.js"; describe("durable background continuation", () => { + it("starts new traces with persisted links to the immediately preceding mission", async () => { + const parent = formatTraceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7"); + const c = continuationFixture("OPEN_TYPES", parent); + const first = await prepareBackgroundContinuation(c.input, c.store); + if (first.state !== "READY") throw Error("Missing first continuation"); + expect(first.item.turn.traceId).toBe("trace:bg-test-1"); + expect(first.item.turn.traceparent).toBeUndefined(); + expect(first.item.turn.continuedTraceparent).toBe(parent); + c.store.updateSpaceAgentRun(first.item.turn.agentRunId!, { status: "COMPLETED" }); + c.store.markRoomAgentMissionSignaled(c.room.id, "background-test-1"); + const second = await prepareBackgroundContinuation({ ...c.input, continuationId: "test-2" }, c.store); + if (second.state !== "READY") throw Error("Missing second continuation"); + const previous = backgroundMissionTraceContext(first.item.turn)!; + expect(second.item.turn.traceId).toBe("trace:bg-test-2"); + expect(second.item.turn.traceparent).toBeUndefined(); + expect(second.item.turn.continuedTraceparent).toBe(formatTraceparent(previous.traceId, previous.parentId, previous.flags)); + expect(second.item.turn.continuedTraceparent).not.toBe(parent); + expect(await prepareBackgroundContinuation({ ...c.input, continuationId: "test-2" }, c.store)).toEqual(second); + }); + it("includes the actual failure reason when resuming after a parsed background action", () => { const c = continuationFixture(); const evidence = JSON.parse(continuationEvidence([{ ...c.action, status: "FAILED", diff --git a/apps/worker/tests/room-continuation-fixture.ts b/apps/worker/tests/room-continuation-fixture.ts index 340e964e0..8de19a7d2 100644 --- a/apps/worker/tests/room-continuation-fixture.ts +++ b/apps/worker/tests/room-continuation-fixture.ts @@ -1,7 +1,7 @@ import { buildCodexAppServerTurnWorkflowId, roomAgentSupervisorQueueItemSchema } from "@space/contracts"; import { InMemorySpaceStore } from "@space/runtime"; -export function continuationFixture(actionType: "OPEN_TYPES" | "CONFIGURE_PANE" = "OPEN_TYPES") { +export function continuationFixture(actionType: "OPEN_TYPES" | "CONFIGURE_PANE" = "OPEN_TYPES", traceparent?: string) { const store = new InMemorySpaceStore(); const room = store.createRoom({ name: "Continuation regression", initialPaneCount: 0 }); const pane = store.getOrCreateRoomAgentPane(room.id); @@ -14,7 +14,7 @@ export function continuationFixture(actionType: "OPEN_TYPES" | "CONFIGURE_PANE" modelId: "gpt-5.6-sol", reasoningEffort: "high", agentSessionId: session.sessionId, agentRunId: `agent_run:${suffix}`, agentUserMessageId: `agent_msg:user-${suffix}`, agentAssistantMessageId: `agent_msg:assistant-${suffix}`, roomAgentMissionId: "mission:continuation", - traceId: `trace:continuation-${suffix}`, selectedToolIds: ["room:inspect", "room:open_types", "room:send"] + traceId: `trace:continuation-${suffix}`, traceparent, selectedToolIds: ["room:inspect", "room:open_types", "room:send"] } }); return store.enqueueRoomAgentMission({ requestId: `request:${suffix}`, clientRequestId: `client-${suffix}`, content: prompt, supervisorWorkflowId: `room-supervisor:${room.id}`, childWorkflowId: buildCodexAppServerTurnWorkflowId(item.turn), diff --git a/apps/worker/tests/room-supervisor-continuation.test.ts b/apps/worker/tests/room-supervisor-continuation.test.ts index a38ef895c..b6db5bd0f 100644 --- a/apps/worker/tests/room-supervisor-continuation.test.ts +++ b/apps/worker/tests/room-supervisor-continuation.test.ts @@ -3,6 +3,10 @@ import { ROOM_AGENT_ENQUEUE_SIGNAL, ROOM_AGENT_STOP_SIGNAL, roomAgentSupervisorI import { continuationFixture } from "./room-continuation-fixture.js"; import { discardSupervisorTurn, finishSupervisorTurn, prepareBackgroundContinuation, supervisorTurnState } from "../src/room-background-continuation.js"; +vi.mock("../src/workflow-tracing.js", () => ({ + withWorkflowTraceContext: (_input: unknown, run: () => Promise) => run() +})); + const temporal = vi.hoisted(() => ({ handlers: new Map void>(), activities: {} as Record, child: vi.fn(), condition: vi.fn(), continueAsNew: vi.fn(), patch: true, counter: 0 diff --git a/apps/worker/tests/workflow-tracing.test.ts b/apps/worker/tests/workflow-tracing.test.ts new file mode 100644 index 000000000..77f17b472 --- /dev/null +++ b/apps/worker/tests/workflow-tracing.test.ts @@ -0,0 +1,381 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { ApplicationFailure } from "@temporalio/workflow"; +import { formatTraceparent } from "@space/contracts"; +import type { SpanExporter } from "@space/runtime"; +import type { WorkflowSpanRecord } from "../src/temporal-trace-context.js"; + +const traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; +const laterTraceparent = "00-11111111111111111111111111111111-2222222222222222-01"; +const privateText = "PRIVATE PROMPT /home/operator/project?token=secret"; + +const context = vi.hoisted(() => ({ + spans: [] as WorkflowSpanRecord[], + info: { + workflowId: "workflow:supervisor:room:one", + workflowType: "supervisorWorkflow", + runId: "8dfb046b-ec46-42bb-bc54-2df32b8b5c36", + firstExecutionRunId: "8dfb046b-ec46-42bb-bc54-2df32b8b5c36", + runStartTime: new Date(1_000), + memo: {} as Record, + attempt: 1 + } +})); + +vi.mock("@temporalio/workflow", async (importOriginal) => { + const original = await importOriginal(); + const { AsyncLocalStorage } = await import("node:async_hooks"); + return { + ...original, + AsyncLocalStorage, + workflowInfo: () => context.info, + proxySinks: () => ({ spaceTracing: { record: (span: WorkflowSpanRecord) => context.spans.push(span) } }) + }; +}); + +import { + backgroundMissionTraceContext, + decodeActivityTraceContext, + decodeTraceLink, + deterministicWorkflowSpanId, + deterministicWorkflowTraceId, + encodeTraceHeaders, + sanitizeWorkflowTraceAttributes, + traceparentFromArgs, + workflowTraceAttributes +} from "../src/temporal-trace-context.js"; +import { interceptors, withWorkflowTraceContext } from "../src/workflow-tracing.js"; +import { createWorkerTracing } from "../src/worker-tracing.js"; + +const parsed = { traceId: "4bf92f3577b34da6a3ce929d0e0e4736", parentId: "00f067aa0ba902b7", flags: "01" }; + +beforeEach(() => { + context.spans.length = 0; + context.info.memo = {}; +}); + +describe("pure Temporal trace context", () => { + it("preserves unrelated headers and decodes binary and standard JSON traceparents", () => { + const existing = { custom: { data: Uint8Array.of(3, 4) } }; + const headers = encodeTraceHeaders(existing, parsed, parsed); + expect(headers.custom).toBe(existing.custom); + expect(existing).toEqual({ custom: { data: Uint8Array.of(3, 4) } }); + expect(decodeActivityTraceContext(headers)).toEqual(parsed); + expect(decodeTraceLink(headers)).toEqual(parsed); + expect(decodeTraceLink(encodeTraceHeaders(headers, parsed))).toBeNull(); + expect(decodeActivityTraceContext({ traceparent: { data: new TextEncoder().encode(JSON.stringify(traceparent)) } })).toEqual(parsed); + }); + + it("ignores malformed, zero, oversized or non-ASCII payloads and can use a valid fallback", () => { + for (const data of [Uint8Array.of(255), new Uint8Array(513), new TextEncoder().encode('"broken'), new TextEncoder().encode("00-00000000000000000000000000000000-00f067aa0ba902b7-01")]) { + expect(decodeActivityTraceContext({ "space-traceparent": { data } })).toBeNull(); + } + expect(decodeActivityTraceContext({ + "space-traceparent": { data: new Uint8Array(513) }, + traceparent: { data: new TextEncoder().encode(traceparent) } + })).toEqual(parsed); + }); + + it("extracts only explicit context and allowlisted identifiers, never payload text", () => { + const args = [{ item: { missionId: "mission:one", turn: { traceparent, traceId: "req:one", roomId: "room:one", prompt: privateText, paneId: privateText } } }]; + expect(traceparentFromArgs(args)).toEqual(parsed); + expect(workflowTraceAttributes(args)).toEqual({ "space.mission_id": "mission:one", "space.trace_id": "req:one", "space.room_id": "room:one" }); + expect(traceparentFromArgs([{ content: { traceparent } }])).toBeNull(); + expect(sanitizeWorkflowTraceAttributes({ + "space.room_id": "room:one", "space.pane_id": privateText, "space.prompt": privateText, "space.url": "https://secret.example" + })).toEqual({ "space.room_id": "room:one" }); + }); + + it("derives replay-stable IDs without drawing from the application random stream", () => { + const random = vi.spyOn(Math, "random").mockImplementation(() => { throw new Error("application RNG must stay untouched"); }); + try { + const first = deterministicWorkflowSpanId(context.info.runId); + expect(first).toMatch(/^[0-9a-f]{16}$/); + expect(first).not.toBe("0000000000000000"); + expect(deterministicWorkflowSpanId(context.info.runId)).toBe(first); + expect(deterministicWorkflowSpanId(`${context.info.runId}-next`)).not.toBe(first); + expect(deterministicWorkflowTraceId(context.info.runId)).toMatch(/^[0-9a-f]{32}$/); + expect(random).not.toHaveBeenCalled(); + } finally { random.mockRestore(); } + }); + + it("gives a background mission an exact reusable context across supervisor runs", () => { + const turn = { traceId: "trace:bg-continuation:one", agentRunId: "agent_run:one", continuedTraceparent: traceparent }; + const first = backgroundMissionTraceContext(turn)!; + expect(first.traceId).not.toBe(parsed.traceId); + expect(backgroundMissionTraceContext({ ...turn, supervisorRunId: "a-different-run" })).toEqual(first); + expect(backgroundMissionTraceContext({ ...turn, agentRunId: "agent_run:two" })).not.toEqual(first); + expect(backgroundMissionTraceContext({ traceId: "req:one" })).toBeNull(); + expect(decodeActivityTraceContext(encodeTraceHeaders({}, first))).toEqual(first); + }); +}); + +describe("workflow tracing scopes", () => { + it("isolates concurrent signal missions and restores the enclosing run context", async () => { + const tracing = interceptors(); + const outbound = tracing.outbound![0]!; + const parents: Record> = {}; + const activity = (key: string) => outbound.scheduleActivity!({ headers: {}, args: [], activityType: "noArgs", seq: 1, options: {} }, async (input) => { + parents[key] = decodeActivityTraceContext(input.headers); + return undefined; + }); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + await tracing.inbound![0]!.execute!({ args: [{ traceparent }], headers: {} }, async () => { + const first = withWorkflowTraceContext({ traceparent, missionId: "mission:first" }, async () => { await gate; await activity("first"); }); + const later = withWorkflowTraceContext({ traceparent: laterTraceparent, missionId: "mission:later" }, async () => { await activity("later"); release(); }); + await Promise.all([first, later]); + await activity("run"); + }); + const first = context.spans.find((span) => span.attributes["space.mission_id"] === "mission:first")!; + const later = context.spans.find((span) => span.attributes["space.mission_id"] === "mission:later")!; + const root = context.spans.find((span) => !span.operation)!; + expect(parents.first).toMatchObject({ traceId: first.traceId, parentId: first.spanId }); + expect(parents.later).toMatchObject({ traceId: later.traceId, parentId: later.spanId }); + expect(parents.run).toMatchObject({ traceId: root.traceId, parentId: root.spanId }); + expect(first.parentSpanId).toBe(parsed.parentId); + expect(later.traceId).toBe("11111111111111111111111111111111"); + expect(later.links).toContainEqual({ traceId: root.traceId, spanId: root.spanId }); + expect(root.traceId).toBe(parsed.traceId); + }); + + it("links consecutive background missions to the immediately preceding emitted span", async () => { + const tracing = interceptors(); + const first = { traceId: "trace:bg-first", agentRunId: "agent_run:first", continuedTraceparent: traceparent }; + const firstContext = backgroundMissionTraceContext(first)!; + const second = { + traceId: "trace:bg-second", agentRunId: "agent_run:second", + continuedTraceparent: formatTraceparent(firstContext.traceId, firstContext.parentId, firstContext.flags) + }; + await tracing.inbound![0]!.execute!({ args: [{ traceparent }], headers: {} }, async () => { + await withWorkflowTraceContext(first, async () => "done"); + await withWorkflowTraceContext(second, async () => "done"); + }); + const missions = context.spans.filter((span) => span.operation === "mission"); + expect(missions[0]).toMatchObject({ traceId: firstContext.traceId, spanId: firstContext.parentId, parentSpanId: null }); + expect(missions[1]!.links).toContainEqual({ traceId: missions[0]!.traceId, spanId: missions[0]!.spanId }); + expect(missions[1]!.traceId).not.toBe(missions[0]!.traceId); + }); + + it("keeps returned failures and thrown errors unchanged and never records their messages", async () => { + const tracing = interceptors(); + // A workflow fails only with a Temporal failure; a plain Error fails (and retries) the task. + const error = ApplicationFailure.create({ message: privateText, type: "TOOL_UNAVAILABLE" }); + await expect(tracing.inbound![0]!.execute!({ args: [], headers: {} }, async () => { throw error; })).rejects.toBe(error); + expect(context.spans[0]).toMatchObject({ outcome: "FAILED", reasonCode: "TOOL_UNAVAILABLE" }); + const second = interceptors(); + const result = { status: "FAILED", message: privateText }; + await expect(second.inbound![0]!.execute!({ args: [], headers: {} }, async () => result)).resolves.toBe(result); + expect(context.spans[1]).toMatchObject({ outcome: "FAILED_RESULT" }); + expect(JSON.stringify(context.spans)).not.toContain(privateText); + }); + + it("records cancellation caught and returned by a supervisor mission without changing the result", async () => { + for (const status of ["INTERRUPTED", "CANCELLED"]) { + const tracing = interceptors(); + const result = { status, reason: privateText }; + await expect(tracing.inbound![0]!.execute!({ args: [{ traceparent }], headers: {} }, async () => + withWorkflowTraceContext({ traceparent }, async () => result))).resolves.toBe(result); + expect(context.spans.at(-2)).toMatchObject({ operation: "mission", outcome: "CANCELLED", reasonCode: "WORKFLOW_CANCELLED" }); + } + expect(JSON.stringify(context.spans)).not.toContain(privateText); + }); + + it("passes through mission callbacks without exporting when no interceptor is active", async () => { + interceptors(); + await expect(withWorkflowTraceContext({ traceparent }, async () => "untraced")).resolves.toBe("untraced"); + expect(context.spans).toEqual([]); + }); +}); + +describe("host workflow sink", () => { + function exporter(record = vi.fn()): SpanExporter { + return { record, flush: async () => {}, close: async () => {}, stats: () => ({ exported: 0, dropped: 0, failed: 0 }) }; + } + + it("does not register sandbox interceptors or sinks when disabled", () => { + const options = createWorkerTracing(null); + expect(options.sinks).toBeUndefined(); + expect(options.interceptors!.workflowModules).toBeUndefined(); + expect(options.interceptors!.activity).toHaveLength(1); + }); + + it("suppresses replay callbacks, sanitizes export attributes and contains exporter failures", () => { + const capture = vi.fn(); + const options = createWorkerTracing(exporter(capture)); + const sink = options.sinks!.spaceTracing.record; + expect(sink.callDuringReplay).toBe(false); + const span: WorkflowSpanRecord = { + traceId: parsed.traceId, spanId: "3333333333333333", parentSpanId: parsed.parentId, + startTimeMs: 1_000, endTimeMs: 1_200, outcome: "FAILED", reasonCode: "TOOL_UNAVAILABLE", + attributes: { "space.room_id": "room:one", "space.prompt": privateText, "space.pane_id": privateText }, + links: [{ traceId: "11111111111111111111111111111111", spanId: "4444444444444444" }] + }; + sink.fn(context.info as never, span); + expect(capture).toHaveBeenCalledOnce(); + expect(capture.mock.calls[0]![0]).toMatchObject({ + name: "workflow supervisorWorkflow", kind: 1, error: true, + startTimeUnixNano: 1_000_000_000n, endTimeUnixNano: 1_200_000_000n, + attributes: { "space.room_id": "room:one", "space.failure_category": "TOOL_UNAVAILABLE", "space.workflow_id": context.info.workflowId } + }); + expect(capture.mock.calls[0]![0].attributes).not.toHaveProperty("space.prompt"); + expect(capture.mock.calls[0]![0].attributes).not.toHaveProperty("space.pane_id"); + const throwing = createWorkerTracing(exporter(() => { throw new Error(privateText); })).sinks!.spaceTracing.record; + expect(() => throwing.fn(context.info as never, span)).not.toThrow(); + sink.fn(context.info as never, { ...span, outcome: privateText }); + sink.fn(context.info as never, { ...span, startTimeMs: Number.NaN }); + expect(capture).toHaveBeenCalledOnce(); + }); +}); + +describe("Opus review: request-independent supervisor runs", () => { + it("roots the production supervisor run in its own trace, links its starting request and keeps continued runs chained", async () => { + const original = { workflowType: context.info.workflowType, runId: context.info.runId }; + context.info.workflowType = "roomAgentSupervisorWorkflow"; + context.info.memo = { traceparent, missionId: "mission:first", spaceTraceId: "req:first-request" }; + try { + let tracing = interceptors(); + let pollParent: ReturnType = null; + await tracing.inbound![0]!.execute!({ args: [{ roomId: "room:one" }], headers: {} }, async () => { + await withWorkflowTraceContext({ missionId: "mission:first", turn: { traceparent, traceId: "req:first-request" } }, async () => "done"); + await tracing.outbound![0]!.scheduleActivity!({ headers: {}, args: [{ missionId: "mission:later" }], activityType: "poll", seq: 1, options: {} }, + async (input) => { pollParent = decodeActivityTraceContext(input.headers); return undefined; }); + }); + const root = context.spans.find((span) => !span.operation)!; + const mission = context.spans.find((span) => span.operation === "mission")!; + expect(root.traceId).not.toBe(parsed.traceId); + expect(root.parentSpanId).toBeNull(); + expect(root.links).toContainEqual({ traceId: parsed.traceId, spanId: parsed.parentId }); + expect(root.attributes["space.mission_id"]).toBeUndefined(); + expect(mission).toMatchObject({ traceId: parsed.traceId, parentSpanId: parsed.parentId }); + expect(pollParent).toMatchObject({ traceId: root.traceId, parentId: root.spanId }); + + // A continued run carries the link header and stays a child of the preceding run. + const previous = { traceId: root.traceId, parentId: root.spanId, flags: "01" }; + context.spans.length = 0; + context.info.runId = "9e0c157c-fd57-43c3-8d65-3e043c9c6d47"; + tracing = interceptors(); + await tracing.inbound![0]!.execute!({ args: [{ roomId: "room:one" }], headers: encodeTraceHeaders({}, previous, previous) }, async () => "next"); + expect(context.spans.find((span) => !span.operation)).toMatchObject({ traceId: root.traceId, parentSpanId: root.spanId }); + } finally { + Object.assign(context.info, original); + } + }); +}); + +describe("Opus reconciliation: legacy background items and workflow-task failures", () => { + it("roots a legacy background item that still carries its source traceparent, keeping that context as a link", async () => { + const tracing = interceptors(); + const legacy = { missionId: "mission:legacy", turn: { traceId: "trace:bg-legacy-one", agentRunId: "agent_run:bg-legacy-one", traceparent } }; + const background = backgroundMissionTraceContext(legacy.turn)!; + await tracing.inbound![0]!.execute!({ args: [{ traceparent }], headers: {} }, async () => withWorkflowTraceContext(legacy, async () => "done")); + const mission = context.spans.find((span) => span.operation === "mission")!; + expect(mission).toMatchObject({ traceId: background.traceId, spanId: background.parentId, parentSpanId: null }); + expect(mission.links).toContainEqual({ traceId: parsed.traceId, spanId: parsed.parentId }); + }); + + it("does not export a FAILED run span for a non-Temporal error, which only fails the retried workflow task", async () => { + const tracing = interceptors(); + const error = new Error(privateText); + await expect(tracing.inbound![0]!.execute!({ args: [], headers: {} }, async () => { throw error; })).rejects.toBe(error); + expect(context.spans).toHaveLength(0); + }); +}); + +describe("workflow span export boundary", () => { + function exporter(record = vi.fn()): SpanExporter { + return { record, flush: async () => {}, close: async () => {}, stats: () => ({ exported: 0, dropped: 0, failed: 0 }) }; + } + const record: WorkflowSpanRecord = { + traceId: parsed.traceId, spanId: "3333333333333333", parentSpanId: null, + startTimeMs: 1_000, endTimeMs: 1_200, outcome: "COMPLETED", attributes: {} + }; + + it("exports only generated : correlation identifiers", () => { + const capture = vi.fn(); + const sink = createWorkerTracing(exporter(capture)).sinks!.spaceTracing.record; + const attributes = workflowTraceAttributes([{ + missionId: "PRIVATE-PROMPT-and-tool-output-must-stay-local", paneId: "private-notes.txt", + roomId: "room:one", traceId: "trace:bg-continuation:one" + }]); + sink.fn(context.info as never, { ...record, attributes }); + const exported = capture.mock.calls[0]![0].attributes; + expect(exported).toMatchObject({ "space.room_id": "room:one", "space.trace_id": "trace:bg-continuation:one" }); + expect(exported).not.toHaveProperty("space.mission_id"); + expect(exported).not.toHaveProperty("space.pane_id"); + }); + + // The forms Space generates: makeSpaceId (`:`), API request ids, + // background continuation ids, Dot and goal hex ids, and derived trace ids. + const generated: Array<[string, string, string]> = [ + ["traceId", "space.trace_id", "trace:bg-6f1c2a4e-9b7d-4c1e-8a2f-3d5e7b9c1a20"], + ["traceId", "space.trace_id", "trace:omp-native-model:V1StGXR8_Z5j"], + ["traceId", "space.trace_id", "trace:memory-maintenance-2026-10-19"], + ["spaceTraceId", "space.trace_id", "trace:V1StGXR8_Z5j"], + ["requestId", "space.request_id", "req:V1StGXR8_Z5j"], + ["parentSpaceTraceId", "space.request_id", "req:V1StGXR8_Z5j"], + ["roomId", "space.room_id", "room:V1StGXR8_Z5j"], + ["paneId", "space.pane_id", "pane:V1StGXR8_Z5j"], + ["missionId", "space.mission_id", "room_agent_mission:V1StGXR8_Z5j"], + ["roomAgentMissionId", "space.mission_id", "room_agent_mission:V1StGXR8_Z5j"], + ["agentRunId", "space.agent_run_id", "agent_run:bg-6f1c2a4e-9b7d-4c1e-8a2f-3d5e7b9c1a20"], + ["agentSessionId", "space.agent_session_id", "agent_session:V1StGXR8_Z5j"], + ["actionId", "space.action_id", "room_agent_action:V1StGXR8_Z5j"], + ["goalId", "space.goal_id", "dot_goal:0123456789abcdef0123456789abcdef"], + ["dotId", "space.dot_id", "dot:0123456789abcdef0123456789abcdef"], + ["turnId", "space.turn_id", "turn:V1StGXR8_Z5j"], + ["cliTaskId", "space.cli_task_id", "cli_task:V1StGXR8_Z5j"], + ["revisionId", "space.revision_id", "cli_revision:V1StGXR8_Z5j"] + ]; + const notIdentifiers = [ + "PRIVATE PROMPT text", "private-notes.txt", "/home/operator/project", "https://example.test/a?token=secret", + "room:has space", "Room:upper-case-kind", "room:", "room:a.b", `room:${"x".repeat(129)}` + ]; + + it.each(generated)("exports a generated %s as %s", (field, key, value) => { + const capture = vi.fn(); + const sink = createWorkerTracing(exporter(capture)).sinks!.spaceTracing.record; + sink.fn(context.info as never, { ...record, attributes: workflowTraceAttributes([{ [field]: value }]) }); + expect(capture.mock.calls[0]![0].attributes[key]).toBe(value); + }); + + // Each value is offered through the workflow-side reader and, as an older + // sandbox could, directly under the exported key; the host drops it either way. + it.each([...new Map(generated.map(([field, key]) => [field, key])).entries()])("drops a %s (%s) that is not a generated identifier", (field, key) => { + const capture = vi.fn(); + const sink = createWorkerTracing(exporter(capture)).sinks!.spaceTracing.record; + for (const value of notIdentifiers) { + sink.fn(context.info as never, { ...record, attributes: workflowTraceAttributes([{ [field]: value }]) }); + sink.fn(context.info as never, { ...record, attributes: { [key]: value } }); + } + expect(capture).toHaveBeenCalledTimes(notIdentifiers.length * 2); + for (const [span] of capture.mock.calls) expect(span.attributes).not.toHaveProperty(key); + }); + + it("drops timestamps outside the OTLP fixed64 range and omits an invalid attempt", () => { + const capture = vi.fn(); + const sink = createWorkerTracing(exporter(capture)).sinks!.spaceTracing.record; + sink.fn({ ...context.info, attempt: -1 } as never, { ...record, startTimeMs: Number.MAX_SAFE_INTEGER, endTimeMs: Number.MAX_SAFE_INTEGER }); + expect(capture).not.toHaveBeenCalled(); + sink.fn({ ...context.info, attempt: -1 } as never, record); + expect(capture).toHaveBeenCalledOnce(); + expect(capture.mock.calls[0]![0].attributes).not.toHaveProperty("space.attempt"); + }); +}); + +describe("signal and update handler failures", () => { + it("exports no span for a non-Temporal handler error, which fails the retried workflow task, but records a Temporal failure", async () => { + const tracing = interceptors(); + const inbound = tracing.inbound![0]!; + const taskError = new Error(privateText); + const failure = ApplicationFailure.create({ message: privateText, type: "TOOL_UNAVAILABLE" }); + await inbound.execute!({ args: [{ traceparent }], headers: {} }, async () => { + await expect(inbound.handleSignal!({ signalName: "plainError", args: [], headers: {} }, async () => { throw taskError; })).rejects.toBe(taskError); + await expect(inbound.handleSignal!({ signalName: "temporalFailure", args: [], headers: {} }, async () => { throw failure; })).rejects.toBe(failure); + return "done"; + }); + const signals = context.spans.filter((span) => span.operation === "signal"); + expect(signals).toHaveLength(1); + expect(signals[0]).toMatchObject({ outcome: "FAILED", reasonCode: "TOOL_UNAVAILABLE", attributes: { "space.signal_name": "temporalFailure" } }); + expect(JSON.stringify(context.spans)).not.toContain(privateText); + }); +}); diff --git a/apps/worker/vitest.config.ts b/apps/worker/vitest.config.ts index c21b63776..bd36c0452 100644 --- a/apps/worker/vitest.config.ts +++ b/apps/worker/vitest.config.ts @@ -8,6 +8,8 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); export default defineConfig({ resolve: { alias: { + "@space/contracts/observability": resolve(__dirname, "../../packages/contracts/src/observability.ts"), + "@space/contracts/workflows": resolve(__dirname, "../../packages/contracts/src/workflows.ts"), "@space/db": resolve(__dirname, "../../packages/db/src/index.ts"), "@space/contracts": resolve(__dirname, "../../packages/contracts/src/index.ts"), "@space/opencode-control": resolve(__dirname, "../../packages/opencode-control/src/index.ts"), diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index e432af4c6..76ba6bfdf 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -52,3 +52,24 @@ background work, and is hidden in the web app. Public installs upgraded from 1.1.x that used Job Copilot, the games, channels, Telegram or the desktop routes set the modules they keep, for example `SPACE_OPTIONAL_MODULES=job-copilot,games`. + +## Optional distributed tracing + +Set `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` to the collector's HTTP(S) base URL, for +example `http://127.0.0.1:4318`, in the API and worker processes that should export +traces. The exporter sends OTLP/HTTP JSON to `/v1/traces`; a URL already ending in +that path is used directly. An unset or invalid endpoint disables export. + +The API exports request spans. The main worker and both memory workers export +workflow, child workflow, mission and activity-attempt spans, with propagation +through Temporal headers, inputs and memo. Continue-as-new preserves run lineage; +a background mission starts a new trace linked to the preceding mission. Each +process uses its existing service name. No tracing runtime package is required. + +The shared exporter bounds its queue to 4,096 spans and batches to 256, flushes +every five seconds, and limits each POST to three seconds. Collector failures +are counted and never block business execution. Workflow replay does not export +spans. Attributes are limited to bounded identifiers, types, durations, attempts, +outcomes and allowlisted reason codes; prompts and error messages are excluded. +See [observability](quality/observability.md) for propagation, replay coverage and +delivery limitations. Unset the endpoint and restart the processes to turn export off. diff --git a/docs/quality/observability.md b/docs/quality/observability.md index 488fc3036..8d7a3d782 100644 --- a/docs/quality/observability.md +++ b/docs/quality/observability.md @@ -215,46 +215,98 @@ Without the endpoint, every activity still writes `activity.started` and ## Traces (W3C trace context) -The trace context is local by default. An optional OTLP/HTTP (JSON) exporter sends one -SERVER span per API request when `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` is set (no -OpenTelemetry dependency; `packages/runtime/src/otlp-span-exporter.ts`). Since -2026-10-16 the worker and both memory workers use the same exporter (services -`space-worker`, `space-memory-maintenance-worker`, `space-memory-mutation-worker`): -one INTERNAL span per activity whose input carries a `traceparent`, as a child of -that span, with the activity type, outcome and attempt only. -- **API:** a valid inbound `traceparent` is continued (same trace ID, new span). - An invalid or missing header gets a new trace ID. Every API log line carries - `w3cTraceId` (Fastify `childLoggerFactory`). -- **Propagation into turns:** API turn starters and the Room Agent coordinator add - the request's `traceparent` to the turn input (an optional contract field) and - to the Temporal memo. A bounded in-memory map from request ID to trace context - (`apps/api/src/trace-context.ts`) does this without passing the request object - through every service. -- **Worker:** worker logs read the trace ID from the activity input. Internal - room-action calls from the worker send a child `traceparent`, so the internal - API request joins the same trace. -- **Workflow code is unchanged.** Correlation is done in the API starter - (input/memo) and in an activity interceptor. The workflow bundle only gains the - optional contract field. - -### Replay compatibility - -No file under `apps/worker/src` that the workflow bundle imports was changed -(`workflows.ts`, `room-supervisor-state.ts`, `turn-runtime-policy.ts`, -`dot-goal-loop.ts`). The `agent-turns` workflow bundle was built for the base -commit and for this change and compared after normalizing paths and webpack -module numbers. The only differences are in `@space/contracts`: -- the new `observability` module; -- the optional `traceparent` turn-input field; -- the optional file search and readiness fields. - -A history without `traceparent` parses exactly as before. Activity inputs are -not part of Temporal's determinism check. - -The read-only live replay (`node scripts/temporal-replay-check.mjs --limit 30`) -could not be run from the agent sandbox that made this change. Its network -namespace has no route to the Temporal container (`172.18.0.5:7233`), and -`127.0.0.1:7233` refuses connections. Run it before deploying this change. +The trace context is local by default. Set `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` +to an HTTP(S) collector base URL (for example `http://127.0.0.1:4318`) to enable +OTLP/HTTP JSON export. The shared runtime exporter appends `/v1/traces` when needed. +There is no OpenTelemetry runtime dependency and no export when the setting is absent +or invalid. Enable it separately for the API and each worker process. + +- **API:** every request exports a SERVER span. A valid inbound `traceparent` + continues the same trace with a new span. Missing or invalid context starts a + new trace. Starters propagate the actual recorded API span in inputs/memo; + Room Agent turns persist that context before enqueue so retries and recovery + do not depend on the bounded in-memory request registry. +- **Workers:** `space-worker`, `space-memory-maintenance-worker`, and + `space-memory-mutation-worker` share `createWorkerTracing`. Workflow execution, + child workflows and every activity attempt export INTERNAL spans. Temporal + headers carry the current workflow scope to activities (including those + without arguments), local activities, child workflows and signals sent by a + workflow. An activity's outbound API call uses its actual recorded span as the + parent. The tracing modules that run in the workflow sandbox import only the + small `@space/contracts/observability` and `@space/contracts/workflows` entry + points, so enabling export does not bring the `@space/contracts` barrel into a + workflow bundle that otherwise avoids it. +- **Long-lived supervisors:** each queued Room Agent mission has its own scope + under its originating API span, with a link to the supervisor run. The + supervisor run serves many requests, so it roots its own trace and only links + to the request that started it. Work that belongs to no single request (the + monitoring polls for background pane work, a stop signal, which carries no + trace context, or a mission queued without context) stays in that supervisor + trace and never joins an earlier request's trace. A paused mission is + re-checked every second, and each check is a short mission span. The run span + itself is exported only when the run ends (completion or continue-as-new), so + while a supervisor is active or idle its mission and poll spans reference a + parent the collector has not received yet. +- **Background continuations** begin a new trace with no parent and carry an + OTLP link that is persisted in `continuedTraceparent`. The first continuation + links to the request span of the operator turn it continues; each later one + links to the preceding background mission span. An item queued by an earlier + version still carries its source request context; it is kept as a link, never + as a parent from another trace. +- **Continue-as-new:** each run has a distinct deterministic span ID, while its + trace remains connected to the preceding run by a parent and an explicit link. +- **Failures:** a Temporal failure other than a cancellation (for example an + `ApplicationFailure`) ends a workflow, signal or update span as `FAILED` with + its safe code (a failed update is rejected and the workflow continues). A + cancellation ends it as `CANCELLED`, and continue-as-new ends a run span as + `CONTINUED_AS_NEW`. Any other error thrown from the workflow + function or a signal or update handler only fails the workflow task, which + Temporal retries while the execution keeps running, so no span is exported for + it. This follows Temporal's default failure policy; Space configures no + `workflowFailureErrorTypes` or `failureExceptionTypes`, which would make such + an error terminal. A mission scope's caller can catch its errors, so an error + leaving a mission callback is exported as a `FAILED` mission span. When a + signal handler's Temporal failure ends the workflow, the signal span is + exported but the run span is not, because the workflow function never returns. +- **Attributes:** bounded workflow/activity IDs and types, Space correlation IDs, + attempt, outcome, duration, activity queue time and allowlisted failure codes. + `space.trace_id` is the application's correlation ID; a distinct request ID uses + `space.request_id`. Only named identifier fields are read. On export a Space + correlation value must match a bounded `:` identifier syntax (for + example `room:V1StGXR8_Z5j`); other or oversized values, such as text with + spaces, file names, paths and URLs, are dropped. The syntax cannot establish + where a value came from, and it cannot remove private content that itself fits + the syntax, so producers must supply generated identifiers. Prompts, + responses, error messages, credentials and arbitrary headers are never read. + Timestamps outside the OTLP range are dropped, and an invalid attempt number + is omitted. API spans retain their existing shape. +- **Rollout:** deploy the API with or before the workers. Turns queued by an + earlier API version keep the parent that version minted without exporting it, + so those missions can show a missing parent in the collector. + +### Replay compatibility and delivery + +The workflow interceptor uses deterministic IDs derived from stable workflow/run +or mission identities and the Temporal workflow clock. It consumes no application +randomness and adds no activities, timers or markers. It sends primitive records +through a host-side sink configured with `callDuringReplay: false`; HTTP, batching +and error handling remain outside the workflow sandbox. No schema migration is +required: old inputs without either optional trace field still parse. + +The integration suite executes real Temporal workflows against a local collector +and replays their captured histories, including an untraced history whose activity +ID was generated with `uuid4()`, and a committed Room Agent supervisor history +recorded by the 1.1.6 workflow code. Replay emits no spans. This proves those histories; +it does not retroactively repair missing context in older runs or replace a +read-only deployment replay against the histories of the target installation. + +Export is best effort: a queue is bounded to 4,096 spans, batches contain up to +256 spans, the flush interval is five seconds, and a POST times out after three +seconds. Full queues and failed exports increment counters without blocking or +retrying business work. Workflow spans export when their execution scope finishes; +a still-running supervisor, process loss or external termination can leave an +unfinished scope. Workflow-task retries can emit the same deterministic span +identity again; replay suppression is not an exactly-once delivery guarantee. ## Redaction policy @@ -290,7 +342,13 @@ No credential or transcript export was added. | `apps/api/tests/observability-metrics.test.ts` | New API metrics, `/metrics` access rule, redaction, CLI disconnect and reconnect counters | | `apps/api/src/__tests__/files-search-errors.test.ts` | `EMPTY_RESULT`/`OK` and each failure code | | `apps/api/tests/files-search-routes.test.ts` | The HTTP error envelope | -| `apps/worker/tests/activity-observability.test.ts` | Interceptor records, outcomes, categories, metrics, loopback endpoint | +| `apps/worker/tests/activity-observability.test.ts` | Interceptor records, headers, outbound parentage, attempt spans, privacy and failure containment | +| `apps/worker/tests/workflow-tracing.test.ts` | Deterministic IDs, concurrent scopes, invalid context, bounded metadata and replay-disabled sink | +| `apps/worker/tests/otlp-worker-integration.test.ts` | Real API, Temporal, child/local/remote activities, retries, both memory queues, continuations, replay, disabled export and slow/failing collectors | +| `apps/worker/tests/otlp-supervisor-integration.test.ts` | The production Room Agent supervisor through the API coordinator: two requests in one W3C trace keep their own API parents, and monitoring polls stay in the supervisor trace | +| `apps/worker/tests/otlp-supervisor-replay.test.ts` | A supervisor history recorded by the 1.1.6 workflow code (completed, stopped and failed missions, background polls) replays on the current code, traced and untraced, without exporting | +| `apps/worker/tests/otlp-task-failure.test.ts` | A non-Temporal error from the workflow function, a signal handler or an update handler fails a task that Temporal retries, with no span exported; Temporal failures export `FAILED` run, signal and update spans | +| `packages/runtime/tests/otlp-span-exporter.test.ts` | Optional valid OTLP links and export after an empty idle flush | | `apps/api/tests/observability-fault-injection.test.ts` | Four injected executor faults and one Temporal start failure, end to end | ## Not covered @@ -337,8 +395,8 @@ No credential or transcript export was added. trace's workflows, joined by workflow id, with their status and error code, and names a failed run's code as the cause when no event carries one. - **Hosts emit no spans.** Since 2026-10-16 the OTLP exporter sends API SERVER - spans and worker activity spans (the worker and both memory workers); the CLI - and Browser Hosts still join traces through log lines only. + spans and worker workflow, mission and activity spans (the worker and both + memory workers); the CLI and Browser Hosts still join traces through log lines only. - **Recovery counters are partial.** Recovery counters exist for plan model retries, CLI terminal reconnects and worker activity retries only. - **Worker endpoints are not scraped yet.** Stable's `deploy/space-worker.service` diff --git a/package-lock.json b/package-lock.json index 4580c1e0b..166f800af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -211,6 +211,8 @@ "undici": "8.10.2" }, "devDependencies": { + "@temporalio/proto": "1.24.0", + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.11" } @@ -3713,6 +3715,25 @@ "node": ">=12.0.0" } }, + "node_modules/@temporalio/testing": { + "version": "1.24.0", + "resolved": "https://registry.npmjs.org/@temporalio/testing/-/testing-1.24.0.tgz", + "integrity": "sha512-XKCuvgb7smKR3oXy5nDgwYSFX/hmg9zTJ2rZQQsVwJzFkga1TbnWDiuT5YMU0INKjoLhnM6c5YA/uLKlp8sdfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@temporalio/activity": "1.24.0", + "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", + "@temporalio/core-bridge": "1.24.0", + "@temporalio/proto": "1.24.0", + "@temporalio/worker": "1.24.0", + "@temporalio/workflow": "1.24.0" + }, + "engines": { + "node": ">= 20.3.0" + } + }, "node_modules/@temporalio/worker": { "version": "1.24.0", "resolved": "https://registry.npmjs.org/@temporalio/worker/-/worker-1.24.0.tgz", diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 4f7d43aa8..0784d81bb 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -13,6 +13,14 @@ "./tool-routing": { "types": "./dist/tool-routing.d.ts", "import": "./dist/tool-routing.js" + }, + "./observability": { + "types": "./dist/observability.d.ts", + "import": "./dist/observability.js" + }, + "./workflows": { + "types": "./dist/workflows.d.ts", + "import": "./dist/workflows.js" } }, "scripts": { diff --git a/packages/contracts/src/schemas.ts b/packages/contracts/src/schemas.ts index 99e9ebfc6..67deb613d 100644 --- a/packages/contracts/src/schemas.ts +++ b/packages/contracts/src/schemas.ts @@ -6774,6 +6774,8 @@ export const dummyTurnInputSchema = z.object({ // Optional W3C trace context of the originating request (additive; see // docs/quality/observability.md). Never part of a workflow id. traceparent: traceparentSchema.optional(), + // Previous mission span for a background continuation that starts a new trace. + continuedTraceparent: traceparentSchema.optional(), // Space Dots Stage 3 (additive): a gateway-enabled native Chat turn. dotId: dotIdSchema.optional(), toolGateway: z.literal("space-dot-tools-v1").optional(), diff --git a/packages/contracts/tests/observability.test.ts b/packages/contracts/tests/observability.test.ts index c7717396d..08e2e1c25 100644 --- a/packages/contracts/tests/observability.test.ts +++ b/packages/contracts/tests/observability.test.ts @@ -92,6 +92,13 @@ describe("additive observability contract fields", () => { expect(dummyTurnInputSchema.safeParse({ ...turn, traceparent: "not-a-trace" }).success).toBe(false); }); + it("preserves an optional continuation link without changing old turn inputs", () => { + const continuedTraceparent = formatTraceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7"); + expect(dummyTurnInputSchema.parse({ ...turn, continuedTraceparent }).continuedTraceparent).toBe(continuedTraceparent); + expect(dummyTurnInputSchema.parse(turn).continuedTraceparent).toBeUndefined(); + expect(dummyTurnInputSchema.safeParse({ ...turn, continuedTraceparent: "invalid" }).success).toBe(false); + }); + it("accepts the old and the new file search response shapes", () => { expect(fileSearchResponseSchema.parse({ query: "a", results: [], totalCount: 0 }).code).toBeUndefined(); expect(fileSearchResponseSchema.parse({ query: "a", results: [], totalCount: 0, code: "EMPTY_RESULT", truncated: false }).code) diff --git a/packages/runtime/src/otlp-span-exporter.ts b/packages/runtime/src/otlp-span-exporter.ts index 86a0edf05..3279e25a1 100644 --- a/packages/runtime/src/otlp-span-exporter.ts +++ b/packages/runtime/src/otlp-span-exporter.ts @@ -4,8 +4,8 @@ * `/v1/traces`). No dependency, no collector required: Space logs stay * joinable by W3C trace id without it. * - * Spans carry only the method, route template, status code, service name and - * Space request id; never URLs with parameters, headers, bodies or user data. + * Callers supply allowlisted route/workflow/activity metadata and correlation + * identifiers; never URLs with parameters, headers, bodies or user data. * Export is best effort: a bounded queue drops (and counts) spans instead of * growing, each POST has a timeout, and a failing collector never affects a * request. @@ -22,6 +22,8 @@ export interface ServerSpan { error: boolean; /** OTLP span kind; SERVER (2) unless given. Worker activities use INTERNAL (1). */ kind?: number; + /** Causal links, such as a prior workflow run or an incoming signal's span. */ + links?: readonly { traceId: string; spanId: string }[]; } export interface SpanExporterStats { @@ -70,24 +72,34 @@ function attributeValue(value: string | number) { return typeof value === "number" && Number.isInteger(value) ? { intValue: String(value) } : { stringValue: String(value) }; } +function otlpLinks(links: ServerSpan["links"]) { + return links?.filter(link => traceIdPattern.test(link.traceId) && spanIdPattern.test(link.spanId) + && !/^0+$/.test(link.traceId) && !/^0+$/.test(link.spanId)) + .map(link => ({ traceId: link.traceId, spanId: link.spanId })); +} + export function otlpTraceRequest(spans: readonly ServerSpan[], serviceName: string) { return { resourceSpans: [{ resource: { attributes: [{ key: "service.name", value: { stringValue: serviceName } }] }, scopeSpans: [{ scope: { name: serviceName }, - spans: spans.map(span => ({ - traceId: span.traceId, - spanId: span.spanId, - ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}), - name: span.name, - kind: span.kind ?? 2, - startTimeUnixNano: span.startTimeUnixNano.toString(), - endTimeUnixNano: span.endTimeUnixNano.toString(), - attributes: Object.entries(span.attributes).map(([key, value]) => ({ key, value: attributeValue(value) })), - // STATUS_CODE_ERROR or STATUS_CODE_UNSET - status: { code: span.error ? 2 : 0 } - })) + spans: spans.map(span => { + const links = otlpLinks(span.links); + return { + traceId: span.traceId, + spanId: span.spanId, + ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}), + name: span.name, + kind: span.kind ?? 2, + startTimeUnixNano: span.startTimeUnixNano.toString(), + endTimeUnixNano: span.endTimeUnixNano.toString(), + attributes: Object.entries(span.attributes).map(([key, value]) => ({ key, value: attributeValue(value) })), + ...(links?.length ? { links } : {}), + // STATUS_CODE_ERROR or STATUS_CODE_UNSET + status: { code: span.error ? 2 : 0 } + }; + }) }] }] }; @@ -122,6 +134,9 @@ export function createOtlpSpanExporter(options: OtlpSpanExporterOptions): SpanEx }; const flush = (): Promise => { if (!inFlight) { + // With an empty queue the async body would finish before the assignment, + // leaving its resolved promise in inFlight and preventing future exports. + if (!queue.length) return Promise.resolve(); inFlight = (async () => { try { while (queue.length) await flushOnce(); diff --git a/packages/runtime/tests/otlp-span-exporter.test.ts b/packages/runtime/tests/otlp-span-exporter.test.ts new file mode 100644 index 000000000..b2d9e35f9 --- /dev/null +++ b/packages/runtime/tests/otlp-span-exporter.test.ts @@ -0,0 +1,76 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createOtlpSpanExporter, type OtlpSpanExporterOptions, type ServerSpan, type SpanExporter } from "../src/otlp-span-exporter.js"; + +const traceId = "4bf92f3577b34da6a3ce929d0e0e4736"; +const parentId = "00f067aa0ba902b7"; + +function span(overrides: Partial = {}): ServerSpan { + return { + traceId, + spanId: "a1b2c3d4e5f60718", + parentSpanId: parentId, + name: "workflow roomAgentMissionWorkflow", + startTimeUnixNano: 1_700_000_000_000_000_000n, + endTimeUnixNano: 1_700_000_000_250_000_000n, + attributes: { "space.outcome": "COMPLETED" }, + error: false, + kind: 1, + ...overrides + }; +} + +const exporters: SpanExporter[] = []; +afterEach(async () => { + await Promise.all(exporters.splice(0).map(exporter => exporter.close())); +}); + +function exporter(fetchImpl: NonNullable) { + const created = createOtlpSpanExporter({ + endpoint: "http://127.0.0.1:4318/", + serviceName: "space-worker", + fetch: fetchImpl, + flushIntervalMs: 60_000 + }); + exporters.push(created); + return created; +} + +describe("shared OTLP span exporter", () => { + it("exports causal links as OTLP identifiers and leaves unlinked spans unchanged", async () => { + const fetchImpl = vi.fn>(async () => ({ ok: true })); + const created = exporter(fetchImpl); + const link = { traceId: "11111111111111111111111111111111", spanId: "2222222222222222" }; + created.record(span({ links: [link, { traceId: "private prompt", spanId: parentId }, + { traceId, spanId: "0000000000000000" }] })); + created.record(span({ links: [] })); + created.record(span({ links: [{ traceId: "00000000000000000000000000000000", spanId: parentId }] })); + created.record(span()); + await created.flush(); + + const [, init] = fetchImpl.mock.calls[0]!; + const [linked, empty, invalid, unlinked] = JSON.parse(init.body).resourceSpans[0].scopeSpans[0].spans; + expect(linked.links).toEqual([link]); + expect(empty).toEqual(unlinked); + expect(invalid).toEqual(unlinked); + expect(unlinked).not.toHaveProperty("links"); + expect(init.body).not.toContain("private prompt"); + }); + + it("continues exporting after empty initial and later flushes", async () => { + const fetchImpl = vi.fn>(async () => ({ ok: true })); + const created = exporter(fetchImpl); + await created.flush(); + expect(fetchImpl).not.toHaveBeenCalled(); + + created.record(span()); + await created.flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(created.stats()).toEqual({ exported: 1, dropped: 0, failed: 0 }); + + await created.flush(); + created.record(span({ spanId: "2222222222222222" })); + await created.flush(); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(created.stats()).toEqual({ exported: 2, dropped: 0, failed: 0 }); + }); +}); From fe9989be77c1790ea76966c7541a7be7d8206023 Mon Sep 17 00:00:00 2001 From: New2thisshit <96698527+New2thisshit@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:23:28 +0200 Subject: [PATCH 2/3] fix(observability): export a paused background mission once; document the base endpoint A background continuation keeps a stable span identity so the next continuation can link to it. A paused item is requeued and re-checked every second under that same identity, so each check exported another span with the same trace and span IDs. Export it once, from the pass that finishes the mission, starting at its first check. Ordinary missions are unchanged. The exporter always appends /v1/traces; the docs now say to configure the collector's base URL instead of claiming a full traces URL is used directly. Both found by the Codex review on #221. Co-Authored-By: Claude Opus 5.5 --- apps/worker/src/workflow-tracing.ts | 23 ++++++++++++++--- apps/worker/src/workflows.ts | 3 ++- apps/worker/tests/workflow-tracing.test.ts | 30 ++++++++++++++++++++++ docs/configuration-reference.md | 5 ++-- docs/quality/observability.md | 7 +++-- 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/apps/worker/src/workflow-tracing.ts b/apps/worker/src/workflow-tracing.ts index 7edb84e9a..6e00da9b1 100644 --- a/apps/worker/src/workflow-tracing.ts +++ b/apps/worker/src/workflow-tracing.ts @@ -41,6 +41,8 @@ interface TraceScope { operation?: WorkflowSpanRecord["operation"]; attributes: Record; links: TraceSpanLink[]; + /** A background mission's span identity is stable, so a paused pass must not export it. */ + stableIdentity?: boolean; } const scopes = new AsyncLocalStorage(); @@ -48,6 +50,8 @@ const { spaceTracing } = proxySinks(); let runScope: TraceScope | null = null; let runInfo: WorkflowInfo | null = null; let scopeSequence = 0; +// First start of each stable background span still waiting to be exported in this run. +let pendingStableStarts = new Map(); /** * A supervisor serves many requests. Its run roots its own trace and only @@ -110,6 +114,7 @@ async function observe(scope: TraceScope, execute: () => Promise, sdkEntry let reasonCode: string | undefined; let invoked = false; let taskFailure = false; + let paused = false; try { let result: T; try { @@ -131,6 +136,8 @@ async function observe(scope: TraceScope, execute: () => Promise, sdkEntry } else if (status === "CANCELLED" || status === "INTERRUPTED") { outcome = "CANCELLED"; reasonCode = "WORKFLOW_CANCELLED"; + } else if (status === "PAUSED") { + paused = true; } } } catch {} @@ -155,7 +162,12 @@ async function observe(scope: TraceScope, execute: () => Promise, sdkEntry } throw error; } finally { - if (!taskFailure) record(scope, outcome, reasonCode); + // A paused background mission is requeued under the same stable span identity: + // export that span once, from the pass that finishes the mission. + if (!taskFailure && !(paused && scope.stableIdentity)) { + record(scope, outcome, reasonCode); + if (scope.stableIdentity) pendingStableStarts.delete(scope.context.parentId); + } } } @@ -178,13 +190,17 @@ function makeOperationScope( parentId: deterministicWorkflowSpanId(runInfo.runId, `${operation}:${scopeSequence}`), flags: parent?.flags ?? continued?.flags ?? "01" }; + // A stable background span covers every pass of its mission in this run. + const startTimeMs = (background && pendingStableStarts.get(context.parentId)) || Date.now(); + if (background) pendingStableStarts.set(context.parentId, startTimeMs); const scope: TraceScope = { context, parentSpanId: parent?.parentId ?? null, - startTimeMs: Date.now(), + startTimeMs, operation, attributes: workflowTraceAttributes(args), - links: [] + links: [], + stableIdentity: background !== null }; addLink(scope, continued); if (background) addLink(scope, incoming); @@ -220,6 +236,7 @@ export function interceptors(): WorkflowInterceptors { runScope = null; runInfo = null; scopeSequence = 0; + pendingStableStarts = new Map(); return { inbound: [{ async execute(input, next) { diff --git a/apps/worker/src/workflows.ts b/apps/worker/src/workflows.ts index b668e8c0a..4a1181b84 100644 --- a/apps/worker/src/workflows.ts +++ b/apps/worker/src/workflows.ts @@ -334,7 +334,8 @@ async function continuingRoomAgentSupervisorWorkflow(rawInput: RoomAgentSupervis try { return await turnScope.run(async () => { const state = await supervisorActivities.checkRoomAgentSupervisorTurn(item); - if (state === "PAUSED") { paused = true; return; } + // The trace scope exports a paused background pass only once the mission finishes. + if (state === "PAUSED") { paused = true; return { status: "PAUSED" as const }; } if (state === "SKIP") { await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason: "Turn is no longer active or was superseded." }); return; diff --git a/apps/worker/tests/workflow-tracing.test.ts b/apps/worker/tests/workflow-tracing.test.ts index 77f17b472..0a91d0c68 100644 --- a/apps/worker/tests/workflow-tracing.test.ts +++ b/apps/worker/tests/workflow-tracing.test.ts @@ -156,6 +156,36 @@ describe("workflow tracing scopes", () => { expect(missions[1]!.traceId).not.toBe(missions[0]!.traceId); }); + it("exports a paused background mission once under its stable span, from its first pass to its finish", async () => { + const tracing = interceptors(); + const background = { traceId: "trace:bg-paused", agentRunId: "agent_run:bg-paused", continuedTraceparent: traceparent }; + const stable = backgroundMissionTraceContext(background)!; + const ordinary = { traceparent, missionId: "room_agent_mission:ordinary" }; + const now = vi.spyOn(Date, "now"); + try { + await tracing.inbound![0]!.execute!({ args: [{ traceparent }], headers: {} }, async () => { + now.mockReturnValue(5_000); + await withWorkflowTraceContext(background, async () => ({ status: "PAUSED" as const })); + await withWorkflowTraceContext(ordinary, async () => ({ status: "PAUSED" as const })); + now.mockReturnValue(6_000); + await withWorkflowTraceContext(background, async () => ({ status: "PAUSED" as const })); + await withWorkflowTraceContext(ordinary, async () => ({ status: "PAUSED" as const })); + now.mockReturnValue(7_000); + await withWorkflowTraceContext(background, async () => ({ status: "COMPLETED" as const })); + }); + } finally { + now.mockRestore(); + } + const missions = context.spans.filter((span) => span.operation === "mission"); + const stableSpans = missions.filter((span) => span.spanId === stable.parentId); + expect(stableSpans).toHaveLength(1); + expect(stableSpans[0]).toMatchObject({ traceId: stable.traceId, startTimeMs: 5_000, endTimeMs: 7_000, outcome: "COMPLETED" }); + // Ordinary missions keep one short span per pass, each with its own identity. + const ordinarySpans = missions.filter((span) => span.spanId !== stable.parentId); + expect(ordinarySpans).toHaveLength(2); + expect(new Set(ordinarySpans.map((span) => span.spanId)).size).toBe(2); + }); + it("keeps returned failures and thrown errors unchanged and never records their messages", async () => { const tracing = interceptors(); // A workflow fails only with a Temporal failure; a plain Error fails (and retries) the task. diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index 76ba6bfdf..5cc3c5d10 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -57,8 +57,9 @@ routes set the modules they keep, for example Set `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` to the collector's HTTP(S) base URL, for example `http://127.0.0.1:4318`, in the API and worker processes that should export -traces. The exporter sends OTLP/HTTP JSON to `/v1/traces`; a URL already ending in -that path is used directly. An unset or invalid endpoint disables export. +traces. The exporter always appends `/v1/traces` and sends OTLP/HTTP JSON there, +so give the base URL, not `.../v1/traces`. An unset or invalid endpoint disables +export. The API exports request spans. The main worker and both memory workers export workflow, child workflow, mission and activity-attempt spans, with propagation diff --git a/docs/quality/observability.md b/docs/quality/observability.md index 8d7a3d782..178f6b26d 100644 --- a/docs/quality/observability.md +++ b/docs/quality/observability.md @@ -217,7 +217,8 @@ Without the endpoint, every activity still writes `activity.started` and The trace context is local by default. Set `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` to an HTTP(S) collector base URL (for example `http://127.0.0.1:4318`) to enable -OTLP/HTTP JSON export. The shared runtime exporter appends `/v1/traces` when needed. +OTLP/HTTP JSON export. The shared runtime exporter always appends `/v1/traces`, so +the setting is the base URL, not the full traces path. There is no OpenTelemetry runtime dependency and no export when the setting is absent or invalid. Enable it separately for the API and each worker process. @@ -243,7 +244,9 @@ or invalid. Enable it separately for the API and each worker process. monitoring polls for background pane work, a stop signal, which carries no trace context, or a mission queued without context) stays in that supervisor trace and never joins an earlier request's trace. A paused mission is - re-checked every second, and each check is a short mission span. The run span + re-checked every second. Each check of an ordinary mission is a short mission + span; a background continuation keeps one stable span identity, so it is + exported once, from its first check until it finishes. The run span itself is exported only when the run ends (completion or continue-as-new), so while a supervisor is active or idle its mission and poll spans reference a parent the collector has not received yet. From 9379c76f07a8c8f4d53b959ac3667f319d519044 Mon Sep 17 00:00:00 2001 From: New2thisshit <96698527+New2thisshit@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:35:40 +0200 Subject: [PATCH 3/3] fix(observability): close the span exporter when a worker fails to start The main and memory-mutation workers create the OTLP exporter before Worker.create, but their try/finally only wraps worker.run(), so a rejected Worker.create skipped close(). Close the exporter on that path and rethrow. The metrics server never throws (it resolves null), and the maintenance worker already closes the exporter in a finally that covers Worker.create. Found by the CodeRabbit review on #221. Co-Authored-By: Claude Opus 5.5 --- apps/worker/src/memory-mutation-worker.ts | 4 ++++ apps/worker/src/worker.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/apps/worker/src/memory-mutation-worker.ts b/apps/worker/src/memory-mutation-worker.ts index 221ce001a..5814db1f7 100644 --- a/apps/worker/src/memory-mutation-worker.ts +++ b/apps/worker/src/memory-mutation-worker.ts @@ -24,6 +24,10 @@ if (!memoryMutationsEnabled(process.env)) { activities: { executePersistedMemoryChangeSet }, ...memoryMutationWorkerOptions, ...createWorkerTracing(spans) + }).catch(async (error: unknown) => { + // The run's finally below closes the exporter; a worker that never starts closes it here. + await spans?.close(); + throw error; }); console.log(JSON.stringify({ service, status: "running", address, namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE })); const stopReloadWatch = reloadWhenServicesCommitChanges(worker, { log: (message) => console.log(JSON.stringify({ service, status: "reloading", message })) }); diff --git a/apps/worker/src/worker.ts b/apps/worker/src/worker.ts index 680927e4a..0ebc2e309 100644 --- a/apps/worker/src/worker.ts +++ b/apps/worker/src/worker.ts @@ -46,6 +46,10 @@ const worker = await Worker.create({ // Structured, correlated activity records and worker metrics. Activity-side // only: the workflow sandbox and its replay behaviour are unchanged. ...createWorkerTracing(spans) +}).catch(async (error: unknown) => { + // The run's finally below closes the exporter; a worker that never starts closes it here. + await spans?.close(); + throw error; }); // Optional loopback-only Prometheus endpoint for worker metrics.