diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 3c9d738d8ba9..d2cdc89bafcf 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -100,6 +100,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-owner-verdict.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-sink-release.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-held-child-lease-renewal.test.ts', @@ -112,6 +113,23 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-lease-renewer.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-main-agent-working-agreement.test.ts', @@ -153,6 +171,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-clear-rewind.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts', @@ -200,6 +219,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/runtime/agent-session-conversation-clear-commit.test.ts', 'src/main/runtime/agent-session-conversation-name-store.test.ts', 'src/main/runtime/agent-session-death-evidence-persistence.test.ts', + 'src/main/runtime/agent-session-generation-end.test.ts', 'src/main/runtime/agent-session-launch-directory.test.ts', 'src/main/runtime/agent-session-launch-env-admission.test.ts', 'src/main/runtime/agent-session-lease-renewal.test.ts', diff --git a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx index b01a0fa66c18..24f78f532ba2 100644 --- a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx +++ b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx @@ -4,7 +4,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionQueuePause } from '../../../src/shared/agent-session-wire' import type { ActionSheetAction } from '../components/ActionSheetModal' import { MobileNativeChatQueuedMessages } from './MobileNativeChatQueuedMessages' -import type { MobileQueuedMessageCard } from './mobile-structured-queued-message-cards' +import { + mobileQueuedMessageCards, + type MobileQueuedMessageCard +} from './mobile-structured-queued-message-cards' vi.mock('react-native', () => ({ Pressable: 'Pressable', @@ -209,6 +212,23 @@ describe('MobileNativeChatQueuedMessages', () => { expect(nodeTypes(rows[1]!)).toContain('CornerDownRight') }) + it('a waiting card still reads Steer while nothing runs, as when its agent died and its cleanup waits', async () => { + const [waiting] = mobileQueuedMessageCards( + [ + { + messageId: 'waiting', + position: 1, + state: 'waiting', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'next' }] } + } + ], + [], + { agentWorking: false } + ) + const mounted = await mount({ cards: [waiting!], onSend: vi.fn(async () => true) }) + expect(texts(mounted).filter((text) => text === 'Send' || text === 'Steer')).toEqual(['Steer']) + }) + it('a command card never steers: Send only while the agent is idle, and no menu', async () => { const onSend = vi.fn(async () => true) const mounted = await mount({ diff --git a/mobile/src/session/mobile-structured-actionable-prompt.test.ts b/mobile/src/session/mobile-structured-actionable-prompt.test.ts new file mode 100644 index 000000000000..c170bc0e50ca --- /dev/null +++ b/mobile/src/session/mobile-structured-actionable-prompt.test.ts @@ -0,0 +1,66 @@ +// The phone answers only the prompts the host says still wait on the person: one an agent that +// has ended raised gets no card, and an older host, which names none, keeps today's rule. + +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from '../../../src/shared/agent-session-journal-types' +import { + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' + +const PENDING = { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null +} + +function approval(itemId: string, sequence: number): AgentJournalRenderItem { + return { + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: PENDING + } + } +} + +function question(itemId: string, sequence: number): AgentJournalRenderItem { + return { + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'question', + question: 'Which one?', + options: [{ id: 'a', label: 'A' }], + resolution: PENDING + } + } +} + +const ITEMS = [approval('dead-approval', 1), question('dead-question', 2)] + +describe('the prompt a phone answers', () => { + it('skips one the host does not name, and answers one it does', () => { + const items = [...ITEMS, approval('live-approval', 3)] + expect( + actionableStructuredApproval({ items, actionablePromptIds: ['live-approval'] })?.itemId + ).toBe('live-approval') + expect( + actionableStructuredQuestion({ items, actionablePromptIds: ['live-approval'] }) + ).toBeNull() + }) + + it('is the first pending one when an older host names none', () => { + expect(actionableStructuredApproval({ items: ITEMS })?.itemId).toBe('dead-approval') + expect(actionableStructuredQuestion({ items: ITEMS })?.itemId).toBe('dead-question') + }) +}) diff --git a/mobile/src/session/mobile-structured-actionable-prompt.ts b/mobile/src/session/mobile-structured-actionable-prompt.ts new file mode 100644 index 000000000000..478927a9088e --- /dev/null +++ b/mobile/src/session/mobile-structured-actionable-prompt.ts @@ -0,0 +1,33 @@ +import { isActionableStructuredAgentSessionPrompt } from '../../../src/shared/structured-agent-session-live-turn' +import type { StructuredAgentSessionState } from '../../../src/shared/structured-agent-session-reducer' +import { + pendingStructuredApproval, + pendingStructuredQuestion, + type StructuredApprovalItem, + type StructuredQuestionItem +} from './mobile-structured-agent-prompts' + +type PromptState = Pick + +/** The approval a person can answer: pending, and named by the host as waiting when it names them + * (`isActionableStructuredAgentSessionPrompt`): one an agent that ended raised is not. */ +export function actionableStructuredApproval(state: PromptState): StructuredApprovalItem | null { + return ( + state.items.find( + (item): item is StructuredApprovalItem => + pendingStructuredApproval(item) && + isActionableStructuredAgentSessionPrompt(item.itemId, state.actionablePromptIds) + ) ?? null + ) +} + +/** The question a person can answer, by the same rule. */ +export function actionableStructuredQuestion(state: PromptState): StructuredQuestionItem | null { + return ( + state.items.find( + (item): item is StructuredQuestionItem => + pendingStructuredQuestion(item) && + isActionableStructuredAgentSessionPrompt(item.itemId, state.actionablePromptIds) + ) ?? null + ) +} diff --git a/mobile/src/session/mobile-structured-agent-session-cancel.ts b/mobile/src/session/mobile-structured-agent-session-cancel.ts index 49232afeb326..457a960c26cc 100644 --- a/mobile/src/session/mobile-structured-agent-session-cancel.ts +++ b/mobile/src/session/mobile-structured-agent-session-cancel.ts @@ -1,7 +1,9 @@ import type { AgentSessionCancelResult } from '../../../src/shared/agent-session-wire' -import type { AgentJournalRenderItem } from '../../../src/shared/agent-session-journal-types' import type { StructuredAgentSessionState } from '../../../src/shared/structured-agent-session-reducer' -import { runningStructuredAgentSessionTurnId } from '../../../src/shared/structured-agent-session-live-turn' +import { + isActionableStructuredAgentSessionPrompt, + runningStructuredAgentSessionTurnId +} from '../../../src/shared/structured-agent-session-live-turn' import type { RpcClient } from '../transport/rpc-client' import { requestStructuredAgentSessionMutation, @@ -10,12 +12,14 @@ import { type PromptIdentity = { itemId: string; expectedRevision: number } +/** The prompt a Stop cancels: one the host says still waits on the person, when it says. */ export function pendingStructuredPromptIdentity( - items: readonly AgentJournalRenderItem[] + state: Pick ): PromptIdentity | undefined { - const prompt = items.find((item) => + const prompt = state.items.find((item) => item.body.kind === 'approval' || item.body.kind === 'question' - ? item.body.resolution.state === 'pending' + ? item.body.resolution.state === 'pending' && + isActionableStructuredAgentSessionPrompt(item.itemId, state.actionablePromptIds) : false ) return prompt ? { itemId: prompt.itemId, expectedRevision: prompt.revision } : undefined diff --git a/mobile/src/session/mobile-structured-agent-session-rpc.ts b/mobile/src/session/mobile-structured-agent-session-rpc.ts index 7b2da967d56c..4802042435b1 100644 --- a/mobile/src/session/mobile-structured-agent-session-rpc.ts +++ b/mobile/src/session/mobile-structured-agent-session-rpc.ts @@ -127,6 +127,8 @@ export async function requestStructuredAgentSessionMutation(args: { fields: Record clientOperationId?: string timeoutMs?: number + /** The chat's agent, which a refusal about it names. */ + agentName?: string }): Promise> { const { client, @@ -136,8 +138,10 @@ export async function requestStructuredAgentSessionMutation(args: { expectedRuntimeFence, fields, clientOperationId, - timeoutMs + timeoutMs, + agentName } = args + const context = agentName ? { agentName } : {} try { const result = await callAgentSession>( client, @@ -171,7 +175,8 @@ export async function requestStructuredAgentSessionMutation(args: { code: result.refusal.code, message: agentSessionRefusalNotice( result.refusal, - phoneWriteKind(fingerprintMethod, fields) + phoneWriteKind(fingerprintMethod, fields), + context ) } } catch (error) { @@ -184,7 +189,7 @@ export async function requestStructuredAgentSessionMutation(args: { return { status: 'failed', message: agentSessionWriteNoticeEnglish( - agentSessionWriteNoticeParts(answered, phoneWriteKind(fingerprintMethod, fields)) + agentSessionWriteNoticeParts(answered, phoneWriteKind(fingerprintMethod, fields), context) ) } } diff --git a/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts b/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts index ed93bda5924a..f75782b657aa 100644 --- a/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts +++ b/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts @@ -145,7 +145,8 @@ export function useMobileNativeChatTurnDisclosure({ rows, turnJournal?.items, stopping, - turnJournal?.submissions + turnJournal?.submissions, + turnJournal ?? {} ) : null if (!ids?.size) { diff --git a/mobile/src/session/use-mobile-structured-agent-mutation.ts b/mobile/src/session/use-mobile-structured-agent-mutation.ts index 4eb8a203121b..d8a6a30fb7bf 100644 --- a/mobile/src/session/use-mobile-structured-agent-mutation.ts +++ b/mobile/src/session/use-mobile-structured-agent-mutation.ts @@ -22,8 +22,10 @@ export function useMobileStructuredAgentMutate(args: { enabled: boolean stateRef: { readonly current: StructuredAgentSessionState } onSendError: (message: string) => void + /** The chat's agent, which a refusal about it names. */ + agentName?: string }): MobileStructuredAgentMutate { - const { client, enabled, onSendError, sessionId, stateRef } = args + const { agentName, client, enabled, onSendError, sessionId, stateRef } = args return useCallback( async ( method: string, @@ -41,7 +43,8 @@ export function useMobileStructuredAgentMutate(args: { fingerprintMethod, sessionId, expectedRuntimeFence: targetFence, - fields + fields, + ...(agentName ? { agentName } : {}) }) if (result.status === 'accepted') { return { @@ -56,6 +59,6 @@ export function useMobileStructuredAgentMutate(args: { onSendError(result.message) return { status: 'rejected' } }, - [client, enabled, onSendError, sessionId, stateRef] + [agentName, client, enabled, onSendError, sessionId, stateRef] ) } diff --git a/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx b/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx index 7a14895fd66e..7b26f7003784 100644 --- a/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx +++ b/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx @@ -266,6 +266,22 @@ describe('mobile structured prompt cancellation', () => { expect(hook.turnId).toBeNull() }) + it("reads Working from the host's answer, and from its rows only when an older host gives none", () => { + // A turn row an agent that has since ended left running: the host says it is not working. + state = { ...state, items: [runningTurn()], working: false } + act(() => { + renderer = create(createElement(Harness, { promptCancelSupported: true })) + }) + expect(hook.isWorking).toBe(false) + act(() => renderer?.unmount()) + const { working: _answered, ...olderHost } = state + state = olderHost + act(() => { + renderer = create(createElement(Harness, { promptCancelSupported: true })) + }) + expect(hook.isWorking).toBe(true) + }) + it('uses the rendered prompt identity when the journal changes before tap', async () => { act(() => { renderer = create(createElement(Harness, { promptCancelSupported: true })) diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index ceb172306486..c0e4512a81d2 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -7,12 +7,15 @@ import { isStructuredAgentSessionMainAgentWorking } from '../../../src/shared/st import { isFinalAgentSessionReadRefusal } from '../../../src/shared/structured-agent-session-read-refusal' import { isStructuredAgentSessionThinking, - runningStructuredAgentSessionTurnId + runningStructuredAgentSessionTurnId, + structuredAgentSessionHostSaysWorking } from '../../../src/shared/structured-agent-session-live-turn' import { selectStructuredAgentTurnActivity } from '../../../src/shared/native-chat-turn-activity' import { - pendingStructuredApproval, - pendingStructuredQuestion, + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' +import { projectStructuredPermission, projectStructuredQuestion } from './mobile-structured-agent-prompts' @@ -88,7 +91,8 @@ export function useMobileStructuredAgentSession(args: { sessionId, enabled, stateRef, - onSendError + onSendError, + ...(agent ? { agentName: tuiAgentDisplayName(agent) ?? agent } : {}) }) const options = useMobileStructuredAgentOptions({ @@ -157,7 +161,10 @@ export function useMobileStructuredAgentSession(args: { const activityText = selectStructuredAgentTurnActivity(state.items, turnId, state.activity)?.text ?? null const thinking = isStructuredAgentSessionThinking(state) - const isWorking = isStructuredAgentSessionMainAgentWorking(turnId, state.submissions, state.fence) + // The host's answer counts only the agent running now; an older host gives none. + const isWorking = structuredAgentSessionHostSaysWorking(state.working, () => + isStructuredAgentSessionMainAgentWorking(turnId, state.submissions, state.fence) + ) const backgroundTasks = useMobileStructuredBackgroundTasks({ sessionKey, state, @@ -196,13 +203,22 @@ export function useMobileStructuredAgentSession(args: { [thinking, activityText, stopping, stopRequestInFlight, queueCapable, state.items] ) const status = state.status === 'idle' ? 'idle' : state.status + // The host's answer where it gives one: a prompt an agent that ended raised takes no answer. const approvalPrompt = useMemo( - () => state.items.find(pendingStructuredApproval) ?? null, - [state.items] + () => + actionableStructuredApproval({ + items: state.items, + actionablePromptIds: state.actionablePromptIds + }), + [state.items, state.actionablePromptIds] ) const questionPrompt = useMemo( - () => state.items.find(pendingStructuredQuestion) ?? null, - [state.items] + () => + actionableStructuredQuestion({ + items: state.items, + actionablePromptIds: state.actionablePromptIds + }), + [state.items, state.actionablePromptIds] ) // What a refused command's line on the phone stands on. const commandRefusalCauses = useMemo( @@ -219,7 +235,7 @@ export function useMobileStructuredAgentSession(args: { queuedMessages, queuePause, submissions: state.submissions, - agentWorking: isStructuredAgentSessionMainAgentWorking(turnId, state.submissions, state.fence), + agentWorking: isWorking, mutate, appendComposerText, onSendError, @@ -278,7 +294,7 @@ export function useMobileStructuredAgentSession(args: { void stopPress.track(() => requestCancel()) }, cancelPrompt: (prompt?: { itemId: string; expectedRevision: number }) => - requestCancel(prompt ?? pendingStructuredPromptIdentity(stateRef.current.items)), + requestCancel(prompt ?? pendingStructuredPromptIdentity(stateRef.current)), permission: projectStructuredPermission(approvalPrompt), question: projectStructuredQuestion(questionPrompt, groupedDraft), respondPermission, diff --git a/mobile/src/session/use-mobile-structured-agent-turn-timing.ts b/mobile/src/session/use-mobile-structured-agent-turn-timing.ts index a701cdbc21d1..d23bb936c545 100644 --- a/mobile/src/session/use-mobile-structured-agent-turn-timing.ts +++ b/mobile/src/session/use-mobile-structured-agent-turn-timing.ts @@ -21,11 +21,14 @@ export function useMobileStructuredAgentTurnTiming( items, submissions, latestTurn, + working, hostClock }: { items: readonly AgentJournalRenderItem[] submissions: readonly AgentJournalSubmission[] latestTurn?: AgentSessionLatestTurn | null + /** The host's working answer, which places rows waiting behind the live turn. */ + working?: boolean hostClock?: StructuredAgentHostClock | null }, turnId: string | null @@ -40,8 +43,8 @@ export function useMobileStructuredAgentTurnTiming( [items, submissions, turnId, latestTurn] ) const turnJournal = useMemo( - () => ({ items, submissions, latestTurn }), - [items, submissions, latestTurn] + () => ({ items, submissions, latestTurn, ...(working !== undefined ? { working } : {}) }), + [items, submissions, latestTurn, working] ) const [latch, setLatch] = useState(null) // Stamp during render (React's derive-from-props pattern) so the first paint of diff --git a/mobile/src/session/use-mobile-structured-prompt-responses.ts b/mobile/src/session/use-mobile-structured-prompt-responses.ts index 093029b2be52..eb2a0b38caa1 100644 --- a/mobile/src/session/use-mobile-structured-prompt-responses.ts +++ b/mobile/src/session/use-mobile-structured-prompt-responses.ts @@ -8,8 +8,10 @@ import { } from '../../../src/shared/agent-session-question-answer' import type { StructuredAgentSessionState } from '../../../src/shared/structured-agent-session-reducer' import { - pendingStructuredApproval, - pendingStructuredQuestion, + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' +import { structuredApprovalResponseTarget, structuredQuestionResponseTarget } from './mobile-structured-agent-prompts' @@ -54,7 +56,7 @@ export function useMobileStructuredPromptResponses(args: { async (optionId: string): Promise => { const target = structuredApprovalResponseTarget( optionId, - stateRef.current.items.find(pendingStructuredApproval) ?? null + actionableStructuredApproval(stateRef.current) ) if (!target) { return false @@ -107,7 +109,7 @@ export function useMobileStructuredPromptResponses(args: { const respondQuestion = useCallback( async (answer: string): Promise => { - const prompt = stateRef.current.items.find(pendingStructuredQuestion) ?? null + const prompt = actionableStructuredQuestion(stateRef.current) if (prompt?.body.questions) { const promptKey = groupedQuestionPromptKey(prompt.itemId, prompt.revision) const grouped = advanceGroupedQuestion({ diff --git a/mobile/src/session/use-mobile-structured-send-with-outcome.ts b/mobile/src/session/use-mobile-structured-send-with-outcome.ts index c8fd8a74b884..16106fc5bbbd 100644 --- a/mobile/src/session/use-mobile-structured-send-with-outcome.ts +++ b/mobile/src/session/use-mobile-structured-send-with-outcome.ts @@ -19,9 +19,9 @@ import type { MobileNativeChatSendErrorReporter } from './use-mobile-native-chat import { sendMobileStructuredAgentSessionMessage } from './mobile-structured-agent-session-send' import { timeoutForDeadline } from './mobile-structured-agent-session-rpc' import { - pendingStructuredApproval, - pendingStructuredQuestion -} from './mobile-structured-agent-prompts' + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' /** Whether a send made now asks the host to queue it. The host's queue waits on any pending prompt; * one this build cannot answer would hold the send forever, so it starts a turn instead. */ @@ -100,9 +100,8 @@ export function useMobileStructuredSendWithOutcome(args: { pending: commandPending, controller: { agent: agent ?? undefined, ...controller }, busy: () => - stateRef.current.items.some( - (item) => pendingStructuredApproval(item) || pendingStructuredQuestion(item) - ) + actionableStructuredApproval(stateRef.current) || + actionableStructuredQuestion(stateRef.current) ? 'prompt' : runningStructuredAgentSessionTurnId(stateRef.current) ? 'working' diff --git a/src/main/acp/acp-structured-host-crash.test.ts b/src/main/acp/acp-structured-host-crash.test.ts index 53465c822765..39032c7a86c6 100644 --- a/src/main/acp/acp-structured-host-crash.test.ts +++ b/src/main/acp/acp-structured-host-crash.test.ts @@ -38,6 +38,21 @@ async function midReply() { batches.push(batch) return append(batch) }) + // A settlement plans its batch at its own turn in the write queue: recorded as it is planned, and + // only when it writes something (an empty plan writes no row). + const appendPlanned = journal.appendPlannedLifecycleBatch + vi.spyOn(journal, 'appendPlannedLifecycleBatch').mockImplementation((input) => + appendPlanned({ + ...input, + plan: () => { + const planned = input.plan() + if (planned.mutations.length > 0 || planned.dispatches.length > 0) { + batches.push({ ...input, mutations: planned.mutations }) + } + return planned + } + }) + ) child.stderr = LAST_WORDS return { ...rig, child, batches } } diff --git a/src/main/acp/acp-structured-host-restore-failed.test.ts b/src/main/acp/acp-structured-host-restore-failed.test.ts index bf6c7a363c8d..75ea27a91e67 100644 --- a/src/main/acp/acp-structured-host-restore-failed.test.ts +++ b/src/main/acp/acp-structured-host-restore-failed.test.ts @@ -225,6 +225,7 @@ describe('the warning row of a replacement whose attach failed', () => { expect(await host.attach(CALLER, attachParams())).toMatchObject({ ok: true }) await host.close(SESSION, 'user-close') // The journal's open fails after the fresh session's link is durable, before its row is written. + // A start settles nothing, so the attach's own open is the only one. vi.spyOn(AgentSessionJournal.prototype, 'open').mockRejectedValueOnce( new Error('journal path unavailable') ) diff --git a/src/main/acp/acp-timeline-grok-subagent-retention.test.ts b/src/main/acp/acp-timeline-grok-subagent-retention.test.ts index edacc93ab735..44065c2843b5 100644 --- a/src/main/acp/acp-timeline-grok-subagent-retention.test.ts +++ b/src/main/acp/acp-timeline-grok-subagent-retention.test.ts @@ -153,12 +153,20 @@ describe('Grok roster ownership after more than 32 spawning groups', () => { { id: 'old', label: 'Original label', + ownerFence: 1, startedAt: 2, settledAt: 301, state: 'completed', tokens: 12 }, - { id: 'sibling', label: 'Original label 2', startedAt: 3, settledAt: 4, state: 'completed' } + { + id: 'sibling', + label: 'Original label 2', + ownerFence: 1, + startedAt: 3, + settledAt: 4, + state: 'completed' + } ]) expect( (await roster(fixture)).flatMap(({ group }) => @@ -192,6 +200,7 @@ describe('Grok roster ownership after more than 32 spawning groups', () => { { id: 'old', label: 'Original label', + ownerFence: 1, startedAt: 2, settledAt: 301, state: kind === 'output' ? 'completed' : 'stopped' diff --git a/src/main/acp/acp-timeline-grok-subagents.test.ts b/src/main/acp/acp-timeline-grok-subagents.test.ts index 2acd725cf284..f3d7adadb158 100644 --- a/src/main/acp/acp-timeline-grok-subagents.test.ts +++ b/src/main/acp/acp-timeline-grok-subagents.test.ts @@ -62,6 +62,7 @@ describe('Grok subagents through the shared timeline', () => { { id: 'subagent-1', label: 'Count note lines', + ownerFence: 1, state: 'completed', tokens: 13778, startedAt: expect.any(Number), diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index 1a761fda810d..767092e491c6 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -102,12 +102,15 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { // A real journal's sequence only ever advances, so the feed's projection // cache must miss on every publish here: this test is about the rename. let sequence = 0 - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot(), the provenance reads and runningTurn() of a journal. const journal = { snapshot: () => ({ items, submissions: [] }), stopMarks: { latest: () => null, revision: () => 0 }, lastActivityAt: () => 1, - cursor: () => ({ epoch: 1, sequence: (sequence += 1) }) + cursor: () => ({ epoch: 1, sequence: (sequence += 1) }), + itemFence: () => undefined, + wroteBeforeOpen: () => false, + runningTurn: () => null } as unknown as AgentSessionJournal const pending: Promise[] = [] const observe = vi.fn((summary, options) => { @@ -191,7 +194,7 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { const { deps, setDisplayName, setRenameError } = makeDeps({ getRepo: () => ({ id: REPO_ID, kind: 'folder', path: '/workspace/platform' }) as Repo }) - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot(), the provenance reads and runningTurn() of a journal. const journal = { lastActivityAt: () => 1, stopMarks: { latest: () => null, revision: () => 0 }, @@ -208,7 +211,10 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { } ], submissions: [] - }) + }), + itemFence: () => undefined, + wroteBeforeOpen: () => false, + runningTurn: () => null } as unknown as AgentSessionJournal const location = { executionHostId: 'local' as const, diff --git a/src/main/claude/claude-background-stop-records.test.ts b/src/main/claude/claude-background-stop-records.test.ts index 0092d9e2f29d..59da29795077 100644 --- a/src/main/claude/claude-background-stop-records.test.ts +++ b/src/main/claude/claude-background-stop-records.test.ts @@ -33,7 +33,14 @@ async function droppedFromRoster() { { sessionId: 'session-1', fence: 7, - journal: { snapshot: () => ({ items: [] }), submissions: () => [] }, + journal: { + snapshot: () => ({ items: [] }), + submissions: () => [], + runningTurn: () => null, + itemFence: () => undefined, + visitItems: () => undefined, + wroteBeforeOpen: () => false + }, adapter: run.adapter }, RECORD, diff --git a/src/main/claude/claude-background-wake-retention.test.ts b/src/main/claude/claude-background-wake-retention.test.ts index 427e24e89609..d65d4a9a4451 100644 --- a/src/main/claude/claude-background-wake-retention.test.ts +++ b/src/main/claude/claude-background-wake-retention.test.ts @@ -40,6 +40,11 @@ async function wiredSession() { }, // No Stop was ever pressed here. stopMarks: { latest: () => null, revision: () => 0 }, + // One generation produced everything here, and every send was accepted by this process. + itemFence: () => undefined, + wroteBeforeOpen: () => false, + // The live generation's own turns are never a died turn, so this is never consulted for one. + runningTurn: () => null, snapshot: () => ({ items: [...run.journalItems.values()] .sort((a, b) => a.sequence - b.sequence) @@ -53,7 +58,7 @@ async function wiredSession() { [ parent.sessionId, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies the feed's cursor, clock, Stop marks, snapshot, submissions and item reads. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies the feed's cursor, clock, Stop marks, snapshot, submissions, item and provenance reads. journal: journal as unknown as Journal, params: { location: { diff --git a/src/main/claude/claude-subagent-permission-request.test.ts b/src/main/claude/claude-subagent-permission-request.test.ts index c3db6a4011b6..43f96eadcaad 100644 --- a/src/main/claude/claude-subagent-permission-request.test.ts +++ b/src/main/claude/claude-subagent-permission-request.test.ts @@ -23,7 +23,8 @@ import type { AgentStatusStructuredSessionSubject } from '../../shared/agent-sta import { AgentHookServer } from '../agent-hooks/server' import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../native-chat/agent-session-journal/journal-host-database-test-support' -import { settleStructuredAgentSessionDeadGeneration } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from '../native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture' +import { StructuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { createStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' @@ -243,7 +244,10 @@ async function pipeline() { await journal.appendItem( identity, { ...card.body, resolution: { ...resolution, resolvedBy: 'client-1', resolvedAt: now() } }, - { fence: FENCE, turnScope: journal.liveTurnScope() } + { + fence: FENCE, + turnScope: new StructuredAgentSessionCurrentWork(journal, FENCE).turnScope() + } ) } const cardId = (): string => { @@ -629,7 +633,7 @@ describe("a Claude subagent's permission request", () => { let settled: Promise | undefined hooks.onEvent = (event) => { if (event.type === 'ended') { - settled = settleStructuredAgentSessionDeadGeneration({ + settled = settleObservedExitForTest({ journal: run.journal, sessionId: SESSION, fence: FENCE, diff --git a/src/main/claude/claude-subagent-resume-after-restart.test.ts b/src/main/claude/claude-subagent-resume-after-restart.test.ts index 1a02e5769846..1f1f7bacee1c 100644 --- a/src/main/claude/claude-subagent-resume-after-restart.test.ts +++ b/src/main/claude/claude-subagent-resume-after-restart.test.ts @@ -19,6 +19,12 @@ import { subagentGroupJournalBody } from '../native-chat/agent-session-journal/j import { claudeSubagentGroupIdentity } from './claude-subagent-journaled-roster' import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { claudeProviderHandle } from '../../shared/agent-session-provider-handle-encoding' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../shared/agent-session-record.test-fixture' +import { settleStructuredAgentSessionLeftovers } from '../native-chat/agent-session-wire/structured-agent-session-leftover-settlement' +import { backgroundSettlementWrites } from '../native-chat/agent-session-wire/structured-agent-session-background-writes' // Frame orders are real sessions', scrubbed. A resumed agent's frames still name its ORIGINAL // spawn call while the announcement names the message call that resumed it, and a new provider @@ -139,10 +145,10 @@ async function openJournal(): Promise { } /** One provider process: a fresh sink and translator over the session's journal. */ -function acquire(journal: AgentSessionJournal) { +function acquire(journal: AgentSessionJournal, fence = 1) { const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) const translator = createClaudeJournalTranslator({ sink: deferred.sink, coalesceMs: 0 }) - deferred.bind({ journal, fence: 1, publish: () => {} }) + deferred.bind({ journal, fence, publish: () => {} }) const settle = async (): Promise => { await expect(deferred.drained()).resolves.toEqual({ ok: true }) } @@ -452,16 +458,37 @@ describe('a Claude subagent resumed after its provider restarted', () => { ) first.translator.handle(taskStarted('agent-a', 'toolu_spawn_a', 'Grok PR')) await first.settle() - // The host dies: nothing sweeps the child, and reopening the journal can say only that - // contact was lost, not when. + // The host dies: nothing sweeps the child. The next startup's reconciliation finds its + // generation's lease released with no proof, so it can say only that contact was lost, not when. await crashed.close() const journal = await openJournal() + expect(rowsListing(journal, 'agent-a')[0]?.agents).toEqual([ + expect.objectContaining({ id: 'agent-a', state: 'working', ownerFence: 1 }) + ]) + const released = agentSessionRecordFixture( + agentSessionLeaseFixture({ + sessionId: 'orca-session', + runtimeFence: 2, + claimStatus: 'released', + ownerProcess: null, + reservedSpawnToken: null + }) + ) + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: (sessionId) => (sessionId === 'orca-session' ? released : null) }, + sessionId: 'orca-session', + journal, + writes: backgroundSettlementWrites(journal) + }) + ).toMatchObject({ ok: true }) const [lost] = rowsListing(journal, 'agent-a')[0]?.agents ?? [] expect(lost).toMatchObject({ id: 'agent-a', state: 'unverifiable' }) expect(lost?.settledAt).toBeUndefined() vi.setSystemTime(5_000_000) - const second = acquire(journal) + // The next start reserves past the released lease. + const second = acquire(journal, 3) second.translator.handle(userTurn('turn-b')) second.translator.handle( frame({ diff --git a/src/main/codex/codex-structured-journal-exit-convergence.test.ts b/src/main/codex/codex-structured-journal-exit-convergence.test.ts index c9f56af83048..187d130e464d 100644 --- a/src/main/codex/codex-structured-journal-exit-convergence.test.ts +++ b/src/main/codex/codex-structured-journal-exit-convergence.test.ts @@ -5,7 +5,6 @@ import { afterEach, expect, it, vi } from 'vitest' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../shared/agent-session-journal-types' import { codexProviderHandle } from '../../shared/agent-session-provider-handle-encoding' import { readAgentJournalTurn } from '../../shared/agent-session-turn-record' -import { isStructuredAgentSessionMainAgentWorking } from '../../shared/structured-agent-session-main-agent-working' import type { AgentSessionRecord } from '../../shared/agent-session-record' import { agentSessionLeaseFixture, @@ -23,7 +22,7 @@ import { type StructuredAgentSessionChildExitSession } from '../native-chat/agent-session-wire/structured-agent-session-child-exit' import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' -import { pendingPromptExists } from '../native-chat/agent-session-wire/structured-agent-session-queued-messages' +import { StructuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' import { createCodexJournalTranslator } from './codex-structured-journal-translation' import { handleCodexSessionExit } from './codex-structured-session-close' import type { CodexSession, CodexStructuredSessionEvent } from './codex-structured-session-state' @@ -46,15 +45,14 @@ afterEach(async () => { } }) -function userVisible(journal: AgentSessionJournal) { +/** What the chat shows with the live generation at `liveFence`: 7 while the child runs, none once + * its exit released it (`structuredAgentSessionCurrentWork`). */ +function userVisible(journal: AgentSessionJournal, liveFence: number | null) { const items = journal.snapshot().items + const work = new StructuredAgentSessionCurrentWork(journal, liveFence) return { - working: isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), - journal.submissions(), - 8 - ), - prompt: pendingPromptExists(journal), + working: work.working(), + prompt: work.hasActionablePrompt(), turnStates: items.flatMap((item) => readAgentJournalTurn(item.body)?.state ?? []), runningCalls: items.filter( (item) => item.body.kind === 'tool-call' && item.body.state === 'running' @@ -149,7 +147,7 @@ it('settles a refused Codex exit through the host in one commit, then permits a ).toEqual({ accepted: true }) await expect(deferred.drained()).resolves.toEqual({ ok: true }) expect(deferred.state().queuedBytes).toBe(0) - expect(userVisible(journal)).toEqual({ + expect(userVisible(journal, 7)).toEqual({ working: true, prompt: true, turnStates: ['running'], @@ -234,13 +232,13 @@ it('settles a refused Codex exit through the host in one commit, then permits a }, sessions: new Map([[SESSION, hostSession]]), flushLifecycle: () => deferred.lifecycleBarrier(), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId, task) => task(), now: () => 5_000 }, event ) - expect(userVisible(journal)).toEqual({ + expect(userVisible(journal, null)).toEqual({ working: false, prompt: false, turnStates: ['interrupted'], diff --git a/src/main/codex/codex-subagent-roster-retention.test.ts b/src/main/codex/codex-subagent-roster-retention.test.ts index 92de0fc2aea3..8b8ae481238b 100644 --- a/src/main/codex/codex-subagent-roster-retention.test.ts +++ b/src/main/codex/codex-subagent-roster-retention.test.ts @@ -12,6 +12,9 @@ import { import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { CodexSubagentRoster } from './codex-subagent-roster' import { MAX_CODEX_SUBAGENT_GROUPS } from './codex-structured-journal-limits' +import { structuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' +import { planStaleStructuredAgentSessionState } from '../native-chat/agent-session-wire/structured-agent-session-stale-state-settlement' +import { scanRecord } from '../native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture' const THREAD = 'parent' const ORIGINAL = 'original' @@ -104,6 +107,7 @@ async function session(refuse?: 'append' | 'publish') { ) } return { + journal, roster, turn, announce, @@ -320,3 +324,39 @@ describe('Codex roster retention through the real sink and journal', () => { } ) }) + +// The roster retains a group while any child in it runs; the generation that wrote it can still end. +describe('a retained running child of a generation that ended', () => { + it('reads as not running and not working once a later generation holds the chat', async () => { + const s = await session() + s.spawn('old-a') + await laterGroups(s, true) + const original = await s.original() + expect(states(original)).toEqual([{ id: 'old-a', state: 'working' }]) + const itemId = original?.row.itemId ?? '' + + // Generation 1 wrote it; the lease now names generation 13, which is live. + const record = scanRecord('session-codex-roster-retention', false) + const work = structuredAgentSessionCurrentWork(s.journal, { record }) + expect(work.isCurrentItem(itemId)).toBe(false) + expect(work.working()).toBe(false) + + const plan = planStaleStructuredAgentSessionState({ + journal: s.journal, + sessionId: record.sessionId, + fence: 13, + acquisitionGeneration: null, + deathEvidence: null, + below: 13, + entriesBelow: 13 + }) + const revised = plan.mutations.find( + (mutation) => mutation.kind === 'item' && mutation.itemId === itemId + ) + const body = revised?.kind === 'item' ? revised.body : undefined + const groups = body?.kind === 'message' ? body.blocks.filter(isSubagentGroupBlock) : [] + expect(groups.flatMap((group) => group.agents.map(({ id, state }) => ({ id, state })))).toEqual( + [{ id: 'old-a', state: 'unverifiable' }] + ) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts b/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts index ca622c5b2f71..9321c7529180 100644 --- a/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts @@ -9,10 +9,8 @@ import { import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' import { AgentSessionSubscribers } from '../agent-session-wire/structured-agent-session-subscribers' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from '../agent-session-wire/structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from '../agent-session-wire/structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from '../agent-session-wire/structured-agent-session-observed-exit.test-fixture' import { createTrackedJournalOpener, openTestJournalHostDatabase, @@ -96,7 +94,7 @@ async function seedWork(count = 201) { } function settle() { - return settleStructuredAgentSessionDeadGeneration({ + return settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -191,7 +189,7 @@ describe('atomic terminal settlement', () => { return result }) ) - expect(await settle()).toEqual({ ok: true }) + expect(await settle()).toMatchObject({ ok: true }) vi.restoreAllMocks() const settled = journal.snapshot() expect(published).toEqual([settled]) @@ -209,7 +207,7 @@ describe('atomic terminal settlement', () => { expect(frames.at(-1)).toMatchObject({ type: 'batch', batch: { cursor: settled.cursor } }) const frameCount = frames.length expect(settled.submissions[0]).toMatchObject({ dispatchState: 'unknown', recovered: true }) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() const calls = settled.items.filter((item) => item.body.kind === 'tool-call') expect(calls).toHaveLength(201) expect( @@ -230,7 +228,7 @@ describe('atomic terminal settlement', () => { 'lifecycle-batch' ]) expect(committed.every((row) => row.recovered)).toBe(true) - expect(await settle()).toEqual({ ok: true }) + expect(await settle()).toMatchObject({ ok: true }) expect(journal.snapshot()).toEqual(settled) expect(rows().slice(diskBefore.length)).toEqual(committed) expect(published).toEqual([settled]) @@ -238,7 +236,7 @@ describe('atomic terminal settlement', () => { await journal.close() journal = await open() expect(journal.snapshot()).toEqual(settled) - expect(await settle()).toEqual({ ok: true }) + expect(await settle()).toMatchObject({ ok: true }) expect(journal.snapshot()).toEqual(settled) } ) @@ -392,12 +390,12 @@ describe('atomic terminal settlement', () => { WHEN NEW.session_id = '${SESSION}' AND json_extract(NEW.row_json, '$.recovered') = 1 AND json_extract(NEW.row_json, '$.kind') = 'item' BEGIN SELECT RAISE(ABORT, 'item rejected after submission'); END`) - expect(await settleStructuredAgentSessionDeadGeneration(input)).toMatchObject({ ok: false }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: false }) expect(journal.snapshot()).toEqual(before) expect(rows()).toEqual(diskBefore) expect(published).toEqual([]) database.db.exec('DROP TRIGGER reject_submission_settlement_item') - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) expect(journal.submission('unanswered')).toMatchObject({ dispatchState: 'rejected', recovered: true, @@ -406,7 +404,7 @@ describe('atomic terminal settlement', () => { const committed = rows().slice(diskBefore.length) expect(committed.filter((row) => row.kind === 'dispatch')).toHaveLength(1) expect(published).toEqual([journal.snapshot()]) - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) expect(rows().slice(diskBefore.length)).toEqual(committed) expect(published).toHaveLength(1) } diff --git a/src/main/native-chat/agent-session-journal/journal-context-controller.ts b/src/main/native-chat/agent-session-journal/journal-context-controller.ts index 7403760e8e91..b76514743ac7 100644 --- a/src/main/native-chat/agent-session-journal/journal-context-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-context-controller.ts @@ -7,6 +7,7 @@ import type { JournalReplacementItem } from './journal-epoch-replacement' import type { JournalOperationReceipt, JournalRowWriter } from './journal-row-writer' import type { JournalReducerState } from './journal-reducer' import type { JournalQueuedMessages } from './journal-queued-messages' +import type { JournalWriteOptions } from './journal-host-database' export class JournalContextController { constructor( @@ -35,8 +36,9 @@ export class JournalContextController { floor: AgentJournalCursor, fence: number, items: readonly JournalReplacementItem[], - receipt: (cursor: AgentJournalCursor) => JournalOperationReceipt + receipt: (cursor: AgentJournalCursor) => JournalOperationReceipt, + options?: JournalWriteOptions ) { - return rewindJournalContext({ ...this.deps, floor, fence, items, receipt }) + return rewindJournalContext({ ...this.deps, floor, fence, items, receipt, options }) } } diff --git a/src/main/native-chat/agent-session-journal/journal-context-rewind.ts b/src/main/native-chat/agent-session-journal/journal-context-rewind.ts index dbaf544e3ef7..e7051cb1ed91 100644 --- a/src/main/native-chat/agent-session-journal/journal-context-rewind.ts +++ b/src/main/native-chat/agent-session-journal/journal-context-rewind.ts @@ -11,6 +11,7 @@ import { buildJournalQueueClearRow } from './journal-stop-and-resume-rows' import { journalQueuePauseRestatement } from './queued-message-pause' import type { JournalQueuedMessages } from './journal-queued-messages' import type { JournalRow } from './journal-row-schema' +import type { JournalWriteOptions } from './journal-host-database' export async function rewindJournalContext(input: { state: () => JournalReducerState @@ -20,6 +21,7 @@ export async function rewindJournalContext(input: { fence: number items: readonly JournalReplacementItem[] receipt: (cursor: AgentJournalCursor) => JournalOperationReceipt + options?: JournalWriteOptions | undefined }): Promise { let receipt: JournalOperationReceipt | undefined const rows = await input.writer.enqueueRows( @@ -77,6 +79,7 @@ export async function rewindJournalContext(input: { seq, ts: item.observedAt ?? ts, fence: input.fence, + ownerFence: item.ownerFence, turnScope: item.turnScope ?? { kind: 'thread' } }), revision @@ -91,7 +94,8 @@ export async function rewindJournalContext(input: { { write: (db) => receipt!.write(db), committed: () => receipt!.committed() - } + }, + input.options ) const last = rows.at(-1)! return { epoch: last.epoch, sequence: last.seq } diff --git a/src/main/native-chat/agent-session-journal/journal-damage.test.ts b/src/main/native-chat/agent-session-journal/journal-damage.test.ts index 8b59940a6a19..168e74484b69 100644 --- a/src/main/native-chat/agent-session-journal/journal-damage.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-damage.test.ts @@ -206,8 +206,8 @@ describe('a row the reader would reject', () => { await expect( journal.replaceEpochItems('legacy_import', 1, [ - { identity: item(5), body: body('replacement') }, - { identity: item(6), body: blankCallId } + { identity: item(5), body: body('replacement'), ownerFence: 1 }, + { identity: item(6), body: blankCallId, ownerFence: 1 } ]) ).rejects.toMatchObject({ code: 'journal_row_rejected' }) diff --git a/src/main/native-chat/agent-session-journal/journal-database.ts b/src/main/native-chat/agent-session-journal/journal-database.ts index 0dee9537b3f1..ebd0e6ab00dd 100644 --- a/src/main/native-chat/agent-session-journal/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-database.ts @@ -18,6 +18,14 @@ import { ensureCommandReceiptsTable } from './command-receipt-schema' import { ensureAgentSessionAttachmentClaimTables } from '../agent-session-attachments/agent-session-attachment-claims' export const JOURNAL_BUSY_TIMEOUT_MS = 5000 + +/** `background`: bookkeeping no person waits on. It never waits for another connection's lock: + * SQLITE_BUSY at once, and its caller retries on its own schedule. */ +export type JournalWriteOptions = { background?: true } + +export function backgroundWrite(background: true | undefined): JournalWriteOptions | undefined { + return background ? { background } : undefined +} /** Bounds the WAL a checkpoint leaves behind; SQLite truncates it back to this after a reset. */ export const JOURNAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024 /** Every commit is fsynced before its caller continues. */ @@ -139,7 +147,26 @@ function migrateJournalSchema(db: Database.Database, stored: number): void { export function runJournalTransaction( db: Database.Database, run: (db: Database.Database) => T, - onStranded: () => void = () => undefined + onStranded: () => void = () => undefined, + options: JournalWriteOptions = {} +): T { + if (!options.background) { + return runImmediateTransaction(db, run, onStranded) + } + // Set and restored around this one synchronous transaction: no other statement runs between. + const busyTimeout = journalPragmaNumber(db, 'busy_timeout') + db.pragma('busy_timeout = 0') + try { + return runImmediateTransaction(db, run, onStranded) + } finally { + db.pragma(`busy_timeout = ${busyTimeout}`) + } +} + +function runImmediateTransaction( + db: Database.Database, + run: (db: Database.Database) => T, + onStranded: () => void ): T { db.exec('BEGIN IMMEDIATE') try { diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 3a79762ee2dd..7dc70fba06a8 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -2,7 +2,7 @@ import type { AgentJournalCursor, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' import type { JournalQueuePauseRestatement } from './queued-message-pause' import { publishNewEpoch } from './journal-epoch-rollover' @@ -10,6 +10,7 @@ import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' import type { JournalWriteBody } from './journal-write-queue' +import type { JournalEpochFounding } from './journal-epoch-founding' export class JournalEpochController { constructor( @@ -25,6 +26,8 @@ export class JournalEpochController { queuePauseRestatement: () => JournalQueuePauseRestatement cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void + /** A published epoch replaces one still held unwritten. */ + founding: Pick } ) {} @@ -36,7 +39,10 @@ export class JournalEpochController { reason, fence, now: this.deps.now(), - onPublished: this.deps.adopt + onPublished: (loaded) => { + this.deps.founding.settled() + this.deps.adopt(loaded) + } }) } @@ -58,7 +64,8 @@ export class JournalEpochController { replace( reason: AgentJournalEpochReason, fence: number, - items: readonly JournalReplacementItem[] + items: readonly JournalReplacementItem[], + options?: JournalWriteOptions ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.identity.sessionId) @@ -72,7 +79,11 @@ export class JournalEpochController { queuePause: this.deps.queuePauseRestatement(), now: this.deps.now, mintEpoch: this.deps.mintEpoch, - onPublished: this.deps.adopt + onPublished: (loaded) => { + this.deps.founding.settled() + this.deps.adopt(loaded) + }, + options }) return this.deps.cursor() }) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts b/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts new file mode 100644 index 000000000000..d59334037231 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts @@ -0,0 +1,62 @@ +// A chat opened with no journal yet. Opening it writes nothing: its first epoch row is held here, +// in memory, and lands in the transaction of the first row the chat writes. A crash before then +// leaves no journal, exactly as before the open, and the next open mints a new epoch. + +import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import type Database from '../../sqlite/sync-database' +import type { JournalLoad } from './journal-open' +import { journalOpenRefusalError } from './journal-open-failure' +import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import { insertJournalRow, publishJournalSessionEpoch } from './journal-row-table' +import type { JournalRow } from './journal-row-schema' + +export class JournalEpochFounding { + private pending: JournalRow | null = null + + constructor(private readonly identity: AgentSessionJournalIdentity) {} + + /** The empty journal a reader sees until the first write founds it. */ + hold(epoch: string, now: number): JournalLoad { + const row: JournalRow = { + kind: 'epoch', + reason: 'session_created', + providerHandle: agentSessionJournalProviderHandle(this.identity), + v: journalRowSchemaVersion([]), + epoch, + seq: 1, + fence: 0, + ts: now + } + const state = createJournalReducerState(this.identity.sessionId, epoch) + applyJournalRow(state, row) + state.oldestSequence = 1 + this.pending = row + return { state, newer: null, damage: null } + } + + /** Whether the epoch still waits for its first write. */ + held(): boolean { + return this.pending !== null + } + + /** First inside a transaction that writes the chat's rows: a rollback takes the epoch with it. + * Its own failure is the journal's open failing, as an eager open's would have been. */ + inTransaction(db: Database.Database): void { + if (!this.pending) { + return + } + try { + insertJournalRow(db, this.identity.sessionId, this.pending) + publishJournalSessionEpoch(db, this.identity, this.pending.epoch) + } catch (error) { + throw journalOpenRefusalError(error) + } + } + + /** The transaction that carried the epoch committed, or another epoch replaced it. */ + settled(): void { + this.pending = null + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 36247e2571ff..c64027b94555 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -96,7 +96,7 @@ describe('journal epoch replacement', () => { const published: JournalLoad[] = [] replace({ - items: [{ identity: item(1), body: { kind: 'status', text: 'republished' } }], + items: [{ identity: item(1), body: { kind: 'status', text: 'republished' }, ownerFence: 1 }], onPublished: (loaded) => published.push(loaded) }) @@ -112,7 +112,7 @@ describe('journal epoch replacement', () => { 'a replace', (journal: AgentSessionJournal) => journal.replaceEpochItems('legacy_import', 1, [ - { identity: item(9), body: { kind: 'status', text: 'republished' } } + { identity: item(9), body: { kind: 'status', text: 'republished' }, ownerFence: 1 } ]) ], ['a rollover', (journal: AgentSessionJournal) => journal.rollEpoch('handle_forked', 1)] diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 67927b401588..2b1b8af334d1 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -11,7 +11,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import type { JournalLoad } from './journal-open' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' @@ -38,6 +38,9 @@ export type JournalReplacementItem = AgentJournalProducerLinkage & { /** Absent for history rebuilt from a source that never stated one: derived from position, as * a legacy row's is, and then written down. */ turnScope?: AgentJournalTurnScope + /** Carried from the epoch it came from (`JournalItemRow.ownerFence`), never the replacing + * writer's fence: replacing history does not make it the current generation's work. */ + ownerFence: number } export function replaceJournalEpoch(input: { @@ -53,6 +56,7 @@ export function replaceJournalEpoch(input: { mintEpoch: () => string /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void + options?: JournalWriteOptions }): void { const epoch = input.mintEpoch() const state = createJournalReducerState(input.identity.sessionId, epoch) @@ -71,6 +75,7 @@ export function replaceJournalEpoch(input: { body: item.body, seq: state.lastSequence + 1, fence: input.fence, + ownerFence: item.ownerFence, ts: item.observedAt ?? input.now(), linkage: item, turnScope: item.turnScope ?? state.derivedTurnScope.scopeFor(item.body) @@ -112,7 +117,7 @@ export function replaceJournalEpoch(input: { insertJournalRow(db, sessionId, row) } publishJournalSessionEpoch(db, input.identity, epoch) - }) + }, input.options) // COMMIT landed: on disk the superseded rows are gone and this epoch is the // live one. The caller adopts that immediately, or a later failure leaves the diff --git a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts index 32b0c846571f..15ee5c71d815 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts @@ -6,7 +6,10 @@ import { resolve } from 'node:path' import { JournalHostDatabase } from './journal-host-database' import { replayJournal, type JournalLoad } from './journal-open' import { serializeJournalRow, type JournalRow } from './journal-row-schema' +import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' +import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { + insertJournalRow, iterateJournalEpochRows, publishJournalSessionEpoch, readJournalSessionEpoch, @@ -137,7 +140,9 @@ export function insertTestJournalRow( insertTestJournalRowJson(db, sessionId, row.seq, serializeJournalRow(row), row.ts) } -/** A raw `row_json` at `seq`, the way a newer build or a bad write would leave it. */ +/** A raw `row_json` at `seq`, the way a newer build or a bad write would leave it. A chat whose + * journal was opened and never written holds its epoch in memory only; the row's own epoch is + * founded first, as that journal's first write would have. */ export function insertTestJournalRowJson( db: Database.Database, sessionId: string, @@ -145,11 +150,31 @@ export function insertTestJournalRowJson( rowJson: string, ts = 1 ): void { + if (readJournalSessionEpoch(db, sessionId) === null) { + const parsed: unknown = JSON.parse(rowJson) + const epoch = + typeof parsed === 'object' && parsed !== null && 'epoch' in parsed ? parsed.epoch : '' + foundTestJournalEpoch(db, sessionId, String(epoch)) + } db.prepare( 'INSERT INTO journal_rows (session_id, epoch, seq, ts, row_json) VALUES (?, ?, ?, ?, ?)' ).run(sessionId, liveEpoch(db, sessionId), seq, ts, rowJson) } +function foundTestJournalEpoch(db: Database.Database, sessionId: string, epoch: string): void { + insertJournalRow(db, sessionId, { + kind: 'epoch', + reason: 'session_created', + providerHandle: agentSessionJournalProviderHandle({ agent: 'claude', providerHandle: null }), + v: journalRowSchemaVersion([]), + epoch, + seq: 1, + fence: 0, + ts: 1 + }) + publishTestJournalEpoch(db, sessionId, epoch) +} + export function updateTestJournalRowJson( db: Database.Database, sessionId: string, diff --git a/src/main/native-chat/agent-session-journal/journal-host-database.ts b/src/main/native-chat/agent-session-journal/journal-host-database.ts index aaea0228b6e9..264fcfae04f5 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database.ts @@ -10,8 +10,10 @@ import type Database from '../../sqlite/sync-database' import { openJournalDatabase, runJournalTransaction, + type JournalWriteOptions, type OpenJournalDatabase } from './journal-database' +export type { JournalWriteOptions } from './journal-database' import { journalOpenRefusalError } from './journal-open-failure' import { AgentSessionJournalError } from './journal-write-guards' @@ -61,10 +63,15 @@ export class JournalHostDatabase { } /** One IMMEDIATE transaction; see `runJournalTransaction`. */ - transaction(run: (db: Database.Database) => T): T { - return runJournalTransaction(this.db, run, () => { - this.stranded = true - }) + transaction(run: (db: Database.Database) => T, options?: JournalWriteOptions): T { + return runJournalTransaction( + this.db, + run, + () => { + this.stranded = true + }, + options + ) } /** Last, after every store has drained. A close that fails keeps the handle, so the retried diff --git a/src/main/native-chat/agent-session-journal/journal-item-fold.ts b/src/main/native-chat/agent-session-journal/journal-item-fold.ts index 10aee4a99c41..6085e6ab88d7 100644 --- a/src/main/native-chat/agent-session-journal/journal-item-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-item-fold.ts @@ -27,8 +27,8 @@ export function upsertJournalItem( itemId: string, revision: number, next: AgentJournalRenderItem, - /** The creating write's fence: the generation a running turn belongs to. */ - fence: number + /** The write's fence, and the provenance it states (`JournalItemRow.ownerFence`). */ + fences: { fence: number; ownerFence?: number } ): void { const tombstoned = state.tombstones.get(itemId) if (tombstoned !== undefined && revision <= tombstoned) { @@ -41,10 +41,14 @@ export function upsertJournalItem( state.derivedTurnScope.observe(itemId, isRootAgentJournalItem(next), existing?.body, next.body) if (!existing) { state.items.set(itemId, next) - state.itemFences.set(itemId, fence) + state.itemFences.set(itemId, fences.ownerFence ?? fences.fence) state.tombstones.delete(itemId) return } + // A revision keeps the provenance it found unless it states one: an owner's observation does. + if (fences.ownerFence !== undefined) { + state.itemFences.set(itemId, fences.ownerFence) + } // Creation sequence is the ordering key; a revision refreshes content only. // `observedAt` is pinned with it: clients sort the timeline by that timestamp, // so letting a revision advance it makes the row jump past everything that diff --git a/src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts b/src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts new file mode 100644 index 000000000000..e275e8102eda --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts @@ -0,0 +1,186 @@ +// Execution provenance (`JournalItemRow.ownerFence`): the generation whose execution produced an +// item, kept apart from each row's writer fence. Every writer states it; the fold prefers it. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { retainedRowReplacement } from '../agent-session-wire/structured-rewind-retained-host-rows' +import type { JournalHostDatabase } from './journal-host-database' +import { + createTrackedJournalOpener, + insertTestJournalRowJson, + liveTestJournalRows, + openTestJournalHostDatabase +} from './journal-host-database-test-support' +import type { AgentSessionJournal } from './journal-store' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: codexProviderHandle('thread-1') +} +const SCOPE = AGENT_JOURNAL_THREAD_SCOPE +const TURN: AgentJournalItemIdentity = { + provider: 'codex', + threadId: 'thread-1', + turnId: 'turn-1', + ordinal: 1 +} +const TURN_ID = agentJournalItemKey(TURN) +const RUNNING: AgentJournalItemBody = { kind: 'turn', turnId: 'turn-1', state: 'running' } +const SETTLED: AgentJournalItemBody = { kind: 'turn', turnId: 'turn-1', state: 'interrupted' } + +let root: string +let database: JournalHostDatabase +const journals = createTrackedJournalOpener() + +function item(ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal } +} + +type StoredItemWrite = { kind?: string; ownerFence?: number; mutations?: StoredItemWrite[] } + +/** Each stored item write's stated provenance, in write order, a lifecycle batch's included. */ +function storedOwnerFences(): (number | undefined)[] { + return liveTestJournalRows(database.db, IDENTITY.sessionId).flatMap((row) => { + const parsed: StoredItemWrite = JSON.parse(row.rowJson) + const writes = parsed.kind === 'lifecycle-batch' ? (parsed.mutations ?? []) : [parsed] + return writes.flatMap((write) => (write.kind === 'item' ? [write.ownerFence] : [])) + }) +} + +async function open(): Promise { + return journals.open({ identity: IDENTITY, stateDirectory: root }) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-provenance-')) + database = openTestJournalHostDatabase(root) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('execution provenance', () => { + it("records a provider observation's owner, and a host revision keeps it", async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 4, ownerFence: 3, turnScope: SCOPE }) + // Host bookkeeping under a later writer fence states no owner of its own. + await journal.appendItem(TURN, SETTLED, { fence: 5, turnScope: SCOPE }) + + expect(journal.itemFence(TURN_ID)).toBe(3) + // Every row states it: none leaves the reader to guess from the writer fence. + expect(storedOwnerFences()).toEqual([3, 3]) + }) + + it('a first write with no stated owner is its writer generation’s', async () => { + const journal = await open() + await journal.appendItem( + item(2), + { kind: 'status', text: 'note' }, + { fence: 6, turnScope: SCOPE } + ) + expect(journal.itemFence(agentJournalItemKey(item(2)))).toBe(6) + expect(storedOwnerFences()).toEqual([6]) + }) + + it('a lifecycle batch states it on each item: the batch owner, else the kept value', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 2, ownerFence: 2, turnScope: SCOPE }) + // A provider batch at the owner's fence creates one item. + await journal.appendLifecycleBatch({ + settlementId: 'provider', + fence: 3, + ownerFence: 3, + mutations: [{ kind: 'item', identity: item(2), body: RUNNING, turnScope: SCOPE }] + }) + // A settlement at a later fence revises the first: it keeps that item's owner. + await journal.appendLifecycleBatch({ + settlementId: 'settlement', + fence: 4, + recovered: true, + mutations: [{ kind: 'item', identity: TURN, body: SETTLED, turnScope: SCOPE }] + }) + + expect(journal.itemFence(TURN_ID)).toBe(2) + expect(journal.itemFence(agentJournalItemKey(item(2)))).toBe(3) + expect(storedOwnerFences()).toEqual([2, 3, 2]) + }) + + it('survives a reopen from storage', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 4, ownerFence: 3, turnScope: SCOPE }) + await journals.closeAll() + + expect((await open()).itemFence(TURN_ID)).toBe(3) + }) + + it('epoch replacement carries each item’s provenance', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 2, ownerFence: 2, turnScope: SCOPE }) + const before = journal.epoch + + await journal.replaceEpochItems('legacy_import', 5, [ + { identity: TURN, body: RUNNING, ownerFence: journal.itemFence(TURN_ID) ?? 0 }, + { identity: item(2), body: { kind: 'status', text: 'new' }, ownerFence: 5 } + ]) + + expect(journal.epoch).not.toBe(before) + // Written at fence 5, the dead generation's turn is still fence 2's work. + expect(journal.itemFence(TURN_ID)).toBe(2) + expect(journal.itemFence(agentJournalItemKey(item(2)))).toBe(5) + }) + + it("a rewind's retained row carries the old fold's provenance; an older record's is never current", () => { + const row = { + itemId: TURN_ID, + body: RUNNING, + observedAt: 1, + ownerFence: 2 + } + expect(retainedRowReplacement(row).ownerFence).toBe(2) + const { ownerFence: _stated, ...older } = row + expect(retainedRowReplacement(older).ownerFence).toBe(0) + }) + + it('a stored provenance that is not a fence is dropped, and the row reads as its writer’s', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 2, ownerFence: 2, turnScope: SCOPE }) + const rows = liveTestJournalRows(database.db, IDENTITY.sessionId) + const stored = rows.find((row) => row.rowJson.includes('"kind":"item"')) + const last = rows.at(-1)?.seq + if (!stored || last === undefined) { + throw new Error('expected the item row') + } + await journals.closeAll() + database = openTestJournalHostDatabase(root) + const parsed: Record = JSON.parse(stored.rowJson) + insertTestJournalRowJson( + database.db, + IDENTITY.sessionId, + last + 1, + JSON.stringify({ + ...parsed, + seq: last + 1, + revision: Number(parsed.revision) + 1, + fence: 7, + ownerFence: -1 + }) + ) + + expect((await open()).itemFence(TURN_ID)).toBe(2) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-item-provenance.ts b/src/main/native-chat/agent-session-journal/journal-item-provenance.ts new file mode 100644 index 000000000000..03e2855994e3 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-item-provenance.ts @@ -0,0 +1,137 @@ +// Execution provenance: the generation whose execution produced an item (`JournalItemRow.ownerFence`) +// and each entry of a container row. +// +// Each entry of a container row: every subagent roster entry and every +// background task. One roster row is revised in place across generations (a resumed provider keeps +// the group's identity), so the row's own provenance cannot say which generation each child +// belongs to. A provider's observation stamps the entries it changed or added with its own +// generation; an entry it carries unchanged keeps the one it had (or, never stamped, the row's +// earlier provenance). Host bookkeeping states no provenance and stamps nothing. + +import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' +import type { JournalReducerState } from './journal-reducer' +import { + isBackgroundTaskBlock, + isSubagentGroupBlock, + type NativeChatBlock +} from '../../../shared/native-chat-types' + +type Entry = { ownerFence?: number } + +/** An entry's observed fields, in a fixed order: its provenance is not part of what was seen. */ +function observationKey(entry: Entry): string { + const { ownerFence: _ownerFence, ...fields } = entry + return JSON.stringify(Object.entries(fields).sort(([left], [right]) => left.localeCompare(right))) +} + +function stamp( + entry: T, + previous: T | undefined, + previousItemFence: number | undefined, + ownerFence: number +): T { + if (previous && observationKey(previous) === observationKey(entry)) { + const kept = previous.ownerFence ?? previousItemFence + return kept === undefined || kept === entry.ownerFence ? entry : { ...entry, ownerFence: kept } + } + // No earlier entry to compare: one that already names its generation (history carried into a new + // epoch) keeps it. + if (!previous && entry.ownerFence !== undefined) { + return entry + } + return entry.ownerFence === ownerFence ? entry : { ...entry, ownerFence } +} + +function hasEntries(body: AgentJournalItemBody): boolean { + return ( + body.kind === 'message' && + body.blocks.some((block) => isSubagentGroupBlock(block) || isBackgroundTaskBlock(block)) + ) +} + +/** The body as written by an observation of generation `ownerFence`, each entry stamped. */ +export function stampJournalEntryProvenance( + body: AgentJournalItemBody, + previous: { body: AgentJournalItemBody; ownerFence?: number } | undefined, + ownerFence: number +): AgentJournalItemBody { + if (body.kind !== 'message' || !hasEntries(body)) { + return body + } + const earlier: readonly NativeChatBlock[] = + previous?.body.kind === 'message' ? previous.body.blocks : [] + const blocks = body.blocks.map((block): NativeChatBlock => { + if (isSubagentGroupBlock(block)) { + const group = earlier.find( + (candidate) => isSubagentGroupBlock(candidate) && candidate.groupId === block.groupId + ) + const agents = group && isSubagentGroupBlock(group) ? group.agents : [] + return { + ...block, + agents: block.agents.map((agent) => + stamp( + agent, + agents.find((candidate) => candidate.id === agent.id), + previous?.ownerFence, + ownerFence + ) + ) + } + } + if (isBackgroundTaskBlock(block)) { + const task = earlier.find( + (candidate) => isBackgroundTaskBlock(candidate) && candidate.taskId === block.taskId + ) + return stamp( + block, + task && isBackgroundTaskBlock(task) ? task : undefined, + previous?.ownerFence, + ownerFence + ) + } + return block + }) + return { ...body, blocks } +} + +/** An entry's generation: its own stamp, else the row's provenance. */ +export function journalEntryOwnerFence( + entry: Entry, + itemFence: number | undefined +): number | undefined { + return entry.ownerFence ?? itemFence +} + +/** The provenance a row states: the writer's own claim when it has one — a provider's observation, + * or history carried into a new epoch — else host bookkeeping's, which keeps the item's, and for + * an item it creates is the fence it writes at. Every item row states it. */ +export function journalItemOwnerFence( + state: Pick, + resolvedItemId: string, + write: { fence: number; ownerFence?: number } +): number { + if (write.ownerFence !== undefined) { + return write.ownerFence + } + const kept = state.items.has(resolvedItemId) ? state.itemFences.get(resolvedItemId) : undefined + return kept ?? write.fence +} + +/** A provider's observation stamps each roster entry and background task it saw + * (`stampJournalEntryProvenance`); host bookkeeping states no provenance and stamps nothing. */ +export function journalObservedBody( + state: Pick, + resolvedItemId: string, + body: AgentJournalItemBody, + ownerFence: number | undefined +): AgentJournalItemBody { + if (ownerFence === undefined) { + return body + } + const previous = state.items.get(resolvedItemId) + return stampJournalEntryProvenance( + body, + previous && { body: previous.body, ownerFence: state.itemFences.get(resolvedItemId) }, + ownerFence + ) +} diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts b/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts index f4cad1f9edc0..9da96daf490c 100644 --- a/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts @@ -245,6 +245,10 @@ describe('claude import', () => { uuid: 'e3a5b7c9-2d13-4f88-94ce-6b0d8f2a4e31' }) ]) + // Imported at fence 1, yet no generation's work: provenance 0, so nothing in it is current. + expect(journal.snapshot().items.map((entry) => journal.itemFence(entry.itemId))).toEqual([ + 0, 0, 0, 0 + ]) }) it('stays aligned when the decoder drops lines the tracker still walks', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-import.ts b/src/main/native-chat/agent-session-journal/journal-legacy-import.ts index 1e6171e94aa9..7c8b67e440a2 100644 --- a/src/main/native-chat/agent-session-journal/journal-legacy-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-legacy-import.ts @@ -138,7 +138,9 @@ export async function prepareLegacyTranscriptImport(input: { replacement.push({ identity, body: legacyItemBody(message, limits), - observedAt: message.timestamp ?? undefined + observedAt: message.timestamp ?? undefined, + // Imported history is no generation's work: never current, whatever it says still runs. + ownerFence: 0 }) } return { ok: true, items: replacement } diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index f942f0185d05..2c31cf33a96b 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -16,9 +16,11 @@ import { } from './journal-row-builders' import type { JournalLifecycleBatchInput, + JournalPlannedLifecycleBatchInput, JournalResolvedLifecycleBatchInput } from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' +import { backgroundWrite, type JournalWriteOptions } from './journal-database' import { journalQueuedRejectionRowBuilders } from './journal-pending-submission-recovery' export class JournalLifecycleBatchAppender { @@ -27,7 +29,8 @@ export class JournalLifecycleBatchAppender { state: () => JournalReducerState cursor: () => AgentJournalCursor enqueueRows: ( - plan: () => readonly ((seq: number, ts: number) => JournalRow)[] + plan: () => readonly ((seq: number, ts: number) => JournalRow)[], + options?: JournalWriteOptions ) => Promise } ) {} @@ -39,7 +42,8 @@ export class JournalLifecycleBatchAppender { const rejections = journalQueuedRejectionRowBuilders( this.deps.state, input.fence, - input.rejectsQueued + input.rejectsQueued, + input.rejectsQueuedOnly ) if (rejections.length > 0 && input.mutations.length === 0) { throw new Error('journal_lifecycle_batch_mutation_bound_exceeded') @@ -58,10 +62,28 @@ export class JournalLifecycleBatchAppender { ...this.planMutations(input, input.mutations) ] }) - .then((rows) => { - const last = rows.at(-1) - return last ? { epoch: last.epoch, sequence: last.seq } : this.deps.cursor() - }) + .then((rows) => this.cursorAfter(rows)) + } + + /** `append`, with every row chosen at the batch's own turn in the queue. */ + appendPlanned(input: JournalPlannedLifecycleBatchInput): Promise { + return this.deps + .enqueueRows(() => { + if (this.wasApplied(input.settlementId)) { + return [] + } + const { mutations, dispatches } = input.plan() + return [ + ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), + ...this.planMutations(input, mutations) + ] + }, backgroundWrite(input.background)) + .then((rows) => this.cursorAfter(rows)) + } + + private cursorAfter(rows: readonly JournalRow[]): AgentJournalCursor { + const last = rows.at(-1) + return last ? { epoch: last.epoch, sequence: last.seq } : this.deps.cursor() } /** The rows a resolved settlement writes, planned at its own turn in the queue: its mutations, @@ -75,12 +97,22 @@ export class JournalLifecycleBatchAppender { } private planMutations( - input: Pick, - mutations: readonly JournalLifecycleMutationInput[] + input: Pick, + stated: readonly JournalLifecycleMutationInput[] ): ((seq: number, ts: number) => JournalRow)[] { if (this.wasApplied(input.settlementId)) { return [] } + // A provider's batch states its generation on every item it writes. + const { ownerFence } = input + const mutations = + ownerFence === undefined + ? stated + : stated.map((mutation) => + mutation.kind === 'item' && mutation.ownerFence === undefined + ? { ...mutation, ownerFence } + : mutation + ) const current = this.deps.state() const options = { ...input, epoch: current.epoch } // Every row in this transaction must advance past the rows planned before it. @@ -141,6 +173,7 @@ export class JournalLifecycleBatchAppender { ...only.linkage, turnScope: only.turnScope, fence: input.fence, + ownerFence: only.ownerFence, recovered: input.recovered }, revisions diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test-fixture.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test-fixture.ts index 509850c78841..cc94120fe538 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test-fixture.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test-fixture.ts @@ -28,8 +28,8 @@ export const LEGACY_MIXED_PARTITIONS = [ [['', 30]], [['', 96]], [ - [':1/2', 117], - [':2/2', 5] + [':1/2', 116], + [':2/2', 6] ], [['', 70]], [ @@ -37,9 +37,9 @@ export const LEGACY_MIXED_PARTITIONS = [ [':2/2', 34] ], [ - [':1/3', 66], - [':2/3', 66], - [':3/3', 32] + [':1/3', 65], + [':2/3', 65], + [':3/3', 34] ], [ [':1/4', 62], diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts index 25b6f5b96531..1943f30dd673 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts @@ -57,6 +57,7 @@ function previousProbe(settlementId: string, mutations: readonly JournalLifecycl ? { ...mutation, body: { ...mutation.body, outcome: 'cancellation' } } : mutation, journalLifecycleMutationItemId(mutation), + Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER ) ) diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts index 4e01628c6d58..525605b9ec3b 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts @@ -136,6 +136,8 @@ function toLifecycleMutationRow(mutation: JournalLifecycleMutationInput): Journa return journalLifecycleMutationRow( sizedAsStopped(mutation), journalLifecycleMutationItemId(mutation), + Number.MAX_SAFE_INTEGER, + // The widest provenance any row can state, so the chunk built from it still fits. Number.MAX_SAFE_INTEGER ) } diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts index 1340b99f3106..9b939c267436 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts @@ -26,6 +26,8 @@ export type JournalLifecycleMutationInput = JournalItemAddress & linkage?: AgentJournalProducerLinkage /** Which turn the row belongs to. Kept from the write that creates the row. */ turnScope: AgentJournalTurnScope + /** The writer's provenance claim (`journalItemOwnerFence`); absent for host bookkeeping. */ + ownerFence?: number } | { kind: 'tombstone' } ) @@ -61,7 +63,9 @@ export function journalLifecycleItemMutation( export function journalLifecycleMutationRow( mutation: JournalLifecycleMutationInput, itemId: string, - revision: number + revision: number, + /** What an item mutation states (`journalItemOwnerFence`). */ + ownerFence: number ): JournalLifecycleMutation { return mutation.kind === 'item' ? { @@ -70,6 +74,7 @@ export function journalLifecycleMutationRow( revision, body: mutation.body, turnScope: mutation.turnScope, + ownerFence, ...agentJournalLinkageFields(mutation.linkage) } : { kind: 'tombstone', itemId, revision } diff --git a/src/main/native-chat/agent-session-journal/journal-open-failure.ts b/src/main/native-chat/agent-session-journal/journal-open-failure.ts index 69ab0973e75d..b9020a8742c9 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-failure.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-failure.ts @@ -88,6 +88,16 @@ export function isJournalWrittenByNewerOrca(error: unknown): boolean { return error instanceof AgentSessionJournalError && error.code === 'journal_read_only' } +/** A failed open no retry gets past in this process: damage, or a newer Orca's journal. */ +export function isJournalOpenFailurePermanent(error: unknown): boolean { + const refusal = journalOpenRefusal(error) + if (refusal.code !== 'agent_session_journal_unreadable') { + return false + } + const reason = refusal.details?.reason + return reason === 'journalCorrupt' || reason === 'journalWrittenByNewerOrca' +} + /** Why a journal open failed. A newer Orca's journal clears when this Orca is updated, so it is * not damage. */ function journalRefusalReason(error: unknown): JournalRefusalReason { diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index 3aacf477d790..6ab4a635de47 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -109,15 +109,18 @@ export async function rejectJournalQueuedSubmissions( export function journalQueuedRejectionRowBuilders( state: () => JournalReducerState, fence: number, - rejection: AgentJournalDispatchRejection + rejection: AgentJournalDispatchRejection, + which: (submission: AgentJournalSubmission) => boolean = () => true ): ((seq: number, ts: number) => JournalRow)[] { - return [...state().submissions.values()].filter(isQueuedAgentJournalSubmission).map((entry) => - journalDispatchRowBuilder(state, { - clientMessageId: entry.clientMessageId, - state: 'rejected', - ...rejection, - fence, - recovered: true - }) - ) + return [...state().submissions.values()] + .filter((entry) => isQueuedAgentJournalSubmission(entry) && which(entry)) + .map((entry) => + journalDispatchRowBuilder(state, { + clientMessageId: entry.clientMessageId, + state: 'rejected', + ...rejection, + fence, + recovered: true + }) + ) } diff --git a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts index 09a965b624ef..5a7c29ef361b 100644 --- a/src/main/native-chat/agent-session-journal/journal-provider-activity.ts +++ b/src/main/native-chat/agent-session-journal/journal-provider-activity.ts @@ -4,13 +4,15 @@ import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { JournalReducerState } from './journal-reducer' import type { JournalRow } from './journal-row-schema' -/** Saved output extends a turn; client actions and recovery writes do not. */ +/** Saved output extends a turn; client actions and recovery writes do not. Keyed by the fence of + * the generation that produced the item (`AgentSessionJournal.itemFence`), as a verdict reads it. */ export function observeJournalProviderActivity( state: Pick, row: JournalRow, itemId: string, body: AgentJournalItemBody, - savedAt: number + savedAt: number, + fence: number ): void { if (row.recovered || !isProviderActivity(body)) { return @@ -25,10 +27,7 @@ export function observeJournalProviderActivity( ) { return } - state.providerActivityAt.set( - row.fence, - Math.max(state.providerActivityAt.get(row.fence) ?? 0, savedAt) - ) + state.providerActivityAt.set(fence, Math.max(state.providerActivityAt.get(fence) ?? 0, savedAt)) } function isProviderActivity(body: AgentJournalItemBody): boolean { diff --git a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts index e87fb450154c..218832e2dffb 100644 --- a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts +++ b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts @@ -10,10 +10,11 @@ import { AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS, AGENT_SESSION_OPERATION_FUTURE_SKEW_MS } from '../../../shared/agent-session-host-authority' +import type { JournalWriteOptions } from './journal-host-database' import type { JournalRow } from './journal-row-schema' import { writeReceiptIfChanged, type JournalOperationReceipt } from './journal-row-writer' import type { JournalQueuedMessagesDeps, JournalSubmissionConsume } from './journal-store-contracts' -import { holdQueuedMessages } from './queued-message-holds' +import { holdQueuedMessages, type QueuedMessageHold } from './queued-message-holds' import { deriveQueuePauses, journalUserStopInForce, @@ -92,8 +93,7 @@ export class JournalQueuedMessages { return this.transact( (db) => updateQueuedMessageText(db, this.deps.sessionId, input), (result) => result.status === 'updated', - undefined, - () => this.editLeases.retire(input.messageId) + { adopted: () => this.editLeases.retire(input.messageId) } ) } @@ -137,17 +137,18 @@ export class JournalQueuedMessages { return row }, () => inserted, - receipt + { receipt } ) } /** Hold one waiting draft whose conversion failed. Stored on the row, so it * survives handle eviction and restart; withdraw and consume clear it in their * own UPDATE. */ - hold(input: { messageIds: readonly string[]; reason: QueuedMessageHoldReason }): Promise { + hold(input: QueuedMessageHold & JournalWriteOptions): Promise { return this.transact( (db) => holdQueuedMessages(db, { ...input, sessionId: this.deps.sessionId }), - (held) => held > 0 + (held) => held > 0, + input ).then(() => undefined) } @@ -216,7 +217,7 @@ export class JournalQueuedMessages { now: this.deps.now() }), (withdrawn) => withdrawn.length > 0, - receipt + { receipt } ) } @@ -240,12 +241,17 @@ export class JournalQueuedMessages { private transact( run: (db: Database.Database) => JournalWriteResult, changed: (result: T) => boolean, - receipt?: JournalOperationReceipt, - adopted?: () => void + { + receipt, + adopted, + background + }: { receipt?: JournalOperationReceipt; adopted?: () => void } & JournalWriteOptions = {} ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const result = this.deps.database().transaction(writeReceiptIfChanged(run, changed, receipt)) + const result = this.deps + .database() + .transaction(writeReceiptIfChanged(run, changed, receipt), background ? { background } : {}) if (changed(result)) { receipt?.committed() adopted?.() @@ -313,15 +319,12 @@ export class JournalQueuedMessages { } /** Applies owed settlements now, so a skipped live transition heals without a reopen. */ - settleOwed(): Promise { + settleOwed(options?: JournalWriteOptions): Promise { + const { sessionId, state, now } = this.deps return this.transact( - (db) => - settleOwedQueuedMessages(db, { - sessionId: this.deps.sessionId, - state: this.deps.state(), - now: this.deps.now() - }), - (settled) => settled > 0 + (db) => settleOwedQueuedMessages(db, { sessionId, state: state(), now: now() }), + (settled) => settled > 0, + options ).then(() => undefined) } @@ -342,7 +345,7 @@ export class JournalQueuedMessages { * crash → downgrade → upgrade, where the old build rejected the leftover with * no hook), then retention runs. */ - repairAndPrune(): Promise { + repairAndPrune(options?: JournalWriteOptions): Promise { // No draft, no work, and no write. if (this.deps.readOnly() || this.list().length === 0) { return Promise.resolve() @@ -361,7 +364,8 @@ export class JournalQueuedMessages { }) ) }, - (changed) => changed > 0 + (changed) => changed > 0, + options ).then(() => undefined) } } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 41da4e5f459b..df5ccb9a976b 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -51,7 +51,8 @@ export type JournalReducerState = { oldestSequence: number highestFence: number items: Map - /** Fence of the writer that created each item: the generation a running turn belongs to. */ + /** Each item's execution provenance (`JournalItemRow.ownerFence`): the generation whose work it + * is, which decides whether it is current. */ itemFences: Map /** Latest saved output per owner, rebuilt during the existing row fold. */ providerActivityAt: Map @@ -109,15 +110,16 @@ export function applyJournalRow( return } const itemId = resolveJournalItemId(state, row.itemId, row.body) - observeJournalProviderActivity(state, row, row.itemId, row.body, savedAt) acceptSubmissionFromProviderItem(state, row.itemId, itemId, row) upsertJournalItem( state, itemId, row.revision, journalRenderItem(itemId, row.revision, row.body, row, statedOrDerivedTurnScope(state, row)), - row.fence + { fence: row.fence, ownerFence: row.ownerFence } ) + const fence = state.itemFences.get(itemId) ?? row.fence + observeJournalProviderActivity(state, row, row.itemId, row.body, savedAt, fence) return } if (isJournalStopOrResumeRow(row)) { @@ -141,13 +143,17 @@ export function applyJournalRow( continue } const { revision, body } = mutation - observeJournalProviderActivity(state, row, mutation.itemId, body, savedAt) const itemId = resolveJournalItemId(state, mutation.itemId, body) acceptSubmissionFromProviderItem(state, mutation.itemId, itemId, row) const producer = journalBatchMutationProducer(row, mutation) const scope = statedOrDerivedTurnScope(state, mutation) const item = journalRenderItem(itemId, revision, body, row, scope, producer, sequenceIndex) - upsertJournalItem(state, itemId, revision, item, row.fence) + upsertJournalItem(state, itemId, revision, item, { + fence: row.fence, + ownerFence: mutation.ownerFence + }) + const fence = state.itemFences.get(itemId) ?? row.fence + observeJournalProviderActivity(state, row, mutation.itemId, body, savedAt, fence) } else { removeJournalItem(state, resolveItemId(state, mutation.itemId), mutation.revision) } diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index a6e3aa03d189..229e1656ec5f 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -13,6 +13,7 @@ import { agentSessionJournalProviderHandle } from '../../../shared/agent-session import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { journalItemOwnerFence, journalObservedBody } from './journal-item-provenance' import type { JournalReducerState } from './journal-reducer' import type { JournalDispatchRow, @@ -51,7 +52,12 @@ export function journalItemRowBuilder( state: () => JournalReducerState, address: AgentJournalItemIdentity | string, body: AgentJournalItemBody, - options: AgentJournalRowAttribution & { fence: number; observedAt?: number; recovered?: true }, + options: AgentJournalRowAttribution & { + fence: number + ownerFence?: number + observedAt?: number + recovered?: true + }, revisions?: Map ): RowBuilder { return (seq, ts) => @@ -61,6 +67,7 @@ export function journalItemRowBuilder( body, seq, fence: options.fence, + ownerFence: options.ownerFence, ts: options.observedAt ?? ts, recovered: options.recovered, linkage: options, @@ -183,10 +190,22 @@ export function journalLifecycleBatchRowBuilder( const revision = nextJournalItemRevision(current, itemId, runningRevisions) return journalLifecycleMutationRow( mutation.kind === 'item' - ? { ...mutation, body: turnEndAfterStop(current, resolved, mutation.body) } + ? { + ...mutation, + body: journalObservedBody( + current, + resolved, + turnEndAfterStop(current, resolved, mutation.body), + mutation.ownerFence + ) + } : mutation, itemId, - revision + revision, + journalItemOwnerFence(current, resolved, { + fence: options.fence, + ownerFence: mutation.kind === 'item' ? mutation.ownerFence : undefined + }) ) }) const row: JournalLifecycleBatchRow = { @@ -225,6 +244,8 @@ export function buildJournalItemRow( body: AgentJournalItemBody seq: number fence: number + /** The writer's provenance claim (`journalItemOwnerFence`); absent for host bookkeeping. */ + ownerFence?: number ts: number recovered?: true linkage?: AgentJournalProducerLinkage @@ -237,13 +258,19 @@ export function buildJournalItemRow( // A tombstoned row keeps its revision in `tombstones`, and the reducer drops // any item at or below it — so a re-add has to outrank the tombstone too. const revision = nextJournalItemRevision(input.state, itemId, input.revisions) - const body = turnEndAfterStop(input.state, resolved, input.body) + const body = journalObservedBody( + input.state, + resolved, + turnEndAfterStop(input.state, resolved, input.body), + input.ownerFence + ) return { kind: 'item', itemId, revision, body, ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), + ownerFence: journalItemOwnerFence(input.state, resolved, input), ...(input.recovered ? { recovered: input.recovered } : {}), turnScope: input.turnScope, ...agentJournalLinkageFields(input.linkage) diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index f018f6307f94..494495248ca2 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -80,12 +80,18 @@ export const AGENT_JOURNAL_EPOCH_REASONS = [ ] as const export type AgentJournalEpochReason = (typeof AGENT_JOURNAL_EPOCH_REASONS)[number] -export type JournalItemRow = JournalRowBase & { - kind: 'item' - itemId: string - revision: number - body: AgentJournalItemBody -} +/** The generation whose execution produced an item: execution provenance, distinct from the + * row's `fence`, which is writer authority. Rides the row like linkage, with no `v` bump: an older + * host ignores it. Absent on rows from before it: the item's creating fence stands in. */ +type JournalItemProvenance = { ownerFence?: number } + +export type JournalItemRow = JournalRowBase & + JournalItemProvenance & { + kind: 'item' + itemId: string + revision: number + body: AgentJournalItemBody + } export type JournalTombstoneRow = JournalRowBase & { kind: 'tombstone' @@ -196,13 +202,14 @@ export type JournalDispatchRow = JournalRowBase & { * Inline like the row base, and for the same reason no `v` bump: an older host * ignores the unknown keys and reads the mutation as root, as it always did. */ export type JournalLifecycleMutation = - | (AgentJournalProducerLinkage & { - kind: 'item' - itemId: string - revision: number - body: AgentJournalItemBody - turnScope?: AgentJournalTurnScope - }) + | (AgentJournalProducerLinkage & + JournalItemProvenance & { + kind: 'item' + itemId: string + revision: number + body: AgentJournalItemBody + turnScope?: AgentJournalTurnScope + }) | { kind: 'tombstone'; itemId: string; revision: number } /** One durable append whose nested mutations share the outer ordering facts. */ diff --git a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts index 6c8811eb4a5b..e6d25b21df9e 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts @@ -6,11 +6,13 @@ import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-sess export function dropUnusableRowAnnotations(record: Record): void { dropUnusableProducerLinkage(record) dropUnusableTurnScope(record) + dropUnusableOwnerFence(record) if (record.kind === 'lifecycle-batch' && Array.isArray(record.mutations)) { for (const mutation of record.mutations) { if (isPlainObject(mutation)) { dropUnusableProducerLinkage(mutation) dropUnusableTurnScope(mutation) + dropUnusableOwnerFence(mutation) } } } @@ -56,6 +58,14 @@ function dropUnusableTurnScope(record: Record): void { } } +/** Provenance this build cannot read, removed like linkage: the item's creating fence stands in. */ +function dropUnusableOwnerFence(record: Record): void { + const value = record.ownerFence + if (value !== undefined && !(Number.isSafeInteger(value) && Number(value) >= 0)) { + delete record.ownerFence + } +} + /** Context facts this build cannot read, removed from the turn row that carries * them. Same reasoning as linkage: they are an annotation on the turn, and * rejecting the row for them would truncate the journal from that row on. */ diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 4975155676b7..0931ec5a530f 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -1,10 +1,12 @@ import type Database from '../../sqlite/sync-database' import { insertJournalRow } from './journal-row-table' import type { JournalHostDatabase } from './journal-host-database' +import type { JournalWriteOptions } from './journal-database' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { JournalRow } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' import type { JournalWriteBody } from './journal-write-queue' +import type { JournalEpochFounding } from './journal-epoch-founding' /** Runs between BEGIN IMMEDIATE and COMMIT, on the SAME connection as the row * insert; a throw rolls the whole append back. Synchronous by construction so @@ -58,6 +60,8 @@ export type JournalRowWriterDeps = { inTransaction?: JournalRowTransactionHook /** After any rollback, so a cache filled inside the transaction cannot outlive it. */ rolledBack?: () => void + /** The chat's held first epoch, written with its first row (`journal-epoch-founding.ts`). */ + founding?: Pick } const BOOKKEEPING_SAVEPOINT = 'journal_row_bookkeeping' @@ -68,7 +72,8 @@ export class JournalRowWriter { enqueue( build: (seq: number, ts: number) => JournalRow, hook?: JournalRowTransactionHook, - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) @@ -78,11 +83,12 @@ export class JournalRowWriter { try { // One INSERT: the chat's epoch pointer moves only when the epoch does. this.deps.database().transaction((db) => { + this.deps.founding?.inTransaction(db) insertJournalRow(db, this.deps.sessionId, row, savedAt) hook?.(db, row) receipt?.write(db, row) this.runBookkeeping(db, row) - }) + }, options) } catch (error) { this.deps.rolledBack?.() throw error @@ -90,6 +96,7 @@ export class JournalRowWriter { // COMMIT landed, so the row is durable: adopt it before anything that can // fail. Rejecting here instead would leave the next append reusing a // sequence the table already holds. The ledger first: it cannot throw, the fold can. + this.deps.founding?.settled() receipt?.committed() this.deps.commit([row], savedAt) return row @@ -100,15 +107,17 @@ export class JournalRowWriter { * durable unless all are, so no reader ever meets some without the rest. */ enqueueRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[], - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): Promise { - return this.deps.serialize(() => this.writeRows(plan, receipt)) + return this.deps.serialize(() => this.writeRows(plan, receipt, options)) } /** `enqueueRows`' write, for a caller already running at its own turn in the queue. */ writeRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[], - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): JournalRow[] { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) const first = this.deps.nextSequence() @@ -122,29 +131,45 @@ export class JournalRowWriter { } try { this.deps.database().transaction((db) => { + this.deps.founding?.inTransaction(db) for (const row of rows) { insertJournalRow(db, this.deps.sessionId, row, ts) this.runBookkeeping(db, row) } receipt?.write(db) - }) + }, options) } catch (error) { this.deps.rolledBack?.() throw error } + this.deps.founding?.settled() receipt?.committed() this.deps.commit(rows, ts) return rows } + /** Writes the chat's held first epoch on its own, for an acquisition: a chat whose agent started + * is kept even if nothing else is written. Nothing when it is founded already. */ + found(): Promise { + return this.deps.serialize(() => { + if (!this.deps.founding?.held()) { + return + } + assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) + this.deps.database().transaction((db) => this.deps.founding?.inTransaction(db)) + this.deps.founding.settled() + }) + } + /** Assign the next sequence, make the row durable, and fold it through the SAME reducer * replay uses — all inside one serialized step — answering where the row landed. */ append( build: (seq: number, ts: number) => JournalRow, hook?: JournalRowTransactionHook, - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): Promise { - return this.enqueue(build, hook, receipt).then((row) => ({ + return this.enqueue(build, hook, receipt, options).then((row) => ({ epoch: row.epoch, sequence: row.seq })) diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 7ef45140b5af..108e1599886a 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -9,6 +9,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import { JournalEpochController } from './journal-epoch-controller' +import { JournalEpochFounding } from './journal-epoch-founding' import { JournalItemAppender } from './journal-item-appender' import { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' import type { JournalLoad } from './journal-open' @@ -63,7 +64,9 @@ export type JournalStoreCollaborators = { } export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { + const founding = new JournalEpochFounding(host.identity) const epochController = new JournalEpochController({ + founding, identity: host.identity, now: host.now, mintEpoch: host.mintEpoch, @@ -112,23 +115,21 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), - rolledBack: () => queuedMessages.invalidate() + rolledBack: () => queuedMessages.invalidate(), + founding }) const lifecycleBatchAppender = new JournalLifecycleBatchAppender({ state: host.state, cursor: host.cursor, - enqueueRows: (plan) => rowWriter.enqueueRows(plan) + enqueueRows: (plan, options) => rowWriter.enqueueRows(plan, undefined, options) }) return { epochController, queuedMessages, stopMarks: new JournalStopMarks({ state: host.state }), - // Behind the stored fact: settles drafts whose consumed submission the loaded journal shows - // refused (a downgrade wrote no hook), then prunes. Bookkeeping, never failing the open. - restore: () => - restoreJournalStore(host, { epochController }).then(() => - queuedMessages.repairAndPruneAtOpen() - ), + // Reads only: the drafts' repair and prune are startup's (`repairAndPrune`), and a chat with + // no journal yet is founded by its first write. + restore: () => restoreJournalStore(host, founding), rowWriter, submissionWriter: new JournalSubmissionWriter({ state: host.state, diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index f9f7502ea8b8..7d924edcdff0 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -8,6 +8,7 @@ import type { AgentJournalProducerLinkage, AgentJournalResetReason, AgentJournalRowAttribution, + AgentJournalSubmission, AgentJournalTurnScope, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -34,6 +35,8 @@ export type ResolveDispatchInput = { clientMessageId: string fence: number recovered?: true + /** Bookkeeping no person waits on (`JournalWriteOptions`); not written to the row. */ + background?: true } & /** A null identity: the provider took the message without echoing an item of its own, as a * conversation command it carries out in place. */ @@ -59,6 +62,9 @@ export type JournalAppendResult = { export type JournalItemAppendOptions = AgentJournalRowAttribution & { fence: number + /** The generation whose execution produced the item: set by a provider's observation, and by + * nothing else. Host bookkeeping leaves it out and keeps the item's (`journalItemOwnerFence`). */ + ownerFence?: number observedAt?: number recovered?: true } @@ -76,6 +82,9 @@ export type JournalLifecycleBatchInput = { settlementId: string mutations: readonly JournalLifecycleMutationInput[] fence: number + /** A provider's own batch: the generation that produced every item it writes + * (`JournalItemAppendOptions.ownerFence`). */ + ownerFence?: number recovered?: true /** Submission verdicts belonging to this settlement, committed before its item rows. */ dispatches?: readonly ResolveDispatchInput[] @@ -83,11 +92,27 @@ export type JournalLifecycleBatchInput = { * follows the messages it failed, and no reader meets one without the other. With none still * queued, the batch is not written either. */ rejectsQueued?: AgentJournalDispatchRejection + /** Narrows `rejectsQueued` to the queued sends this names. */ + rejectsQueuedOnly?: (submission: AgentJournalSubmission) => boolean +} + +/** A lifecycle batch whose rows are chosen at its own turn in the write queue, so a write queued + * ahead of it (an answer, a Stop) is what it plans from. */ +export type JournalPlannedLifecycleBatchInput = Pick< + JournalLifecycleBatchInput, + 'settlementId' | 'fence' | 'recovered' +> & { + /** Bookkeeping no person waits on (`JournalWriteOptions`). */ + background?: true + plan: () => { + mutations: readonly JournalLifecycleMutationInput[] + dispatches: readonly ResolveDispatchInput[] + } } export type JournalResolvedLifecycleBatchInput = Omit< JournalLifecycleBatchInput, - 'mutations' | 'rejectsQueued' | 'dispatches' + 'mutations' | 'rejectsQueued' | 'rejectsQueuedOnly' | 'dispatches' > & { /** Read from the fold with every earlier write landed; may return none. */ resolve: () => readonly JournalLifecycleMutationInput[] @@ -120,6 +145,8 @@ export type JournalSubmissionConsume = { /** The queue's own send: refused in the consume's transaction while the queue's pause holds * the card. Send-now omits it. */ yieldsToPause?: true + /** The queue's automatic send is bookkeeping no person waits on (`JournalWriteOptions`). */ + background?: true } export type JournalItemAppendInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-store-open.ts b/src/main/native-chat/agent-session-journal/journal-store-open.ts index 1a5db4fe0d27..79fb36b64d77 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-open.ts @@ -1,23 +1,14 @@ -import type { - AgentJournalItemBody, - AgentJournalItemIdentity -} from '../../../shared/agent-session-journal-types' import type { JournalLoad } from './journal-open' import { failLoadOnUnloadableJournal } from './journal-open-failure' -import { staleSubagentRosterRevisions } from './journal-subagent-liveness' -export async function openJournalStoreState(input: { +/** Replays the saved journal. What a gone writer left live is read as saved: its owner's exit, the + * next acquisition, or startup settles it, never this open. */ +export function openJournalStoreState(input: { sessionId: string replay: () => JournalLoad | null start: () => void adopt: (loaded: JournalLoad) => void - appendItem: ( - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody, - fence: number - ) => Promise - highestFence: () => number -}): Promise { +}): void { const loaded = input.replay() // An epoch named but holding no row (a crash inside an older build's repair) has nothing to // keep, so it is founded afresh like a chat with no journal; no row is deleted. @@ -27,22 +18,4 @@ export async function openJournalStoreState(input: { } failLoadOnUnloadableJournal(input.sessionId, loaded) input.adopt(loaded) - await settleStaleSubagentRosters(input, loaded) -} - -/** Retires a `working` subagent roster the previous host never got to settle. */ -async function settleStaleSubagentRosters( - input: { - appendItem: ( - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody, - fence: number - ) => Promise - highestFence: () => number - }, - loaded: JournalLoad -): Promise { - for (const revision of staleSubagentRosterRevisions(loaded.state.items.values())) { - await input.appendItem(revision.identity, revision.body, input.highestFence()) - } } diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index e8cb28e6d321..94b9805305f9 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -1,13 +1,8 @@ -// Bringing a store's in-memory state up from disk. -// -// Split out of the store for the same reason its collaborators were: this is the -// ORDERING between replay and the notices an open owes, and none of it belongs -// to the store's public surface. Every step here reads or writes through the -// same host the collaborators use, so the store keeps the state and this owns -// the sequence. +// Bringing a store's in-memory state up from disk. It reads and never writes: a chat with no +// journal yet gets an empty one held in memory (`journal-epoch-founding.ts`), and what a gone +// writer left live is settled by the ownership event that ended it, never by this open. -import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' -import type { JournalEpochController } from './journal-epoch-controller' +import type { JournalEpochFounding } from './journal-epoch-founding' import { replayJournal } from './journal-open' import { journalOpenRefusalError } from './journal-open-failure' import type { JournalStoreHost } from './journal-store-collaborators' @@ -16,7 +11,7 @@ import { AgentSessionJournalError } from './journal-write-guards' export async function restoreJournalStore( host: JournalStoreHost, - collaborators: { epochController: JournalEpochController } + founding: Pick ): Promise { const database = host.database() if (database.readOnly) { @@ -28,18 +23,10 @@ export async function restoreJournalStore( ) ) } - return openJournalStoreState({ + openJournalStoreState({ sessionId: host.identity.sessionId, replay: () => replayJournal(database.db, host.identity.sessionId), - start: () => collaborators.epochController.start('session_created', 0), - adopt: host.adopt, - // Journal-open revisions are host recovery, not new provider output. - appendItem: (identity, body, fence) => - host.journal().appendItem(identity, body, { - fence, - turnScope: AGENT_JOURNAL_THREAD_SCOPE, - recovered: true - }), - highestFence: () => host.state().highestFence + start: () => host.adopt(founding.hold(host.mintEpoch(), host.now())), + adopt: host.adopt }) } diff --git a/src/main/native-chat/agent-session-journal/journal-store.test.ts b/src/main/native-chat/agent-session-journal/journal-store.test.ts index 5c5637de2dfa..6b2488c259a6 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.test.ts @@ -141,7 +141,7 @@ describe('sequences', () => { const rendered = (): string | null => activeStructuredAgentSessionTurnId(journal.snapshot().items) const bothAgreeOn = async (turnId: string | null): Promise => { - expect(journal.activeTurnId()).toBe(turnId) + expect(journal.runningTurn()?.turnId ?? null).toBe(turnId) expect(rendered()).toBe(turnId) } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 8fe8c970b0a7..2ef3d678ddd3 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -1,6 +1,7 @@ // Append-only journal store for one agent session. It owns the chat's fold and write queue, and no // connection: every statement goes through the host's one journal database. +import { QueuedMessageReopenFloor } from './queued-message-reopen-floor' import type { AgentJournalDispatchRejection } from '../../../shared/agent-session-failure-words' import { randomUUID } from 'node:crypto' import type { @@ -13,7 +14,6 @@ import type { AgentJournalSubmission, AgentJournalThreadGoal, AgentJournalTurnLifecycle, - AgentJournalTurnScope, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' @@ -21,13 +21,12 @@ import { currentAgentSessionThreadGoalBySequence } from '../../../shared/agent-s import type { AgentSessionContextUsage } from '../../../shared/agent-session-context-usage' import { latestStructuredAgentContextFacts } from '../../../shared/structured-agent-session-context-usage' import { - activeStructuredAgentSessionTurnIdBySequence, - liveStructuredAgentSessionTurnScope, - newestStructuredAgentSessionTurnBySequence + newestStructuredAgentSessionTurnBySequence, + runningStructuredAgentSessionTurnItemBySequence } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' import { readJournalSince } from './journal-cursor' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import { journalRowsAfterReader, type JournalLoad } from './journal-open' import { markJournalPendingSubmissionsUnknown, @@ -48,6 +47,7 @@ import type { JournalItemAppendOptions, JournalItemLinkageVisitor, JournalLifecycleBatchInput, + JournalPlannedLifecycleBatchInput, JournalReadSince, JournalSubmissionConsume, JournalSubmissionInput, @@ -60,7 +60,7 @@ import { journalQueueReopenRowBuilder, journalStopEventRowBuilder } from './journal-stop-and-resume-rows' -import type { AgentJournalEpochReason, JournalStopEvent } from './journal-row-schema' +import type { AgentJournalEpochReason, JournalRow, JournalStopEvent } from './journal-row-schema' import type { JournalOperationReceipt, JournalRowTransactionHook, @@ -86,8 +86,8 @@ export class AgentSessionJournal { private state: JournalReducerState private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } - private reopenUnmarked: AgentJournalCursor | null = null - private onCommitted: (() => void) | null = null + private readonly reopen: QueuedMessageReopenFloor + private onCommitted: ((rows?: readonly JournalRow[]) => void) | null = null private readonly queue: JournalWriteQueue private readonly rowWriter: JournalRowWriter private readonly epochController: JournalEpochController @@ -127,7 +127,7 @@ export class AgentSessionJournal { for (const row of rows) { applyJournalRow(this.state, row, savedAt) } - this.onCommitted?.() + this.onCommitted?.(rows) }, notifyCommitted: () => this.onCommitted?.(), journal: () => this, @@ -140,6 +140,7 @@ export class AgentSessionJournal { this.submissionWriter = collaborators.submissionWriter this.stepWriter = collaborators.stepWriter this.queuedMessages = collaborators.queuedMessages + this.reopen = new QueuedMessageReopenFloor(this.queuedMessages) this.stopMarks = collaborators.stopMarks this.restore = collaborators.restore this.context = new JournalContextController({ @@ -167,17 +168,25 @@ export class AgentSessionJournal { ) } - /** Where the reopen's pause begins when this handle could not write its mark: where the mark - * would have gone. Null once a mark is written. Per handle, so the next open marks again. */ - reopenFloor(): AgentJournalCursor | null { - return this.reopenUnmarked - } + /** Where this handle opened the journal (`wroteBeforeOpen`). */ + openedAt = (): AgentJournalCursor => this.openedThrough + + /** Where the reopen's pause begins while no mark holds it: where this handle's mark would go. + * Null once a mark is written. Per handle, so the next open derives it again. */ + reopenFloor = (): AgentJournalCursor | null => this.reopen.get() async open(): Promise { await this.restore() this.openedThrough = this.cursor() } + /** A conversation's open, which always follows the chat stopping (a quit, a crash or a close): + * a card it finds waits for a turn from where this handle opened, derived with nothing written, + * unless an earlier mark already holds it. The mark that follows records the same start. */ + holdReopenFromOpen(): void { + this.reopen.holdFromOpen(this.openedThrough) + } + /** Refuses every later write and resolves once the admitted ones have landed. Holds no * connection, so there is nothing to release and nothing that can fail. */ close(): Promise { @@ -188,7 +197,7 @@ export class AgentSessionJournal { /** Told of every durable change, epoch replacements included, so a reader learns of a write * without its writer saying so. One listener: a later call replaces it. It must not throw. */ - observeCommits(listener: () => void): void { + observeCommits(listener: (rows?: readonly JournalRow[]) => void): void { this.onCommitted = listener } @@ -223,14 +232,11 @@ export class AgentSessionJournal { } } - /** The turn this journal has published as running — the same read a client's snapshot gives, - * without materialising one. */ - activeTurnId = (): string | null => - activeStructuredAgentSessionTurnIdBySequence(this.state.items.values()) - - /** Where a row written now belongs: the running turn, or the conversation. */ - liveTurnScope = (): AgentJournalTurnScope => - liveStructuredAgentSessionTurnScope(this.state.items.values()) + /** The turn this journal records as running, with its record's item, whichever generation + * opened it — the same read a client's snapshot gives, without materialising one. Whether it is + * current work is the host projection's to say (`structuredAgentSessionCurrentWork`). */ + runningTurn = (): { item: AgentJournalRenderItem; turnId: string } | null => + runningStructuredAgentSessionTurnItemBySequence(this.state.items.values()) /** The newest turn record whatever state it settled in, for readers that need the outcome. */ newestTurn = (): AgentJournalTurnLifecycle | null => @@ -247,12 +253,11 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt - /** Fence of the writer that created the item, while it is in the timeline. */ + /** The item's execution provenance (`JournalItemRow.ownerFence`), while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) /** Latest saved output at this fence, excluding client actions and recovery bookkeeping. */ - lastProviderActivityAt = (fence: number): number | undefined => - this.state.providerActivityAt.get(fence) + lastProviderActivityAt = (fence: number) => this.state.providerActivityAt.get(fence) submissions = (): AgentJournalSubmission[] => [...this.state.submissions.values()] @@ -315,26 +320,29 @@ export class AgentSessionJournal { } /** This open found waiting cards an earlier handle wrote (`queued-message-pause.ts`). */ - appendQueueReopen(fence: number, since?: number): Promise { - return this.rowWriter.append(journalQueueReopenRowBuilder(() => this.state, fence, since)) + appendQueueReopen(fence: number, since?: number, options?: JournalWriteOptions) { + const build = journalQueueReopenRowBuilder(() => this.state, fence, since) + return this.rowWriter.append(build, undefined, undefined, options) } - /** Marks the reopen when a card waits or is mid-hand-off (it may come back to waiting), from - * `since` when the chat stopped before now; a failed write leaves where the mark would have gone - * as the pause's start (`reopenFloor`), and throws. */ - async markQueueReopen(fence: number, since?: number): Promise { - if (this.queuedMessages.awaitReopenMark()) { - const sequence = since ?? this.state.lastSequence + 1 - this.reopenUnmarked = { epoch: this.state.epoch, sequence } - await this.appendQueueReopen(fence, since) - this.reopenUnmarked = null - } + /** Marks the reopen (`QueuedMessageReopenFloor.mark`), from `since` when the chat stopped before + * now; a failed write leaves the pause's start where the mark would have gone, and throws. */ + markQueueReopen(fence: number, since?: number, options?: JournalWriteOptions): Promise { + const at = { epoch: this.state.epoch, sequence: since ?? this.state.lastSequence + 1 } + return this.reopen.mark(at, () => this.appendQueueReopen(fence, since, options)) } appendLifecycleBatch(input: JournalLifecycleBatchInput): Promise { return this.lifecycleBatchAppender.append(input) } + /** A lifecycle batch planned at its own turn in the queue (`JournalPlannedLifecycleBatchInput`). */ + appendPlannedLifecycleBatch = (input: JournalPlannedLifecycleBatchInput) => + this.lifecycleBatchAppender.appendPlanned(input) + + /** An acquisition's write of a chat's first epoch (`JournalRowWriter.found`). */ + foundEpoch = (): Promise => this.rowWriter.found() + /** Several writes as one turn in the queue; see `JournalStepWriter`. */ appendSteps: JournalStepWriter['append'] = (steps) => this.stepWriter.append(steps) @@ -386,9 +394,10 @@ export class AgentSessionJournal { replaceEpochItems( reason: AgentJournalEpochReason, fence: number, - items: readonly JournalReplacementItem[] + items: readonly JournalReplacementItem[], + options?: JournalWriteOptions ): Promise { - return this.epochController.replace(reason, fence, items) + return this.epochController.replace(reason, fence, items, options) } private adoptLoadedJournal(loaded: JournalLoad): void { diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts index d9362e6008ad..750598e488e1 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts @@ -21,7 +21,8 @@ import { } from '../../codex/codex-subagent-roster' import type { openAgentSessionJournal } from './journal-store-factory' import { createTrackedJournalOpener } from './journal-host-database-test-support' -import { lostLiveWorkJournalBody, staleSubagentRosterRevisions } from './journal-subagent-liveness' +import { lostLiveWorkJournalBody } from './journal-subagent-liveness' +import { settleStaleStructuredAgentSessionState } from '../agent-session-wire/structured-agent-session-stale-state-settlement' import { endedUnseenMessageBody } from './journal-terminal-settlement' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' @@ -55,6 +56,26 @@ async function open(overrides: Partial { + const body = lostLiveWorkJournalBody(item.body) + return body ? [{ body }] : [] + }) +} + +/** What the next ownership event settles once the generation that wrote at fence 0 is gone. */ +async function settleGoneGeneration(journal: Awaited>): Promise { + await settleStaleStructuredAgentSessionState({ + journal, + sessionId: 'session-1', + fence: 1, + acquisitionGeneration: null, + deathEvidence: null, + below: 1 + }) +} + /** The row as the producer writes it: the structured block plus its twin. */ function rosterRow(agents: NativeChatSubagentEntry[]) { return { @@ -126,9 +147,9 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) -describe('staleSubagentRosterRevisions', () => { +describe('lostLiveWorkJournalBody', () => { it('settles a child the previous host left working, and moves the twin with it', () => { - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ renderItem([ { id: 'a', label: 'read_readme', state: 'working', startedAt: 10 }, { id: 'b', label: 'read_package', state: 'completed', startedAt: 10, settledAt: 20 } @@ -146,7 +167,7 @@ describe('staleSubagentRosterRevisions', () => { it('settles a background task the previous host left live, and moves the twin with it', () => { const row = backgroundTaskRow() - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ { itemId: agentJournalItemKey(row.identity), revision: 1, @@ -164,7 +185,7 @@ describe('staleSubagentRosterRevisions', () => { // The child stopped being observable at an unknown moment. A stamp taken now // would report the time the app was down as how long the child ran. it('records no terminal timestamp for a child whose run length is unknown', () => { - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ renderItem([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }]) ]) @@ -173,7 +194,7 @@ describe('staleSubagentRosterRevisions', () => { it('records no terminal timestamp for a background task whose run length is unknown', () => { const row = backgroundTaskRow(backgroundTaskBlock({ settledAt: 20 })) - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ { itemId: agentJournalItemKey(row.identity), revision: 1, @@ -188,15 +209,13 @@ describe('staleSubagentRosterRevisions', () => { it('owes nothing for a roster whose children all settled', () => { expect( - staleSubagentRosterRevisions([ - renderItem([{ id: 'a', label: 'read', state: 'completed', settledAt: 20 }]) - ]) + lostLiveWork([renderItem([{ id: 'a', label: 'read', state: 'completed', settledAt: 20 }])]) ).toEqual([]) }) it('leaves rows that carry no roster alone', () => { expect( - staleSubagentRosterRevisions([ + lostLiveWork([ { itemId: 'orca:plain', revision: 1, @@ -207,19 +226,9 @@ describe('staleSubagentRosterRevisions', () => { ]) ).toEqual([]) }) - - // Appending under a fresh identity would add a second row rather than revise - // the one on disk, so an unaddressable key is left exactly as it is. - it('skips a row whose key cannot be parsed back to its identity', () => { - expect( - staleSubagentRosterRevisions([ - { ...renderItem([{ id: 'a', label: 'r', state: 'working' }]), itemId: 'not-a-key' } - ]) - ).toEqual([]) - }) }) -describe('journal reopen after the writing host is gone', () => { +describe('the settlement after the writing host is gone', () => { it('writes composed roster, task, reasoning and text-twin corrections as one settlement step', async () => { const journal = await open() const roster = rosterRow([ @@ -309,6 +318,7 @@ describe('journal reopen after the writing host is gone', () => { clock += 3_600_000 const reopened = await open() + await settleGoneGeneration(reopened) const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(afterRestart.recovered).toBe(true) @@ -330,6 +340,7 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const reopened = await open() + await settleGoneGeneration(reopened) expect(reopened.snapshot().items).toHaveLength(before) expect(reopened.snapshot().items.at(-1)?.revision).toBe(2) }) @@ -347,13 +358,14 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const reopened = await open() + await settleGoneGeneration(reopened) const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(taskOf(afterRestart.body)).toMatchObject({ state: 'unverifiable' }) expect(twinOf(afterRestart.body)).toBe('Background command "sleep 20" stopped reporting') }) - it('writes nothing on a second reopen once every child is settled', async () => { + it('writes nothing on a second settlement once every child is settled', async () => { const live = await open() const row = rosterRow([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }]) await live.appendItem(row.identity, row.body, { @@ -363,10 +375,12 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const once = await open() + await settleGoneGeneration(once) const revision = once.snapshot().items.at(-1)?.revision await once.close() const twice = await open() + await settleGoneGeneration(twice) expect(twice.snapshot().items.at(-1)?.revision).toBe(revision) }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts index 9a99ecf8d65d..1d40cb085aa7 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts @@ -4,24 +4,17 @@ // the children, and sweeps whatever is still `working` when the provider goes // away. A host that DIED — crash, quit, force-restart — does neither: its last // revision goes on saying `working`, and nothing replays those children, so no -// later event can ever settle them. Opening the journal is the one moment a new -// host can state the truth about the old one: contact was lost. That is +// later event can ever settle them. The leftover settlement at the next ownership +// event states the truth about the old one: contact was lost. That is // `unverifiable`, never a synthesized exit — see // `docs/reference/ssh-execution-boundary.md`. // // Reconciles JOURNAL ROWS, not roster state. A producer that continues a roster // from the journal reads these rows after this revision, so it inherits the -// verdict rather than the dead host's `working`. +// verdict rather than the dead host's `working`. Each entry is settled only when the +// generation that last observed it is the one that ended (`journal-item-provenance.ts`). -import { - agentJournalItemKey, - parseAgentJournalItemKey -} from '../../../shared/agent-session-journal-item-key' -import type { - AgentJournalItemBody, - AgentJournalItemIdentity, - AgentJournalRenderItem -} from '../../../shared/agent-session-journal-types' +import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' import { isSubagentGroupFallbackText, normalizeSubagentState, @@ -40,61 +33,51 @@ import { type NativeChatSubagentGroupBlock } from '../../../shared/native-chat-types' -export type JournalSubagentLivenessRevision = { - identity: AgentJournalItemIdentity - body: AgentJournalItemBody -} +/** Which entries the lost session ran, by each entry's own provenance + * (`journal-item-provenance.ts`); absent, every entry. */ +export type LostLiveWorkEntries = (entry: { ownerFence?: number }) => boolean -/** The revisions a reopened journal owes: one per row still claiming a live - * child. Empty — the common case — when nothing was left mid-flight. */ -export function staleSubagentRosterRevisions( - items: Iterable -): JournalSubagentLivenessRevision[] { - const revisions: JournalSubagentLivenessRevision[] = [] - for (const item of items) { - const body = lostLiveWorkJournalBody(item.body) - if (!body) { - continue - } - // A key that will not parse cannot be re-addressed, and appending under a - // fresh identity would duplicate the row rather than revise it. - const identity = parseAgentJournalItemKey(item.itemId) - if (!identity || agentJournalItemKey(identity) !== item.itemId) { - continue - } - revisions.push({ identity, body }) - } - return revisions -} +const EVERY_ENTRY: LostLiveWorkEntries = () => true /** The row once the host lost the session running its live work: every working child and * in-flight background task `unverifiable`, plain-text twins restated. Null when none is live. + * `lost` narrows it to the entries the lost session ran: a later generation's stay as they are. * `lostAt` is when a live host saw the session end; a reopened journal cannot know it. */ export function lostLiveWorkJournalBody( body: AgentJournalItemBody, - lostAt?: number + { lost = EVERY_ENTRY, lostAt }: { lost?: LostLiveWorkEntries; lostAt?: number } = {} ): AgentJournalItemBody | null { - if (body.kind !== 'message' || !body.blocks.some(hasStaleLiveWork)) { + if ( + body.kind !== 'message' || + !body.blocks.some((block) => hasWorkingChild(block, lost) || hasLiveBackgroundTask(block, lost)) + ) { return null } - return { ...body, blocks: settleBlocks(body.blocks, lostAt) } + return { ...body, blocks: settleBlocks(body.blocks, lost, lostAt) } } -function hasStaleLiveWork(block: NativeChatBlock): boolean { - return hasWorkingChild(block) || hasLiveBackgroundTask(block) +function isWorkingChild(agent: NativeChatSubagentGroupBlock['agents'][number]): boolean { + return normalizeSubagentState(agent.state) === 'working' } -function hasWorkingChild(block: NativeChatBlock): block is NativeChatSubagentGroupBlock { +function hasWorkingChild( + block: NativeChatBlock, + lost: LostLiveWorkEntries +): block is NativeChatSubagentGroupBlock { return ( isSubagentGroupBlock(block) && - block.agents.some((agent) => normalizeSubagentState(agent.state) === 'working') + block.agents.some((agent) => isWorkingChild(agent) && lost(agent)) ) } -function hasLiveBackgroundTask(block: NativeChatBlock): block is NativeChatBackgroundTaskBlock { +function hasLiveBackgroundTask( + block: NativeChatBlock, + lost: LostLiveWorkEntries +): block is NativeChatBackgroundTaskBlock { return ( isBackgroundTaskBlock(block) && - !isSettledBackgroundTaskState(normalizeBackgroundTaskState(block.state)) + !isSettledBackgroundTaskState(normalizeBackgroundTaskState(block.state)) && + lost(block) ) } @@ -102,13 +85,17 @@ function hasLiveBackgroundTask(block: NativeChatBlock): block is NativeChatBackg * unknown moment, and stamping the reopen would report the time the app was * down as how long it ran. Readers already draw an unverifiable child with no * stamp as having no known run length. */ -function settleBlocks(blocks: readonly NativeChatBlock[], lostAt?: number): NativeChatBlock[] { +function settleBlocks( + blocks: readonly NativeChatBlock[], + lost: LostLiveWorkEntries, + lostAt?: number +): NativeChatBlock[] { const backgroundTaskTwinText = new Map() const settled = blocks.map((block) => { - if (hasWorkingChild(block)) { - return settleGroup(block, lostAt) + if (hasWorkingChild(block, lost)) { + return settleGroup(block, lost, lostAt) } - if (hasLiveBackgroundTask(block)) { + if (hasLiveBackgroundTask(block, lost)) { const next = settleBackgroundTask(block) backgroundTaskTwinText.set( backgroundTaskFallbackText(block), @@ -140,12 +127,13 @@ function settleBlocks(blocks: readonly NativeChatBlock[], lostAt?: number): Nati function settleGroup( block: NativeChatSubagentGroupBlock, + lost: LostLiveWorkEntries, lostAt?: number ): NativeChatSubagentGroupBlock { return { ...block, agents: block.agents.map((agent) => - normalizeSubagentState(agent.state) === 'working' + isWorkingChild(agent) && lost(agent) ? { ...agent, state: 'unverifiable' as const, diff --git a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts index 7b03b4f457f8..1369f57892ee 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts @@ -46,7 +46,8 @@ export function applyJournalSubmission( itemId, 0, journalRenderItem(itemId, 0, row.body, row, turnScope), - row.fence + // A person's message is no generation's work: its provenance is the fence that accepted it. + { fence: row.fence } ) } diff --git a/src/main/native-chat/agent-session-journal/journal-submission-writer.ts b/src/main/native-chat/agent-session-journal/journal-submission-writer.ts index 1fc0ac21d0da..d397a74f6dc3 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-writer.ts @@ -6,6 +6,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import type { JournalQueuedMessages } from './journal-queued-messages' +import { backgroundWrite } from './journal-database' import { journalSubmissionHook, type JournalAttachmentClaim } from './journal-submission-hook' import type { JournalReducerState } from './journal-reducer' import { journalDispatchRowBuilder, journalSubmissionRowBuilder } from './journal-row-builders' @@ -48,7 +49,8 @@ export class JournalSubmissionWriter { return rowWriter.append( journalSubmissionRowBuilder(state, identity, input, consume), journalSubmissionHook(queuedMessages, claimAttachments, input, consume), - receipt + receipt, + backgroundWrite(consume?.background) ) } @@ -64,6 +66,11 @@ export class JournalSubmissionWriter { input: ResolveDispatchInput, hook?: JournalRowTransactionHook ): Promise { - return this.deps.rowWriter.append(journalDispatchRowBuilder(this.deps.state, input), hook) + return this.deps.rowWriter.append( + journalDispatchRowBuilder(this.deps.state, input), + hook, + undefined, + backgroundWrite(input.background) + ) } } diff --git a/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts b/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts index dba12dc49d3f..3d9f3b0ce85b 100644 --- a/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts @@ -117,7 +117,7 @@ it.each([ turnId: 'old', state: 'interrupted' }) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() const ended = journal.snapshot() await settle(observedAt, lastProvenAliveAt) expect(journal.snapshot()).toEqual(ended) @@ -145,7 +145,7 @@ it.each([{ kind: 'tool-call' }, { providerTurnId: '' }])( state: 'interrupted', completedAt: 1_000 }) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() await expectSendStillWritable() } ) @@ -329,7 +329,7 @@ it.each([ } expect(journal.itemBody('orca:extension-7')).toHaveProperty('laterField', { kind: 'tool-call' }) expect(journal.itemBody('orca:extension-6')).toHaveProperty('contextUsage', laterContextUsage) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() expect(journal.snapshot().items).toHaveLength(inputs.length + 1) const ended = journal.snapshot() await settle(observedAt, lastProvenAliveAt) diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts index fd5a774c7bf2..3d5b208f0277 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts @@ -18,13 +18,15 @@ import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import { agentSessionSendBodyFingerprint } from '../../../shared/structured-agent-session-send-mutation' import type { AgentSessionJournal } from './journal-store' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import type { JournalRowTransactionHook } from './journal-row-writer' import type { QueuedMessagePositionMove } from './queued-message-positions' /** Why the host can no longer hand a send over, which also says which sends it is. */ export type UnsentSendHold = - /** At open, and the delivery loop's first step: what an earlier host process accepted. */ - | { cause: 'hostRestarted' } + /** What an earlier host process accepted: by default, what was on disk when this handle opened; + * `which` names it for a handle opened after this process accepted sends of its own. */ + | { cause: 'hostRestarted'; which?: (submission: AgentJournalSubmission) => boolean } /** A close of the chat; `which` narrows it to what a close that did not complete closed. */ | { cause: 'chatClosed'; which?: (submission: AgentJournalSubmission) => boolean } @@ -87,8 +89,11 @@ export async function holdUnsentSends( /** In place of the queued sends: those a child that ended before it answered its start was * handed and never echoed. It ran none, so each is unsent as surely as a queued one. */ unrun?: true + /** What the rows are written through: the journal, or bookkeeping's background handle. */ + writes?: Pick } ): Promise { + const writes = input.writes ?? journal const { hold } = input const unsent = journal .submissions() @@ -96,9 +101,10 @@ export async function holdUnsentSends( input.unrun ? isUnansweredHandedOverSubmission(entry) : isQueuedAgentJournalSubmission(entry) && - (hold.cause === 'hostRestarted' - ? journal.wroteBeforeOpen(entry.acceptedSequence) - : (hold.which?.(entry) ?? true)) + (hold.which?.(entry) ?? + (hold.cause === 'hostRestarted' + ? journal.wroteBeforeOpen(entry.acceptedSequence) + : true)) ) .sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0)) if (unsent.length === 0) { @@ -162,11 +168,16 @@ export async function holdUnsentSends( } try { // The send names its card in the same row, so no surface draws it once the card is gone. - await journal.resolveDispatch( + await writes.resolveDispatch( card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject, keep ) } catch (error) { + // Another connection holds the database: nothing was written, and the fallback would meet + // the same lock, so the caller tries again. + if (isSqliteContentionFailure(error)) { + throw error + } if (!keep) { failures.push(error) continue @@ -179,7 +190,7 @@ export async function holdUnsentSends( cause: hold.cause, error: error instanceof Error ? error.message : String(error) }) - await journal.resolveDispatch(reject).catch((fallback: unknown) => failures.push(fallback)) + await writes.resolveDispatch(reject).catch((fallback: unknown) => failures.push(fallback)) } } if (failures.length > 0) { diff --git a/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts b/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts index bc8478602d01..2330a203d467 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts @@ -91,7 +91,7 @@ afterEach(async () => { }) describe('draft bookkeeping inside a journal append', () => { - it('a throwing draft transition still commits the rejection row, and the next open recovers the draft', async () => { + it('a throwing draft transition still commits the rejection row, and the next startup recovers the draft', async () => { let journal = await open() await queueAndConsume(journal, 'draft-1') const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) @@ -113,6 +113,9 @@ describe('draft bookkeeping inside a journal append', () => { ) await journal.close() journal = await open() + // An open only reads; startup's repair (`settleOneAtStartup`) recovers it. + expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') + await journal.queuedMessages.repairAndPrune() expect(journal.submission('sub-draft-1')?.dispatchState).toBe('rejected') expect(journal.queuedMessages.get('draft-1')).toMatchObject({ state: 'returned', diff --git a/src/main/native-chat/agent-session-journal/queued-message-holds.ts b/src/main/native-chat/agent-session-journal/queued-message-holds.ts index c3f90590c117..6134594778da 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-holds.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-holds.ts @@ -6,15 +6,13 @@ import type Database from '../../sqlite/sync-database' import type { QueuedMessageHoldReason } from './queued-message-table' +export type QueuedMessageHold = { messageIds: readonly string[]; reason: QueuedMessageHoldReason } + /** Hold waiting drafts from auto-sending. The hold retires with the row: consume * and withdraw clear it in their own UPDATE. Returns how many rows it newly reached. */ export function holdQueuedMessages( db: Database.Database, - input: { - sessionId: string - messageIds: readonly string[] - reason: QueuedMessageHoldReason - } + input: QueuedMessageHold & { sessionId: string } ): number { const update = db.prepare( `UPDATE queued_messages SET hold_reason = ? diff --git a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts new file mode 100644 index 000000000000..c9d81ccdb13c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts @@ -0,0 +1,33 @@ +// Where the reopen mark of what an earlier host process left starts. + +import { describe, expect, it } from 'vitest' +import { queuedMessageReopenMarkStart } from './queued-message-reopen-floor' + +const CURRENT = 'epoch-2' + +describe('the reopen mark after a restart', () => { + it("starts at this handle's floor when it is in the current epoch", () => { + expect( + queuedMessageReopenMarkStart( + { epoch: CURRENT, sequence: 9 }, + { epoch: CURRENT, sequence: 4 }, + CURRENT + ) + ).toBe(9) + }) + + it('starts just past where this process first opened the chat when no floor is set', () => { + expect(queuedMessageReopenMarkStart(null, { epoch: CURRENT, sequence: 4 }, CURRENT)).toBe(5) + }) + + it('is not written from a floor an older epoch left, so it never holds cards queued since', () => { + // A failed startup mark, then a rewind that moved the epoch: both cursors are the old epoch's. + expect( + queuedMessageReopenMarkStart( + { epoch: 'epoch-1', sequence: 9 }, + { epoch: 'epoch-1', sequence: 8 }, + CURRENT + ) + ).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts new file mode 100644 index 000000000000..d8e819a7951c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts @@ -0,0 +1,53 @@ +// Where a reopened chat's queue pause starts before any reopen mark is written: derived from where +// the conversation's handle opened, so nothing has to land first (`queued-message-pause.ts`). + +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { DerivedQueuePause } from './queued-message-pause' + +/** Just past `opened` when a card waits for a reopen mark and no earlier mark holds it already; + * null otherwise. */ +export function queuedMessageReopenFloor( + queue: { awaitReopenMark: () => boolean; pauses: () => readonly DerivedQueuePause[] }, + opened: AgentJournalCursor +): AgentJournalCursor | null { + return queue.awaitReopenMark() && !queue.pauses().some((pause) => pause.reason === 'restarted') + ? { epoch: opened.epoch, sequence: opened.sequence + 1 } + : null +} + +/** This handle's floor while no mark holds the pause (`AgentSessionJournal.reopenFloor`). */ +export class QueuedMessageReopenFloor { + private unmarked: AgentJournalCursor | null = null + + constructor(private readonly queue: Parameters[0]) {} + + get = (): AgentJournalCursor | null => this.unmarked + + holdFromOpen(opened: AgentJournalCursor): void { + this.unmarked = queuedMessageReopenFloor(this.queue, opened) ?? this.unmarked + } + + /** Marks the reopen when a card waits or is mid-hand-off (it may come back to waiting); a failed + * write leaves `at`, where the mark would have gone, as the pause's start, and throws. */ + async mark(at: AgentJournalCursor, append: () => Promise): Promise { + if (this.queue.awaitReopenMark()) { + this.unmarked = at + await append() + this.unmarked = null + } + } +} + +/** Where the reopen mark of what an earlier host process left starts: this handle's unmarked + * floor, else just past where this process first opened the chat; null when neither is in the + * journal's current epoch, since a mark from now would hold cards this process queued since. */ +export function queuedMessageReopenMarkStart( + floor: AgentJournalCursor | null, + processOpened: AgentJournalCursor, + epoch: string +): number | null { + if (floor?.epoch === epoch) { + return floor.sequence + } + return processOpened.epoch === epoch ? processOpened.sequence + 1 : null +} diff --git a/src/main/native-chat/agent-session-journal/queued-message-store.test.ts b/src/main/native-chat/agent-session-journal/queued-message-store.test.ts index 991b384b6402..9be4ce8cc160 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-store.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-store.test.ts @@ -74,6 +74,13 @@ async function open(): Promise { return journal } +/** An open as startup makes it: then the cards' repair and prune (`settleOneAtStartup`). */ +async function openAtStartup(): Promise { + const journal = await open() + await journal.queuedMessages.repairAndPrune() + return journal +} + async function queueDraft(journal: AgentSessionJournal, messageId: string, text = 'queued text') { return journal.queuedMessages.insert({ messageId, @@ -595,23 +602,31 @@ describe('withdraw', () => { }) }) -describe('open-time repair and retention', () => { - it('a failed repair is reported and skipped, never failing the open', async () => { +describe('startup repair and retention', () => { + it('an open repairs nothing; a failed startup repair leaves the journal usable', async () => { + let journal = await open() + await queueDraft(journal, 'draft-1') + await consumeDraft(journal, 'draft-1') + await journal.rejectQueuedSubmissions(0, HOST_RESTARTED) + await journal.close() + const db = new Database(journalDatabasePath(root)) + db.prepare( + "UPDATE queued_messages SET state = 'dispatched', hold_reason = NULL, consumed_as = 'sub-draft-1' WHERE message_id = ?" + ).run('draft-1') + db.close() + journal = await open() + // Opening only reads: the cut-short hand-off waits for startup's repair. + expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') const repair = vi .spyOn(JournalQueuedMessages.prototype, 'repairAndPrune') .mockRejectedValueOnce(new Error('SQLITE_FULL')) - const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) try { - const journal = await open() - expect(warn).toHaveBeenCalledWith( - '[journal-open] queued-message repair skipped:', - expect.objectContaining({ error: 'SQLITE_FULL' }) - ) - await queueDraft(journal, 'draft-1') - expect(journal.queuedMessages.list()).toHaveLength(1) + // Startup reports it (`structured-agent-session-startup-settlement.ts`) and goes on. + await expect(journal.queuedMessages.repairAndPrune()).rejects.toThrow('SQLITE_FULL') + await queueDraft(journal, 'draft-2') + expect(journal.queuedMessages.list()).toHaveLength(2) } finally { repair.mockRestore() - warn.mockRestore() } }) @@ -622,7 +637,7 @@ describe('open-time repair and retention', () => { { origin: 'client' as const, cause: 'chatClosed' as const }, { origin: 'host' as const, cause: 'chatClosed' as const } ])( - 'a skipped hook for a $origin hand-off cut short ($cause) is repaired at open, back to waiting', + 'a skipped hook for a $origin hand-off cut short ($cause) is repaired at startup, back to waiting', async ({ origin, cause }) => { let journal = await open() await queueDraft(journal, 'draft-1') @@ -638,7 +653,7 @@ describe('open-time repair and retention', () => { "UPDATE queued_messages SET state = 'dispatched', hold_reason = NULL, consumed_as = 'sub-draft-1' WHERE message_id = ?" ).run('draft-1') db.close() - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.get('draft-1')).toMatchObject({ state: 'waiting', holdReason: null, @@ -666,7 +681,7 @@ describe('open-time repair and retention', () => { "UPDATE queued_messages SET state = 'dispatched', returned_reason = NULL, returned_rejection = NULL WHERE message_id = ?" ).run('draft-1') db.close() - journal = await open() + journal = await openAtStartup() const row = journal.queuedMessages.get('draft-1') expect(row?.state).toBe('returned') expect(row?.returnedReason).toBe(refusal('refused while downgraded').reason) @@ -685,7 +700,7 @@ describe('open-time repair and retention', () => { ).run('draft-1') db.close() clock += QUEUED_MESSAGE_REPLAY_WINDOW_MS + 1_000 - journal = await open() + journal = await openAtStartup() // The same settlement the live hook applies. expect(journal.queuedMessages.get('draft-1')).toMatchObject({ state: 'waiting', @@ -702,7 +717,7 @@ describe('open-time repair and retention', () => { await journal.close() // Reopen "25 hours" later: the submission is still queued/pending. clock += QUEUED_MESSAGE_REPLAY_WINDOW_MS + 60 * 60 * 1000 - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') // The delivery loop's leftover rejection now sends it back to waiting. await journal.rejectQueuedSubmissions(0, HOST_RESTARTED) @@ -738,7 +753,7 @@ describe('open-time repair and retention', () => { }) await journal.close() clock += QUEUED_MESSAGE_REPLAY_WINDOW_MS + 1_000 - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.list().map((row) => [row.messageId, row.state])).toEqual([ ['waiting-1', 'waiting'], ['returned-1', 'returned'] @@ -754,7 +769,7 @@ describe('open-time repair and retention', () => { }) await journal.close() clock += 1_000 - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.get('withdrawn-1')?.state).toBe('withdrawn') }) }) diff --git a/src/main/native-chat/agent-session-journal/queued-message-table.ts b/src/main/native-chat/agent-session-journal/queued-message-table.ts index 9f2559804486..82f360f10833 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-table.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-table.ts @@ -129,6 +129,14 @@ export function listQueuedMessages(db: Database.Database, sessionId: string): Qu .flatMap((row) => readStoredQueuedMessageRow(row) ?? []) } +/** Every chat holding a card, of any state: the chats startup re-derives cards for. */ +export function listSessionsWithQueuedMessages(db: Database.Database): string[] { + return db + .prepare('SELECT DISTINCT session_id FROM queued_messages') + .all() + .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) +} + export function getQueuedMessage( db: Database.Database, sessionId: string, diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts index c49d266e3432..82703eaf0d70 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts @@ -215,9 +215,14 @@ describe('a background task', () => { rig.assembler.apply({ type: 'item.open', item: 'ran', body: backgroundTask('ran', 'working') }) rig.assembler.apply({ type: 'item.close', item: 'ran', body: backgroundTask('ran', 'done') }) rig.assembler.apply({ type: 'session.ended', verdict: { state: 'unverifiable' } }) - expect((await rig.row(providerItemId('item', 'bg')))?.body).toEqual( - backgroundTask('bg', 'unverifiable') - ) + // The entry keeps the generation that ran it (`ownerFence`), stamped when it was observed. + const unverifiable = backgroundTask('bg', 'unverifiable') + expect((await rig.row(providerItemId('item', 'bg')))?.body).toEqual({ + ...unverifiable, + blocks: unverifiable.blocks.map((block) => + block.type === 'background-task' ? { ...block, ownerFence: 1 } : block + ) + }) expect(await backgroundTaskState(rig, 'ran')).toBe('done') }) }) diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-settlement.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-settlement.ts index ae6e2ef32c85..51da7f70e8b7 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-settlement.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-settlement.ts @@ -97,7 +97,7 @@ export function providerTimelineSettlement( const settled = !covered ? null : (terminalAgentJournalBody(body, end) ?? - (scope === 'session' ? lostLiveWorkJournalBody(body, ending?.lostAt) : null)) + (scope === 'session' ? lostLiveWorkJournalBody(body, { lostAt: ending?.lostAt }) : null)) const identity = settled ? parseAgentJournalItemKey(itemId) : null if (settled && identity) { // No linkage: a revision keeps the row's own producer. diff --git a/src/main/native-chat/agent-session-wire/agent-session-history-page.ts b/src/main/native-chat/agent-session-wire/agent-session-history-page.ts index b1667daa512c..d2714ff3cbf5 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-history-page.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-history-page.ts @@ -36,6 +36,15 @@ import { submissionBytesByItemId } from './agent-session-history-page-bounds' import { offPageSubagentRoster } from './agent-session-history-subagent-roster' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' + +/** What a page publishes of the host's current-work projection (`structuredAgentSessionCurrentWork`): + * the newest turn record as clients are told it, and the prompts still waiting on the person. + * Absent, a page states the raw record and no prompt ids, as an older host did. */ +export type AgentSessionPublishedWork = Pick< + StructuredAgentSessionCurrentWork, + 'publishedLatestTurn' | 'actionablePromptIds' | 'working' +> export { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from './agent-session-history-page-bounds' @@ -80,19 +89,20 @@ export function readAgentSessionHistory( /** Reduced state to read against. A synchronous multi-page catch-up passes one * snapshot for the whole run so each page costs its own rows, not the timeline. */ snapshot: AgentJournalSnapshot = journal.snapshot(), - scope: AgentSessionHistoryScope = 'every-agent' + scope: AgentSessionHistoryScope = 'every-agent', + work?: AgentSessionPublishedWork ): AgentSessionHistoryResult { const limit = resolveHistoryLimit(request.limit) if (request.direction === 'after') { - return readForward(journal, snapshot, request.cursor, limit) + return readForward(journal, snapshot, request.cursor, limit, work) } const cursor = request.direction === 'before' ? request.cursor : undefined if (cursor) { if (cursor.epoch !== snapshot.cursor.epoch) { - return historyReset(snapshot, 'epoch_changed') + return historyReset(snapshot, 'epoch_changed', work) } if (cursor.sequence > snapshot.cursor.sequence) { - return historyReset(snapshot, 'cursor_ahead') + return historyReset(snapshot, 'cursor_ahead', work) } } const scoped = @@ -117,7 +127,8 @@ export function readAgentSessionHistory( nextCursor: items[0] ? { epoch: snapshot.cursor.epoch, sequence: items[0].sequence } : undefined, - subagentRoster: offPageSubagentRoster(scoped, items) + subagentRoster: offPageSubagentRoster(scoped, items), + work }) } } @@ -128,7 +139,8 @@ export function readAgentSessionHistory( * check re-reduces if anything did advance the journal between pages. */ export function createAgentSessionCatchUpReader( - journal: AgentSessionJournal + journal: AgentSessionJournal, + work?: AgentSessionPublishedWork ): (request: AgentSessionHistoryRequest) => AgentSessionHistoryResult { let snapshot = journal.snapshot() return (request) => { @@ -136,20 +148,34 @@ export function createAgentSessionCatchUpReader( if (live.epoch !== snapshot.cursor.epoch || live.sequence !== snapshot.cursor.sequence) { snapshot = journal.snapshot() } - return readAgentSessionHistory(journal, request, snapshot) + return readAgentSessionHistory(journal, request, snapshot, 'every-agent', work) } } export function readAgentSessionHydrationPage( journal: AgentSessionJournal, - fence?: number + fence?: number, + work?: AgentSessionPublishedWork ): AgentSessionHistoryPage { - return buildHydrationPage(journal.snapshot(), fence) + return buildHydrationPage(journal.snapshot(), fence, work) +} + +/** The current-work fields a page carries, for a frame that carries no page. */ +export function agentSessionPublishedWorkFields( + journal: AgentSessionJournal, + work: AgentSessionPublishedWork +): Pick { + return { + latestTurn: work.publishedLatestTurn(snapshotLatestTurn(journal.snapshot())), + actionablePromptIds: work.actionablePromptIds(), + working: work.working() + } } function buildHydrationPage( snapshot: AgentJournalSnapshot, - fence?: number + fence?: number, + work?: AgentSessionPublishedWork ): AgentSessionHistoryPage { const items = conversationWindow(snapshot.items, AGENT_SESSION_HISTORY_MAX_LIMIT) const bounded = boundHistoryItemsByBytes( @@ -169,18 +195,20 @@ function buildHydrationPage( ? { epoch: snapshot.cursor.epoch, sequence: bounded.items[0].sequence } : undefined, fence, - subagentRoster: offPageSubagentRoster(snapshot.items, bounded.items) + subagentRoster: offPageSubagentRoster(snapshot.items, bounded.items), + work }) } function historyReset( snapshot: AgentJournalSnapshot, - reset: Extract['reset'] + reset: Extract['reset'], + work: AgentSessionPublishedWork | undefined ): AgentSessionHistoryResult { return { ok: false, reset, - page: buildHydrationPage(snapshot) + page: buildHydrationPage(snapshot, undefined, work) } } @@ -188,18 +216,19 @@ function readForward( journal: AgentSessionJournal, snapshot: AgentJournalSnapshot, cursor: AgentJournalCursor | undefined, - limit: number + limit: number, + work: AgentSessionPublishedWork | undefined ): AgentSessionHistoryResult { if (!cursor) { // Why: forward paging replays rows after a position; without one there is // nothing to be after, and silently serving the tail would hand the client // a page it cannot place. - return historyReset(snapshot, 'cursor_ahead') + return historyReset(snapshot, 'cursor_ahead', work) } // One lookahead preserves hasNewer without rereading the entire remaining journal per page. const since = journal.readSince(cursor, limit + 1) if (!since.ok) { - return historyReset(snapshot, since.reset) + return historyReset(snapshot, since.reset, work) } const submissionBytes = submissionBytesByItemId(snapshot.submissions) // The page cost is EVERYTHING variable it carries: items with their @@ -225,7 +254,7 @@ function readForward( canonicalItemId: (itemId) => journal.canonicalItemId(itemId) }) if (!projected.ok) { - return historyReset(snapshot, projected.reset) + return historyReset(snapshot, projected.reset, work) } let contentBytes = pageContentBytes(projected.batch.items, projected.batch.removedItemIds) while (rows.length > 1 && contentBytes > HISTORY_PAGE_CONTENT_BUDGET_BYTES) { @@ -237,7 +266,7 @@ function readForward( canonicalItemId: (itemId) => journal.canonicalItemId(itemId) }) if (!shrunk.ok) { - return historyReset(snapshot, shrunk.reset) + return historyReset(snapshot, shrunk.reset, work) } projected = shrunk contentBytes = pageContentBytes(projected.batch.items, projected.batch.removedItemIds) @@ -257,7 +286,7 @@ function readForward( // break the client's keying. A bounded tail replaces the client's state // wholesale, which applies the removal without carrying the id, and the // client resumes from the live cursor past this row. - return historyReset(snapshot, 'cursor_compacted') + return historyReset(snapshot, 'cursor_compacted', work) } const lastSequence = rows.at(-1)?.seq ?? cursor.sequence return { @@ -271,7 +300,8 @@ function readForward( hasOlder: cursor.sequence > 0, hasNewer: since.rows.length > rows.length, fallbackCursor: cursor, - nextCursor: { epoch: cursor.epoch, sequence: lastSequence } + nextCursor: { epoch: cursor.epoch, sequence: lastSequence }, + work }) } } @@ -287,6 +317,7 @@ function buildPage(input: { nextCursor: AgentJournalCursor | undefined fence?: number subagentRoster?: AgentSessionSubagentRosterEntry[] + work?: AgentSessionPublishedWork }): AgentSessionHistoryPage { const epoch = input.snapshot.cursor.epoch const pageItemIds = new Set(input.items.map((item) => item.itemId)) @@ -311,8 +342,15 @@ function buildPage(input: { hasOlder: input.hasOlder, hasNewer: input.hasNewer, ...(input.subagentRoster === undefined ? {} : { subagentRoster: input.subagentRoster }), - // From the whole timeline, never the page: a long turn's record sits below any window. - latestTurn: snapshotLatestTurn(input.snapshot) + // From the whole timeline, never the page: a long turn's record sits below any window. Scoped + // by the host's projection where the reader has one: an ended generation's turn runs nowhere. + ...(input.work + ? { + latestTurn: input.work.publishedLatestTurn(snapshotLatestTurn(input.snapshot)), + actionablePromptIds: input.work.actionablePromptIds(), + working: input.work.working() + } + : { latestTurn: snapshotLatestTurn(input.snapshot) }) } } diff --git a/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts new file mode 100644 index 000000000000..1f22d20e6d23 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts @@ -0,0 +1,140 @@ +// The stop-note refresh frame carries rows, so it restates the host's current work as every rows +// frame does: a client reads a rows frame without it as an older host's and falls back to its own +// raw derivation (an ended turn as Working, an ended agent's prompt as answerable). + +import { describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { + AgentSessionHistoryPage, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession +} from '../../../shared/structured-agent-session-reducer' +import { isActionableStructuredAgentSessionPrompt } from '../../../shared/structured-agent-session-live-turn' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { refreshDerivedStopNotes } from './agent-session-stop-note-refresh' +import { structuredAgentSessionStopNoteIdentity } from './structured-agent-session-command-turn' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { Subscriber } from './structured-agent-session-subscribers' + +const staleTurn: AgentJournalRenderItem = { + itemId: 'turn-old', + revision: 1, + sequence: 1, + observedAt: 1, + body: { kind: 'turn', turnId: 't-old', state: 'running', startedAt: 1 } +} +const stalePrompt: AgentJournalRenderItem = { + itemId: 'prompt-old', + revision: 1, + sequence: 2, + observedAt: 2, + body: { + kind: 'approval', + title: 'Run ls', + detail: null, + options: [{ id: 'allow', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } +} +const noteId = agentJournalItemKey(structuredAgentSessionStopNoteIdentity('t-old')) +const unconfirmed = { + kind: 'status' as const, + text: 'x', + failure: { kind: 'cancelUnconfirmed' as const } +} +const projectedNote: AgentJournalRenderItem = { + itemId: noteId, + revision: 2, + sequence: 3, + observedAt: 3, + body: { kind: 'status', text: 'Stopped.' } +} +const items = [staleTurn, stalePrompt, projectedNote] + +/** What a current host publishes: the ended generation's turn and prompt are not current. */ +function page(): AgentSessionHistoryPage { + return { + sessionId: 's', + epoch: 'e', + direction: 'tail', + items, + removedItemIds: [], + submissions: [], + window: { + oldest: { epoch: 'e', sequence: 1 }, + newest: { epoch: 'e', sequence: 3 }, + nextCursor: { epoch: 'e', sequence: 1 } + }, + liveCursor: { epoch: 'e', sequence: 3 }, + hasOlder: false, + hasNewer: false, + latestTurn: null, + actionablePromptIds: [], + working: false + } +} + +describe('the stop-note refresh frame', () => { + it("restates the host's current work, so a client keeps the host's answers", () => { + const emitted: AgentSessionSubscribeEvent[] = [] + const journal = { + visitItems: (visit: (itemId: string, sequence: number, body: unknown) => void) => + visit(noteId, 3, unconfirmed), + snapshot: () => ({ + sessionId: 's', + cursor: { epoch: 'e', sequence: 3 }, + items, + submissions: [] + }), + itemBody: (itemId: string) => (itemId === noteId ? unconfirmed : null) + } + // The generation live now is fence 2; both stale items came from fence 1. + const work = new StructuredAgentSessionCurrentWork( + { + runningTurn: () => ({ item: staleTurn, turnId: 't-old' }), + itemFence: () => 1, + visitItems: (visit) => { + for (const item of items) { + visit(item.itemId, item.sequence, item.body) + } + }, + submissions: () => [], + wroteBeforeOpen: () => false + }, + 2 + ) + const subscriber = { sessionId: 's', cursor: { epoch: 'e', sequence: 3 }, fence: 2 } + refreshDerivedStopNotes( + { emit: (_subscriber, event) => emitted.push(event), isActive: () => true }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the refresh reads only the subscriber's session, cursor and fence. + subscriber as unknown as Subscriber, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the refresh reads only visitItems, snapshot and itemBody, all supplied. + journal as unknown as AgentSessionJournal, + 10, + work + ) + + expect(emitted).toHaveLength(1) + const frame = emitted[0] + if (frame?.type !== 'batch') { + throw new Error('expected a batch') + } + expect(frame.batch.items.length).toBeGreaterThan(0) + expect(frame).toMatchObject({ working: false, actionablePromptIds: [], latestTurn: null }) + + let state = reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'event', + event: { type: 'snapshot', sessionId: 's', page: page(), fence: 2 } + }) + state = reduceStructuredAgentSession(state, { type: 'event', event: frame }) + expect(state.working).toBe(false) + expect(state.latestTurn).toBeNull() + expect(isActionableStructuredAgentSessionPrompt('prompt-old', state.actionablePromptIds)).toBe( + false + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts index 5f86048af0c1..83ce92fc7cf4 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts @@ -6,16 +6,23 @@ import { oversizedHistoryItem } from './agent-session-history-page-bounds' import { emptyAgentSessionBatch } from './agent-session-empty-batch' +import { + agentSessionPublishedWorkFields, + type AgentSessionPublishedWork +} from './agent-session-history-page' import type { SubscriberDeliveryPort } from './agent-session-subscriber-catch-up' import type { Subscriber } from './structured-agent-session-subscribers' import { isStructuredAgentSessionStopNote } from './structured-agent-session-command-turn' -/** A stale in-flight history page can land after its note's live correction was out of window. */ +/** A stale in-flight history page can land after its note's live correction was out of window. + * `work`: the host's current work, which a frame carrying rows restates, as every such frame does + * (a client reads a rows frame without it as an older host's). */ export function refreshDerivedStopNotes( port: Pick, subscriber: Subscriber, journal: AgentSessionJournal, - hostNow: number + hostNow: number, + work?: AgentSessionPublishedWork ): void { let hasUnconfirmedStop = false journal.visitItems((itemId, _sequence, body) => { @@ -39,6 +46,7 @@ export function refreshDerivedStopNotes( sessionId: subscriber.sessionId, batch: { ...emptyAgentSessionBatch(subscriber.cursor), items: pending }, fence: subscriber.fence, + ...(work ? agentSessionPublishedWorkFields(journal, work) : {}), hostNow }) pending = [] diff --git a/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts b/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts index df98d8ba902b..668c433de9e0 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts @@ -12,7 +12,11 @@ import { import { sameJournalCursor } from '../agent-session-journal/journal-cursor' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { emptyAgentSessionBatch } from './agent-session-empty-batch' -import { createAgentSessionCatchUpReader } from './agent-session-history-page' +import { + agentSessionPublishedWorkFields, + createAgentSessionCatchUpReader +} from './agent-session-history-page' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { subscriberQueuedMessagesChanged, type SubscriberFieldHooks @@ -22,6 +26,11 @@ import type { Subscriber } from './structured-agent-session-subscribers' export type SubscriberDeliveryPort = { hooks: SubscriberFieldHooks & { readCommands?: (sessionId: string) => AgentSessionSlashCommand[] | undefined + /** The host's projection of current work, which every frame's turn and prompt fields state. */ + readCurrentWork?: ( + sessionId: string, + journal: AgentSessionJournal + ) => StructuredAgentSessionCurrentWork | undefined } emit: ( subscriber: Subscriber, @@ -49,12 +58,13 @@ export function deliverToSubscriber( hostNow, ...(publishedActivity !== undefined ? { activity: publishedActivity } : {}) } + const work = port.hooks.readCurrentWork?.(subscriber.sessionId, journal) // Caught up, so there are no rows to read: every publish behind a commit's own delivery. if (sameJournalCursor(subscriber.cursor, journal.cursor())) { - emitCaughtUp(port, subscriber, emitCheckpoint, shared) + emitCaughtUp(port, subscriber, emitCheckpoint, shared, { journal, work }) return } - const readPage = createAgentSessionCatchUpReader(journal) + const readPage = createAgentSessionCatchUpReader(journal, work) while (true) { const result = readPage({ sessionId: subscriber.sessionId, @@ -73,12 +83,13 @@ export function deliverToSubscriber( ...shared }) subscriber.cursor = page.liveCursor ?? page.window.nextCursor + subscriber.workView = work?.viewKey() return } const page = result.page const advanced = page.window.nextCursor.sequence > subscriber.cursor.sequence if (!advanced) { - emitCaughtUp(port, subscriber, emitCheckpoint, shared) + emitCaughtUp(port, subscriber, emitCheckpoint, shared, { journal, work }) return } port.emit( @@ -94,6 +105,10 @@ export function deliverToSubscriber( }, fence: subscriber.fence, ...(page.latestTurn !== undefined ? { latestTurn: page.latestTurn } : {}), + ...(page.actionablePromptIds !== undefined + ? { actionablePromptIds: page.actionablePromptIds } + : {}), + ...(page.working !== undefined ? { working: page.working } : {}), ...shared }, // On a multi-page catch-up the draft list rides only the final page, or a @@ -101,6 +116,7 @@ export function deliverToSubscriber( { withholdQueued: page.hasNewer } ) subscriber.cursor = page.window.nextCursor + subscriber.workView = work?.viewKey() if (!page.hasNewer || !port.isActive(subscriber)) { return } @@ -114,19 +130,34 @@ function emitCaughtUp( shared: { hostNow: number activity?: AgentSessionTurnActivity | null - } + }, + current: { journal: AgentSessionJournal; work: StructuredAgentSessionCurrentWork | undefined } ): void { const commandsChanged = port.hooks.readCommands !== undefined && (port.hooks.readCommands(subscriber.sessionId) ?? null) !== subscriber.commands const queuedChanged = subscriberQueuedMessagesChanged(port.hooks, subscriber) - if (emitCheckpoint || shared.activity !== undefined || commandsChanged || queuedChanged) { + // A generation that ended (or began) with no row, or an exit this host saw with no lease write, + // changes what is current: the caught-up frame restates the view. + const { work } = current + const workChanged = work !== undefined && work.viewKey() !== subscriber.workView + if ( + emitCheckpoint || + shared.activity !== undefined || + commandsChanged || + queuedChanged || + workChanged + ) { port.emit(subscriber, { type: 'batch', sessionId: subscriber.sessionId, batch: emptyAgentSessionBatch(subscriber.cursor), fence: subscriber.fence, + ...(workChanged ? agentSessionPublishedWorkFields(current.journal, work) : {}), ...shared }) + if (workChanged) { + subscriber.workView = work.viewKey() + } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index 29a033a4e7b9..aaefa5a457d9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -50,6 +50,7 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-hand import { StructuredAgentArgumentsError } from '../structured-agent-arguments-error' import { USER_MESSAGE_SOURCE } from '../../../shared/agent-session-message-source' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { startUpHost } from './structured-agent-session-leftover-settlement.test-fixture' const CALLER = { callerKey: 'client-1' } @@ -252,6 +253,9 @@ async function writeAsEarlierProcess( }) await write(journal, record.lease.runtimeFence) await journal.close() + // The next host is a new process, which has not opened the chat yet. + await host.flushAllStreamedEvents() + await startHost() } function earlierSubmission(id: string, text: string, handoverRecorded?: true) { @@ -557,8 +561,7 @@ describe('what an earlier host process left behind', () => { await writeAsEarlierProcess(async (journal, fence) => { await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence }) }) - - await host.revealSession(SESSION) + await startUpHost(host) expect(await submission('queued')).toMatchObject({ dispatchState: 'rejected', @@ -582,10 +585,7 @@ describe('what an earlier host process left behind', () => { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) }) - await host.flushAllStreamedEvents() - await startHost() - - await host.revealSession(SESSION) + await startUpHost(host) expect(await submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) expect(await submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true }) @@ -676,7 +676,7 @@ describe('a start that fails while messages wait on it', () => { }) describe('Stop withdraws what is queued', () => { - it('never meets a crash leftover: the open it runs settles it first (W17a)', async () => { + it('never meets a crash leftover: startup settles it first (W17a)', async () => { await writeAsEarlierProcess(async (journal, fence) => { await journal.appendSubmission({ ...earlierSubmission('person', 'p', true), @@ -686,6 +686,7 @@ describe('Stop withdraws what is queued', () => { }) await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) }) + await startUpHost(host) expect(await stop()).toMatchObject({ ok: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts index 2c180c121b33..f61746cb05ee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts @@ -146,7 +146,8 @@ async function startStructuredAgentSessionAgent( } const params = structuredAgentSessionResumeParams( record, - structuredAgentSessionResumeOperationId(context.now()) + structuredAgentSessionResumeOperationId(context.now()), + context.sessions.get(sessionId)?.lastEndedChild ) if (!params) { return record.lease.unreconciled diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 12c2f54a1793..c5c3c3edb168 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -15,7 +15,6 @@ import type { import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' -import type { StructuredAgentSessionConversationOpenOptions } from './structured-agent-session-conversation-open' export type StructuredAgentSessionAttachContext = { deps: StructuredAgentSessionHostDeps @@ -34,12 +33,10 @@ export type StructuredAgentSessionAttachContext = { serialize: (sessionId: string, task: () => Promise) => Promise now: () => number publishStatus: (sessionId: string) => void + generationEnded?: StructuredAgentSessionLifetimeContext['generationEnded'] /** Joining a stop's close ends the child's record through the one exit handler. */ endExitedChild: StructuredAgentSessionLifetimeContext['endExitedChild'] wakeDelivery?: StructuredAgentSessionLifetimeContext['wakeDelivery'] /** The conversation's one open journal, opened when closed; see `conversation-open`. */ - openConversation: ( - sessionId: string, - options?: StructuredAgentSessionConversationOpenOptions - ) => Promise + openConversation: (sessionId: string) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index 06e325fefd37..a8368d067893 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -84,4 +84,6 @@ export function endStructuredAgentSessionReleasedChild( context.runtimeState.currentEventSink(sessionId)?.close() context.runtimeState.discardEventSink(sessionId) context.publishStatus?.(sessionId) + // The released acquisition was the live generation: readers and the queued-card drain re-derive. + context.generationEnded?.(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index c3271ff3d477..a437a1db496a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -36,6 +36,7 @@ import type { StructuredAgentRegistry } from './structured-agent-registry' import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' import { resolveAgentSessionReplayOutcome } from './structured-agent-session-replay-outcome' import { readAgentSessionHydrationPage } from './agent-session-history-page' +import { heldStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { acquireOwner } from './structured-agent-session-acquisition' import { importAdoptedTranscript, @@ -284,7 +285,12 @@ export async function performAttach( value: { sessionId, fence, - page: readAgentSessionHydrationPage(attached.journal, fence), + // The lease this attach just took is the live generation. + page: readAgentSessionHydrationPage( + attached.journal, + fence, + heldStructuredAgentSessionCurrentWork(store, sessionId, attached.journal, undefined) + ), unconfirmedClientMessageIds: attached.unconfirmedClientMessageIds, ...(tabId ? { tabId } : {}) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index 5ababd586e6a..6e2b2ced5f7f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -22,8 +22,6 @@ import { pinnedAgentSessionLaunchEnv } from './structured-agent-session-launch-env' import { refuseAgentSessionMutation } from './structured-agent-session-mutation-admission' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' -import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { @@ -148,10 +146,7 @@ async function runAttachUnderAbort( const probe = await withAgentSessionCreatePhase('probe_owner', recordPhase, () => context.runtimeState.probeOwner(sessionId) ) - // Read before the reserve clears it: how the previous generation ended decides how whatever it - // left running is settled. const priorRecord = context.deps.store.getRecord(sessionId) - const priorDeathEvidence = priorRecord?.lease.deathEvidence ?? null const launchDirectory = !priorRecord && isFloatingWorkspaceId(params.location.workspaceId) ? (options.hostLaunchDirectory ?? @@ -207,9 +202,7 @@ async function runAttachUnderAbort( optionRevision: () => context.runtimeState.optionRevisions.current(sessionId), ...(options.onAcquisitionFailed ? { onAcquisitionFailed: options.onAcquisitionFailed } : {}), openConversation: async (record) => { - const conversation = await context.openConversation(record.sessionId, { - acquisition: true - }) + const conversation = await context.openConversation(record.sessionId) if (!conversation) { throw new Error('agent_session_identity_required') } @@ -227,16 +220,14 @@ async function runAttachUnderAbort( ? attemptSink : (context.runtimeState.currentEventSink(sessionId) ?? attemptSink) if (acquiredOwner) { - // Before the drain: the buffered events are the new child's, never a stale row's. - await settleStaleStructuredAgentSessionState({ - journal: attached.journal, - sessionId, - fence, - acquisitionGeneration, - deathEvidence: priorDeathEvidence, - replaced: context.deps.store.replacedRuntime(sessionId), - failureTextContext: structuredAgentSessionFailureWordsContext(priorRecord) - }) + // The chat is kept from its first start, written to or not; never a reason to fail it. + await attached.journal.foundEpoch().catch((error: unknown) => + context.deps.logger.warn("writing a chat's first journal row failed", { + scope: 'journal-founding', + sessionId, + error + }) + ) } await bindAndDrain(eventSink, attached.journal, fence, (activity) => context.subscribers.publish(sessionId, attached.journal, activity) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts index e22c5dea9fdc..8e7f39c6b5f8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts @@ -17,6 +17,7 @@ import { } from '../agent-session-journal/journal-host-database-test-support' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-stale-state-settlement' import { attachJournal, journalIdentityFor, @@ -66,17 +67,27 @@ function adapterWith(providerHistoryWindow?: () => Promise Promise { return async (record) => - ( - await openStructuredAgentSessionConversationJournal( - { journalDatabase, logger: recordingStructuredAgentSessionLogger().logger }, - record, - { - acquisition: true - } - ) - ).session.journal + (await openStructuredAgentSessionConversationJournal({ journalDatabase }, record)).session + .journal } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts new file mode 100644 index 000000000000..42faabced952 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts @@ -0,0 +1,273 @@ +// Every background path meeting another connection's write lock: an exit's own settlement and +// release, the retry's recovery of a pending rewind, and the queue's automatic drain healing its +// bookkeeping. Each fails at once, never holding the main thread or the chat's lane for the busy +// timeout, and the retry lands the work once the lock lifts. A refusal of the automatic send whose +// stored hold also fails still shows the card as not sent. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { JournalHostDatabase } from '../agent-session-journal/journal-host-database' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + exitChild, + leaveUnfinishedWork, + REASONING, + TURN_KEY, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' +import { + holdWriteLock, + longestStall, + releaseWriteLocks, + stallsOver +} from './structured-agent-session-write-lock.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await releaseWriteLocks() + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Longer than a background write would ever wait, shorter than the busy timeout a person's has. */ +const LOCK_MS = 1_200 + +describe('under another connection’s write lock', () => { + it("an exit's own settlement and release fail at once, and the retry lands both after", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + await holdWriteLock(current.root, LOCK_MS) + + expect(await longestStall(() => exitChild(current))).toBeLessThan(60) + expect(turnState(current)).toBe('running') + + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 8_000 }) + await vi.waitFor( + () => expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('released'), + { timeout: 8_000 } + ) + }, 20_000) + + it("an agent dying mid-row costs one wait, its last row's, and the exit's bookkeeping none", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let sinkFailedAt = Infinity + vi.spyOn(console, 'error').mockImplementation((message: unknown) => { + if (String(message).includes('writing provider events to the chat journal failed')) { + sinkFailedAt = performance.now() + } + }) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + // Longer than a person's write waits (`JOURNAL_BUSY_TIMEOUT_MS`), so the row's write gives up. + await holdWriteLock(current.root, 6_500) + + const stalls = await stallsOver(60, async () => { + // The agent's last words are a person's content: written as on main, waiting its 5 s. + current + .providerEvents() + .appendItem( + REASONING, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'last words' }] }, + { turnScope: { kind: 'turn', turnItemId: TURN_KEY } } + ) + await exitChild(current) + }) + + // One wait, the row's own busy timeout (5 s, plus SQLite's sleep granularity on a loaded + // machine), and it is the row's: the sink reports its failure as that wait ends. + expect(stalls).toHaveLength(1) + const [wait] = stalls + expect(wait!.ms).toBeLessThanOrEqual(5_500) + expect(sinkFailedAt).toBeGreaterThanOrEqual(wait!.endedAt - wait!.ms) + expect(sinkFailedAt).toBeLessThanOrEqual(wait!.endedAt) + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 10_000 }) + await vi.waitFor( + () => expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('released'), + { timeout: 8_000 } + ) + }, 30_000) + + it('a relaunch under the lock never holds the main thread, and its startup lands after', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + + // Every write that runs while the lock is held: none may be one that waits it out. + const waiting: unknown[] = [] + let held = false + const transaction = JournalHostDatabase.prototype.transaction + vi.spyOn(JournalHostDatabase.prototype, 'transaction').mockImplementation(function ( + this: JournalHostDatabase, + run, + options + ) { + if (held && !options?.background) { + waiting.push(new Error('a write that waits for the lock')) + } + return transaction.call(this, run, options) + }) + + // Another process holds the lock from before this one starts: the restart reconcile, the + // restore's reconcile and recovery, and the startup scan all meet it. + const stall = await longestStall(() => + current.crashRestartHostProcess(async () => { + const { released } = await holdWriteLock(current.root, LOCK_MS) + held = true + void released.then(() => (held = false)) + }) + ) + expect(waiting).toEqual([]) + // The bar QA holds the app to; the property above is what keeps it there under load. + expect(stall).toBeLessThan(100) + // Once it lifts, the retry reconciles, resolves the recovery the restore could not, and settles + // the turn as an unlocked startup does: the rig's probe proves nothing, so unverifiable. + await vi.waitFor(() => expect(turnState(current)).toBe('unverifiable'), { timeout: 8_000 }) + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + unreconciled: false, + handoffStage: null + }) + }, 20_000) + + it('a relaunch under a lock longer than a minute of rounds still settles once it lifts, unasked', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + let released: Promise = Promise.resolve() + + await current.crashRestartHostProcess(async () => { + released = (await holdWriteLock(current.root, 40_000)).released + }) + await released + // Nothing else writes or reads: only the retry's own backoff, at most 2 s, comes back to it. + await vi.waitFor(() => expect(turnState(current)).toBe('unverifiable'), { timeout: 3_000 }) + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.working).toBe(false) + expect(page.ok && page.page.latestTurn?.turn.state).not.toBe('running') + }, 60_000) + + it('a pending rewind the retry recovers ends its round at once, never holding the lane', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const { reconciliation, serialize } = current.host.collaboratorsForTests() + await current.store.transitionHandoff(SESSION, (record) => ({ + ...record, + rewind: { + operationId: 'rewind-op', + callerKey: 'client-1', + itemId: agentJournalItemKey({ provider: 'claude', sessionId: 'claude-s', uuid: 'kept' }), + expectedEpoch: 'epoch-x', + phase: 'prepared', + retained: [] + } + })) + await holdWriteLock(current.root, LOCK_MS) + + let laneWaited = Infinity + const stall = await longestStall(async () => { + reconciliation.signal(SESSION) + await new Promise((resolve) => setImmediate(resolve)) + const queuedAt = performance.now() + const lane = serialize(SESSION, async () => { + laneWaited = performance.now() - queuedAt + }) + await reconciliation.attempted(SESSION) + await lane + }) + expect(stall).toBeLessThan(60) + expect(laneWaited).toBeLessThan(60) + expect(current.store.getRecord(SESSION)?.rewind?.phase).toBe('prepared') + expect(retryOwes(reconciliation, SESSION)).toBe(true) + + // Claude rewind is unsupported: once the lock lifts, the retry settles it refused. + await vi.waitFor( + () => expect(current.store.getRecord(SESSION)?.rewind?.phase).toBe('refused'), + { timeout: 8_000 } + ) + }, 20_000) + + it("the drain's bookkeeping heal fails at once, and its send waits for the retry", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + await current.send('after this', 'queue-if-active').result + // The turn's end wakes the drain, which heals skipped bookkeeping before it sends; another + // connection takes the lock just as it starts. + const owed = vi.spyOn(JournalQueuedMessages.prototype, 'settlementOwed').mockReturnValue(true) + const heal = JournalQueuedMessages.prototype.settleOwed + let waited = Infinity + vi.spyOn(JournalQueuedMessages.prototype, 'settleOwed').mockImplementationOnce(async function ( + this: JournalQueuedMessages, + ...args + ) { + owed.mockRestore() + await holdWriteLock(current.root, LOCK_MS) + const started = performance.now() + try { + return await heal.apply(this, args) + } finally { + waited = performance.now() - started + } + }) + + await current.settleAccepted(working, 'a') + await vi.waitFor(() => expect(waited).toBeLessThan(Infinity)) + expect(waited).toBeLessThan(60) + expect(current.dispatch).toHaveBeenCalledTimes(1) + + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2), { timeout: 8_000 }) + }, 20_000) +}) + +describe('a refused automatic send whose stored hold fails too', () => { + it('reads as not sent from memory, is not named next, and is not sent again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + const diskFull = () => Object.assign(new Error('database or disk is full'), { errcode: 13 }) + const append = vi + .spyOn(AgentSessionJournal.prototype, 'appendSubmission') + .mockRejectedValueOnce(diskFull()) + const hold = vi.spyOn(JournalQueuedMessages.prototype, 'hold').mockRejectedValueOnce(diskFull()) + + await current.settleAccepted(working, 'a') + await vi.waitFor(() => expect(hold).toHaveBeenCalledTimes(1)) + await new Promise((resolve) => setTimeout(resolve, 300)) + + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages).toEqual([ + expect.objectContaining({ + messageId: queued.value.queued.messageId, + paused: true, + pausedReason: QUEUED_MESSAGE_PAUSED_SEND_FAILED + }) + ]) + expect(page.ok && page.page.nextQueuedMessageId).toBeNull() + expect(append).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index cf44fc3d66b5..0f87c4164497 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -7,10 +7,7 @@ import type { AgentChildWorkView } from '../../../shared/agent-status-child-work import { structuredChildWorkLegacyTasks } from '../../../shared/structured-agent-session-child-work-legacy' import { structuredRunningChildWork } from '../../../shared/agent-child-work-listing' import { readStructuredAgentSessionHistoryResult } from './structured-agent-session-history-result' -import { - structuredQueueSendGate, - tryReadQueuePublication -} from './structured-agent-session-queued-publication' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import type { AgentSessionHistoryScope } from './agent-session-history-page' import type { AgentSessionSubscribers, @@ -45,17 +42,19 @@ export class StructuredAgentSessionBackgroundTaskChannel { scope?: AgentSessionHistoryScope ): Promise { const journal = (await this.conversation(request.sessionId)).journal + const work = hostStructuredAgentSessionCurrentWork( + { store: this.deps.store, sessions: this.sessions }, + request.sessionId + ) const result = readStructuredAgentSessionHistoryResult({ journal, record: this.deps.store.getRecord(request.sessionId), request, - scope + scope, + ...(work ? { work } : {}) }) const backgroundTasks = this.read(request.sessionId) - const queue = tryReadQueuePublication( - journal, - structuredQueueSendGate(this.deps.store, request.sessionId) - ) + const queue = this.subscribers.readQueuePublication(request.sessionId) const hostNow = this.deps.now?.() ?? Date.now() return { ...result, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts new file mode 100644 index 000000000000..c3e443f7ca7f --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -0,0 +1,242 @@ +// Background bookkeeping meeting another connection's write lock: it fails at once and ends the +// retry's round (no 5 s wait on the main thread), while a person's write in the same tick still +// waits the lock out. One episode's give-up is never a latch: a commit or a signal re-arms it, and +// a queued send given up on reads as not sent even when its stored hold cannot be written. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + currentJournal, + exitWhileSettlementFails, + leaveUnfinishedWork, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' +import { + holdWriteLock, + longestStall, + releaseWriteLocks +} from './structured-agent-session-write-lock.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + vi.useRealTimers() + await releaseWriteLocks() + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) + +/** A chat whose exit left its settlement owed to the retry, with nothing refusing it now. */ +async function settlementOwed(): Promise { + rig = await createQueuedMessageTestRig({ restartable: true }) + await rig.workingSend() + await leaveUnfinishedWork(rig, { prompt: true }) + const { database } = await exitWhileSettlementFails(rig) + database.db.exec('DROP TRIGGER reject_recovered') + return rig +} + +describe("another connection's write lock", () => { + it('ends a background round at once, never stalling the main thread, and a send then waits only for the lock', async () => { + const current = await settlementOwed() + const { reconciliation } = current.host.collaboratorsForTests() + const { released } = await holdWriteLock(current.root, 1_500) + + // The exit's own refused settlement left a backoff: its timer runs the round. + const stall = await longestStall(async () => { + reconciliation.signal(SESSION) + await reconciliation.attempted(SESSION) + }) + expect(stall).toBeLessThan(60) + expect(retryOwes(reconciliation, SESSION)).toBe(true) + expect(turnState(current)).toBe('running') + + // A person's send waits the lock out (its own 5 s budget), never behind background work. + const sent = performance.now() + const { result } = current.send('go on') + expect(await result).toMatchObject({ ok: true }) + await released + expect(performance.now() - sent).toBeLessThan(2_000) + // Its commit proves the connection free: a fresh episode lands the settlement at once. + await eventually(() => expect(turnState(current)).toBe('interrupted')) + }) + + it('refuses a background transaction at once while a write in the same tick waits it out', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const { store } = rig + const database = openTestJournalHostDatabase(rig.root) + const busyTimeout = database.db.pragma('busy_timeout', { simple: true }) + const touch = + (by: number) => (record: Parameters[1]>[0]) => ({ + ...record, + lease: { ...record.lease, lastRenewedAt: record.lease.lastRenewedAt + by } + }) + const { released } = await holdWriteLock(rig.root, 400) + + const started = performance.now() + const background = store.transitionHandoff(SESSION, touch(1), { background: true }) + const user = store.transitionHandoff(SESSION, touch(2)) + + // Refused, though the lock lifts long inside its 5 s: it never waited. + await expect(background).rejects.toSatisfy(isSqliteContentionFailure) + await expect(user).resolves.toMatchObject({ sessionId: SESSION }) + expect(performance.now() - started).toBeGreaterThan(300) + await released + expect(database.db.pragma('busy_timeout', { simple: true })).toBe(busyTimeout) + }) +}) + +describe("the pass's reopen mark", () => { + it('is background: under a real lock it fails at once, ends the round, and lands once the lock lifts', async () => { + rig = await createQueuedMessageTestRig() + const current = rig + await current.workingSend() + await current.send('queued before the crash', 'queue-if-active').result + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const mark = AgentSessionJournal.prototype.markQueueReopen + const tried: { options: unknown; failed: boolean; ms: number }[] = [] + vi.spyOn(AgentSessionJournal.prototype, 'markQueueReopen').mockImplementation(async function ( + this: AgentSessionJournal, + ...args: Parameters + ) { + // The first mark meets another connection's lock, taken after the pass's earlier writes. + if (tried.length === 0) { + await holdWriteLock(current.root, 1_000) + } + const started = performance.now() + try { + await mark.apply(this, args) + tried.push({ options: args[2], failed: false, ms: performance.now() - started }) + } catch (error) { + const ms = performance.now() - started + tried.push({ options: args[2], failed: isSqliteContentionFailure(error), ms }) + throw error + } + }) + await current.crashRestartHostProcess() + + await vi.waitFor(() => expect(tried.some((attempt) => !attempt.failed)).toBe(true), { + timeout: 8_000 + }) + expect(tried[0]).toMatchObject({ options: { background: true }, failed: true }) + expect(tried[0]!.ms).toBeLessThan(150) + expect(tried.at(-1)).toMatchObject({ options: { background: true }, failed: false }) + }) +}) + +describe('an episode the retry gives up', () => { + /** Lets every visit a fired timer began run to its end, with no fake time passing. */ + async function settleTurns(): Promise { + for (let turn = 0; turn < 50; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } + } + + async function exhaust(current: QueuedMessageTestRig): Promise> { + const attempts = vi + .spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') + .mockRejectedValue(locked()) + // Past the episode's budget of rounds failing in a row (`RECONCILIATION_GIVE_UP_MS`). + for (let round = 0; round < 100; round += 1) { + await vi.advanceTimersByTimeAsync(2_000) + await settleTurns() + } + const given = attempts.mock.calls.length + await vi.advanceTimersByTimeAsync(120_000) + await settleTurns() + // Then no timer: the set is kept, and nothing retries on its own. + expect(attempts.mock.calls.length).toBe(given) + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(true) + attempts.mockRestore() + return attempts + } + + it.each(['a commit on the connection', 'a new signal'])( + 'is re-armed by %s, and the work lands', + async (trigger) => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) + const current = await settlementOwed() + await exhaust(current) + expect(turnState(current)).toBe('running') + + const { reconciliation } = current.host.collaboratorsForTests() + if (trigger === 'a new signal') { + reconciliation.signal(SESSION) + } else { + await currentJournal(current).appendQueueResume( + current.store.getRecord(SESSION)!.lease.runtimeFence + ) + } + await settleTurns() + + expect(turnState(current)).toBe('interrupted') + expect(retryOwes(reconciliation, SESSION)).toBe(false) + } + ) +}) + +describe('a queued send the retry gives up on', () => { + it('reads as not sent from memory when its stored hold cannot be written either', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockRejectedValue(locked()) + const hold = vi.spyOn(JournalQueuedMessages.prototype, 'hold').mockRejectedValue(locked()) + + await current.settleAccepted(working, 'a') + for (let round = 0; round < 100; round += 1) { + await vi.advanceTimersByTimeAsync(2_000) + await new Promise((resolve) => setImmediate(resolve)) + } + + expect(hold).toHaveBeenCalledTimes(1) + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages).toEqual([ + expect.objectContaining({ + messageId: queued.value.queued.messageId, + paused: true, + pausedReason: QUEUED_MESSAGE_PAUSED_SEND_FAILED + }) + ]) + expect(page.ok && page.page.nextQueuedMessageId).toBeNull() + }) + + it('holds the card with "Couldn\'t send" at once for a refusal', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + await current.send('after this', 'queue-if-active').result + vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockRejectedValueOnce( + new Error('disk full') + ) + + await current.settleAccepted(working, 'a') + await eventually(async () => { + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages?.[0]?.pausedReason).toBe( + QUEUED_MESSAGE_PAUSED_SEND_FAILED + ) + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts new file mode 100644 index 000000000000..b1e8dbae5e01 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts @@ -0,0 +1,84 @@ +// What bookkeeping no person waits on may write: the retry's visit, the queue's automatic drain, +// and an exit's own settlement and release. Each handle carries the writes such a path makes, +// every one passing `{ background: true }` (`JournalWriteOptions`), so another connection's lock +// fails it at once instead of holding the main thread for the busy timeout. The paths read through +// types without these writes, so none can reach a write that waits. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { JournalContextController } from '../agent-session-journal/journal-context-controller' +import type { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' + +/** Bookkeeping's write options, for a call that takes them directly. */ +export const BACKGROUND_WRITE = { background: true } as const +const BACKGROUND = BACKGROUND_WRITE + +type Background = { readonly background: true } + +/** A settlement's one write (`settleStructuredAgentSessionLeftovers`). */ +export type BackgroundSettlementWrites = Pick & + Background + +export type BackgroundJournalWrites = BackgroundSettlementWrites & + Pick< + AgentSessionJournal, + 'resolveDispatch' | 'appendSubmission' | 'markQueueReopen' | 'replaceEpochItems' + > & { + queuedMessages: Pick + context: Pick + } + +/** A lease release's one write. */ +export type BackgroundLeaseWrites = Pick & Background + +export type BackgroundStoreWrites = BackgroundLeaseWrites & + Pick + +export function backgroundSettlementWrites( + journal: Pick +): BackgroundSettlementWrites { + return { + ...BACKGROUND, + appendPlannedLifecycleBatch: (input) => + journal.appendPlannedLifecycleBatch({ ...input, ...BACKGROUND }) + } +} + +export function backgroundJournalWrites(journal: AgentSessionJournal): BackgroundJournalWrites { + const { queuedMessages: queued, context } = journal + return { + ...backgroundSettlementWrites(journal), + resolveDispatch: (input, hook) => journal.resolveDispatch({ ...input, ...BACKGROUND }, hook), + appendSubmission: (input, consume, receipt) => + journal.appendSubmission(input, consume && { ...consume, ...BACKGROUND }, receipt), + markQueueReopen: (fence, since) => journal.markQueueReopen(fence, since, BACKGROUND), + replaceEpochItems: (reason, fence, items) => + journal.replaceEpochItems(reason, fence, items, BACKGROUND), + queuedMessages: { + hold: (input) => queued.hold({ ...input, ...BACKGROUND }), + repairAndPrune: () => queued.repairAndPrune(BACKGROUND), + settleOwed: () => queued.settleOwed(BACKGROUND) + }, + context: { + rewind: (floor, fence, items, receipt) => + context.rewind(floor, fence, items, receipt, BACKGROUND) + } + } +} + +export function backgroundLeaseWrites( + store: Pick +): BackgroundLeaseWrites { + return { + ...BACKGROUND, + transitionHandoff: (sessionId, transition) => + store.transitionHandoff(sessionId, transition, BACKGROUND) + } +} + +export function backgroundStoreWrites(store: AgentSessionRecordStore): BackgroundStoreWrites { + return { + ...backgroundLeaseWrites(store), + recordOperationOutcome: (args) => store.recordOperationOutcome(args, BACKGROUND) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts index b6cd948fa4dc..1e508636ebf0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts @@ -156,6 +156,8 @@ export function mutateWithChatStop( windDown = { owed, ctx } }, withdrewQueued: withdrew, + noteFence: () => + context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? ctx.fence, ...(ownsLatestStop ? { opensSettle: true as const } : {}) } ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index 524b4fd4d3f2..158c6c0b048c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -10,8 +10,13 @@ import { refuse } from '../../../shared/agent-session-wire-refusals' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' -import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' +import type { + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild +} from './structured-agent-session-host-types' import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' +import type { BackgroundLeaseWrites } from './structured-agent-session-background-writes' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { releaseStoredStructuredAgentSessionOwnerAfterExit } from './structured-agent-session-lease-release' import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' @@ -102,7 +107,10 @@ function closeProviderRoot( /** Whether the lease still names the child this host last proved gone, unreleased: only that * in-memory proof lets this host release it without a probe, so the handle carrying it stays. */ export function structuredAgentSessionEndedChildHoldsLease( - context: Pick, + context: { + deps: Pick + sessions: ReadonlyMap> + }, sessionId: string ): boolean { const ended = context.sessions.get(sessionId)?.lastEndedChild @@ -116,11 +124,13 @@ export function structuredAgentSessionEndedChildHoldsLease( } /** Writes the release a proven exit allows when the exit handler could not: a start and the - * handle's close re-derive it, and a failure is reported, never anyone's refusal. Resolves - * whether the lease still names that child. */ + * handle's close re-derive it, and a failure is reported, never anyone's refusal. The retry + * writes it through its background handle, where another connection's lock throws for its next + * round. Resolves whether the lease still names that child. */ export async function releaseLeaseOfEndedStructuredAgentSessionChild( context: Pick, - sessionId: string + sessionId: string, + background?: BackgroundLeaseWrites ): Promise { const ended = context.sessions.get(sessionId)?.lastEndedChild if (!ended || !structuredAgentSessionEndedChildHoldsLease(context, sessionId)) { @@ -128,16 +138,20 @@ export async function releaseLeaseOfEndedStructuredAgentSessionChild( } await releaseStoredStructuredAgentSessionOwnerAfterExit({ store: context.deps.store, + writes: background ?? context.deps.store, sessionId, expectedFence: ended.fence, now: context.now(), ...(ended.reason ? { exitReason: ended.reason } : {}) - }).catch((error: unknown) => + }).catch((error: unknown) => { + if (background && isSqliteContentionFailure(error)) { + throw error + } context.deps.logger.warn("releasing an exited agent's lease failed", { scope: 'ended-child-lease-release', sessionId, error }) - ) + }) return structuredAgentSessionEndedChildHoldsLease(context, sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 694268a1f8bf..47312959a38e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -9,19 +9,28 @@ import { } from '../../../shared/agent-session-failure' import { PROVIDER_EXIT_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' import type { StructuredAgentSessionHostSession, StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' -import { endProviderChild } from './structured-agent-session-provider-child' +import { recordProviderChildEnd } from './structured-agent-session-provider-child' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import { + backgroundLeaseWrites, + backgroundSettlementWrites +} from './structured-agent-session-background-writes' import { releaseStoredStructuredAgentSessionOwnerAfterExit, type StructuredAgentSessionLeaseStore } from './structured-agent-session-lease-release' import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' -import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' +import { + settleStructuredAgentSessionLeftovers, + type StructuredAgentSessionExitSettlement, + type StructuredAgentSessionLeftoverStore +} from './structured-agent-session-leftover-settlement' +import type { StaleStructuredAgentSessionStateJournal } from './structured-agent-session-stale-state-settlement' import { captureUnfinishedStructuredAgentSessionWork, type DeadGenerationJournal, @@ -47,16 +56,26 @@ export type StructuredAgentSessionChildExit = { export type StructuredAgentSessionChildExitSession = Pick< StructuredAgentSessionHostSession, 'child' | 'lastEndedChild' -> & { journal: DeadGenerationJournal & Pick } +> & { journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal } + +/** The host retry's signal (`StructuredAgentSessionRetry.signal`): the one edge every way a + * generation ends goes through. `exit`: the exit's own settlement, owed when + * it did not land. */ +export type StructuredAgentSessionGenerationEnded = ( + sessionId: string, + options?: { restate?: boolean; exit?: StructuredAgentSessionExitSettlement } +) => void export type StructuredAgentSessionChildExitContext< TSession extends StructuredAgentSessionChildExitSession = StructuredAgentSessionHostSession > = { - store: StructuredAgentSessionLeaseStore + store: StructuredAgentSessionLeaseStore & StructuredAgentSessionLeftoverStore sessions: Map flushLifecycle: (sessionId: string) => Promise - publishFence: (sessionId: string, session: TSession) => void publishStatus?: (sessionId: string) => void + /** A generation ended: every reader re-derives current work, the queued-card drain is + * scheduled with no journal write needed, and the host's retry settles what is owed. `restate`: each chat re-baselines at the moved fence. */ + generationEnded: StructuredAgentSessionGenerationEnded /** The delivery loop hands over whatever is queued once the child is off the record. */ wakeDelivery?: (sessionId: string) => void /** An ended child's start, if still open, has nothing left to time. */ @@ -145,20 +164,27 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< ? ('hostRestarted' as const) : ('chatClosed' as const) : undefined + // Death first: before the tail, the settlement or the release writes anything, every reader + // projects this generation as ended, so no frame they publish reads it Working and no prompt it + // raised is answerable. The child stays on record (its sink drains the tail) until `endChild`. + recordProviderChildEnd(session, { + generation: child.generation, + fence: child.fence, + // A close keeps the cause of the stop that asked for it, and ends where it was asked. + cause: expected ? (close?.cause ?? 'evict') : 'exit', + reason: expected ? (close?.reason ?? null) : exit.reason, + ...(!expected && exit.failure ? { failure: exit.failure } : {}), + duringStartup: exitedDuringStartup, + // The adapter publishes an exit only once it saw the root go, first-hand or proven. + rootGone: true, + ...(expected && close ? { endedAt: close.requestedAt } : {}) + }) + context.generationEnded(sessionId) const endChild = (): void => { context.startupAttempts?.childEnded(sessionId, child) - endProviderChild(session, { - generation: child.generation, - fence: child.fence, - // A close keeps the cause of the stop that asked for it, and ends where it was asked. - cause: expected ? (close?.cause ?? 'evict') : 'exit', - reason: expected ? (close?.reason ?? null) : exit.reason, - ...(!expected && exit.failure ? { failure: exit.failure } : {}), - duringStartup: exitedDuringStartup, - // The adapter publishes an exit only once it saw the root go, first-hand or proven. - rootGone: true, - ...(expected && close ? { endedAt: close.requestedAt } : {}) - }) + if (session.child === child) { + session.child = null + } context.publishStatus?.(sessionId) } const record = context.store.getRecord(sessionId) @@ -168,6 +194,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< context.wakeDelivery?.(sessionId) return } + let settlement: StructuredAgentSessionExitSettlement | null = null try { // The exited child's own writes land first: its dead generation is settled from all of them. try { @@ -191,13 +218,11 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // Folded before the fallback's end is built, so the end reads it (`turnEndAfterStop`). await close?.recorded const generation = child.generation ?? 'unknown' - // The exit proves this child gone, as the record's death evidence later says: what it left - // `unverifiable` is revised now, not at the next open. + // The exit proves this child gone, as the record's death evidence says once released: what it + // left `unverifiable` is revised now. const watched = { ownerFence: child.fence, observedAt } - const settled = await settleStructuredAgentSessionDeadGeneration({ - journal: session.journal, - sessionId, - fence: child.fence, + settlement = { + ownerFence: child.fence, settlementId: `${expected ? 'expected-close:' : PROVIDER_EXIT_ROW_PREFIX}${sessionId}:${child.fence}:${generation}`, pendingSubmissionReason: expected ? 'provider_closed_before_acknowledgement' @@ -222,18 +247,36 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< : {}), exit: watched, ...(unrunRejection ? { unrunRejection } : {}) - }) - if (!settled.ok) { - logExitFailure(context, sessionId, 'exit-settlement', settled.error) } } finally { - // The root's exit was observed, so the owner is released even when terminal settlement could - // not be durably accepted. Bare cause: whatever this settlement could not write is settled from - // it later (the settle recording it queues, or the next open or acquire). + // What the exited child left is settled first, at its own fence, as its own closing tail lands: + // a row its closed sink still delivers lands too, and wakes the retry to settle it once the + // release below moves the fence. Both are bookkeeping (`structured-agent-session-background- + // writes.ts`): another connection's lock fails them at once, and what did not commit stays owed + // to the retry, the settlement as the exit's debt and the release as its repair. let released = false + let owed: StructuredAgentSessionExitSettlement | null = null + let contended = false + if (settlement) { + const settled = await settleStructuredAgentSessionLeftovers({ + store: context.store, + sessionId, + journal: session.journal, + writes: backgroundSettlementWrites(session.journal), + exit: settlement, + // Ended whether or not the release lands: the host saw the root go. + ended: { fence: child.fence, rootGone: true } + }) + if (!settled.ok) { + owed = settlement + contended = isSqliteContentionFailure(settled.error) + logExitFailure(context, sessionId, 'exit-settlement', settled.error) + } + } try { await releaseStoredStructuredAgentSessionOwnerAfterExit({ store: context.store, + writes: backgroundLeaseWrites(context.store), sessionId, expectedFence: child.fence, now: context.now(), @@ -242,6 +285,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< }) released = true } catch (error) { + contended ||= isSqliteContentionFailure(error) logExitFailure(context, sessionId, 'exit-owner-release', error) } if (context.route) { @@ -256,10 +300,14 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< } endChild() // A reader re-baselines on a death of the child's own; a close Orca asked for moves no fence a - // reader holds, as a client resends a message when its fence moves. - if (released && !expected) { - context.publishFence(sessionId, session) - } + // reader holds, as a client resends a message when its fence moves, and discards a reply (the + // Stop's own) issued at the fence before. Either way every reader learns the generation ended + // and the queued-card drain runs, whether or not the release or the settlement was written. + context.generationEnded(sessionId, { + restate: released && !expected, + ...(owed ? { exit: owed } : {}), + ...(contended ? { contended } : {}) + }) context.wakeDelivery?.(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index 2ea3a16024be..35763ea56a30 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -145,7 +145,7 @@ describe('Claude root-exit stop', () => { store, sessions, flushLifecycle: (id) => runtimeState.lifecycleBarrier(id), - publishFence: () => undefined, + generationEnded: () => undefined, publishStatus, serialize: (_sessionId, task) => task(), now: () => NOW + 30 * 60_000, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index 00a0644136c3..3c5ca8cd311a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -397,9 +397,9 @@ it('sends after a proven exit whose resume-point write and lease release both fa expect.arrayContaining(['claude-close-resume-point', 'exit-owner-release']) ) ) - expect(lease()?.claimStatus).toBe('live') + // The host's retry writes the release from this host's proof of that exit; no send waits on it. + await eventually(() => expect(lease()?.claimStatus).toBe('released')) - // The start writes the release from this host's proof of that exit. await send('Carry on.') await resumedWith(connection, 'Carry on.') }) @@ -419,7 +419,8 @@ it('sends after a crash with a long reason whose own lease release failed', asyn ) await eventually(() => expect(child()).toBeNull()) await eventually(() => expect(scopes()).toContain('exit-owner-release')) - expect(lease()?.claimStatus).toBe('live') + // The host's retry writes the release the exit's own write could not. + await eventually(() => expect(lease()?.claimStatus).toBe('released')) await send('Carry on.') await resumedWith(connection, 'Carry on.') @@ -526,7 +527,8 @@ it('has the idle reaper join a close still unverifiable, ending the record with expect(connection.closeCount).toBe(2) expect(lease()).toMatchObject({ claimStatus: 'released', ownerProcess: null }) - expect(host.hasSession(SESSION)).toBe(false) + // Closed once the host's retry has settled what the ended child left. + await eventually(() => expect(host.hasSession(SESSION)).toBe(false)) }) it("ends a Claude journal-sink failure as Orca's own fault, never a quiet rest", async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index c369ce1a5602..1d99f1379f71 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -7,7 +7,8 @@ import type { SubscriberFieldHooks } from './agent-session-subscriber-frame-fiel import { AgentSessionSubscribers } from './structured-agent-session-subscribers' import { structuredQueueSendGate, - tryReadQueuePublication + tryReadQueuePublication, + type QueuedDrainHolds } from './structured-agent-session-queued-publication' import type { StructuredAgentSessionHostDeps, @@ -15,6 +16,8 @@ import type { } from './structured-agent-session-host-types' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' +import { heldStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import type { StructuredAgentSessionStatusSubscriber } from './structured-agent-session-status-feed' import { createStructuredAgentSessionHostStatusFeed } from './structured-agent-session-host-status-feed' import { @@ -35,7 +38,12 @@ export class StructuredAgentSessionClientDelivery { private readonly sessions: Map, now: () => number, private readonly deps: () => StructuredAgentSessionHostDeps, - private readonly onJournalActivity: (sessionId: string) => void, + /** The queued-card drain: every publish wakes it (`activity`: a journal write, which renews the + * idle clock; a generation's end is not one), and a card it holds back is never named next. */ + private readonly queue: { + onJournalActivity: (sessionId: string, activity?: boolean) => void + drain: QueuedDrainHolds + }, onAgentStarted: (sessionId: string) => void, /** A session's child records changed; the chat strip republishes from them. */ onChildWorkChanged: (sessionId: string) => void, @@ -56,7 +64,7 @@ export class StructuredAgentSessionClientDelivery { this.statusFeed.journalProjection(sessionId, journal)?.state ?? null }) this.sendSettlement = new StructuredAgentSessionSendSettlement((sessionId) => - this.requireJournal(sessionId) + this.settlementReading(sessionId, this.requireJournal(sessionId)) ) this.waitForSendSettlement = this.sendSettlement.wait this.subscribers = new AgentSessionSubscribers({ @@ -64,10 +72,15 @@ export class StructuredAgentSessionClientDelivery { readQueuePublication: (sessionId) => tryReadQueuePublication( sessions.get(sessionId)?.journal, - structuredQueueSendGate(this.deps().store, sessionId) + structuredQueueSendGate( + { store: this.deps().store, sessions }, + sessionId, + this.queue.drain + ) ), readBackgroundTasks, - onJournalPublished: (sessionId, journal) => this.publishJournal(sessionId, journal) + onJournalPublished: (sessionId, journal) => this.publishJournal(sessionId, journal), + readCurrentWork: (sessionId, journal) => this.currentWork(sessionId, journal) }) } @@ -110,10 +123,35 @@ export class StructuredAgentSessionClientDelivery { this.statusFeed.publish(sessionId) const journal = this.sessions.get(sessionId)?.journal if (journal) { - this.sendSettlement.publish(sessionId, journal) + this.sendSettlement.publish(sessionId, this.settlementReading(sessionId, journal)) } } + /** A generation ended — an exit, a release — which can change what is current with no journal + * row. The one edge for every way a generation ends: every reader re-derives current work (the + * status feed, settlement waits, each chat's frames) and the queued-card drain is scheduled, + * whatever became of the cleanup's own write. An end is no activity: it renews no idle clock. + * `restate`: each chat re-baselines at the fence, as a death of the child's own does. */ + publishGenerationEnded = (sessionId: string, options: { restate?: boolean } = {}): void => { + const journal = this.sessions.get(sessionId)?.journal + if (!journal) { + return + } + if (options.restate) { + const fence = structuredAgentSessionConversationFence(this.deps().store, sessionId) + this.subscribers.snapshot(sessionId, journal, fence) + return + } + this.subscribers.deliverFrames(sessionId, journal) + this.publishJournal(sessionId, journal, false) + } + + /** A held chat's current work (`structuredAgentSessionCurrentWork`); null when none is held. */ + readCurrentWork = (sessionId: string) => { + const journal = this.sessions.get(sessionId)?.journal + return journal ? this.currentWork(sessionId, journal) : null + } + publishRestored = (sessionId: string): void => { this.statusFeed.publish(sessionId, undefined, { replay: true }) this.turnCompletionFeed.observe(sessionId, undefined, { historical: true }) @@ -148,14 +186,28 @@ export class StructuredAgentSessionClientDelivery { this.sendSettlement.closeAll() } - private publishJournal(sessionId: string, journal: AgentSessionJournal): void { + private publishJournal(sessionId: string, journal: AgentSessionJournal, activity = true): void { this.statusFeed.publish(sessionId, journal) - this.sendSettlement.publish(sessionId, journal) + this.sendSettlement.publish(sessionId, this.settlementReading(sessionId, journal)) // Derived here rather than per-subscriber: this edge runs whether or not anyone is // subscribed, which is the whole reason a backgrounded chat can complete at all. After the // status publish, so it reads the projection that publish cached. this.turnCompletionFeed.observe(sessionId, journal) - this.onJournalActivity(sessionId) + this.queue.onJournalActivity(sessionId, activity) + } + + /** The current-work projection over this journal (`structuredAgentSessionCurrentWork`). */ + private currentWork(sessionId: string, journal: AgentSessionJournal) { + const session = this.sessions.get(sessionId) + return heldStructuredAgentSessionCurrentWork(this.deps().store, sessionId, journal, session) + } + + private settlementReading(sessionId: string, journal: AgentSessionJournal) { + return { + submissions: () => journal.submissions(), + cursor: () => journal.cursor(), + activeTurnId: () => this.currentWork(sessionId, journal).activeTurnId() + } } private requireJournal(sessionId: string): AgentSessionJournal { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts index 30810b9b9f3e..302bc99646af 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts @@ -44,6 +44,10 @@ import type { StructuredAgentRegistry } from './structured-agent-registry' import { conversationCommandBlocked } from './structured-conversation-command-admission' import { structuredAgentSessionCommandTurn } from '../../../shared/structured-agent-session-command-turn-identity' import { refuseQueuedCommand } from './structured-agent-session-queued-command-refusal' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' export { structuredAgentSessionCommandTurn } from '../../../shared/structured-agent-session-command-turn-identity' @@ -90,11 +94,12 @@ export function structuredAgentSessionAwaitedCommand( : undefined } -/** Whether the journal's running turn is a command's, which takes no input while it runs. */ +/** Whether the live generation's running turn is a command's, which takes no input while it runs + * (`structuredAgentSessionCurrentWork`). */ export function structuredAgentSessionCommandRunning( - journal: Pick + work: Pick ): boolean { - const turnId = journal.activeTurnId() + const turnId = work.activeTurnId() return turnId !== null && isStructuredAgentSessionCommandTurnId(turnId) } @@ -220,7 +225,8 @@ async function openCommandTurn( clientMessageId, state: 'pending', fence: ctx.fence, - turnScope: ctx.journal.liveTurnScope() + // The child it goes to is live at `fence`: an ended generation's turn is not joined. + turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() }) const startedAt = ctx.now() const running = agentJournalTurnBody({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 431ca9e761de..bba4aeec2ca2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -84,6 +84,10 @@ export function createStructuredAgentSessionConversationLifetime(host: { isDisposed: () => disposed, deliveryActive: host.deliveryActive, childWork: host.readChildWork, + store: { + getRecord: (sessionId) => deps().store.getRecord(sessionId), + replacedRuntime: (sessionId) => deps().store.replacedRuntime(sessionId) + }, hasOpenDispatch: (sessionId) => { const record = deps().store.getRecord(sessionId) return record !== null && deps().hasOpenDispatch?.(record) === true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index 85738139603c..062e3008c85c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -1,19 +1,14 @@ // The one way a conversation's journal becomes open on this host: for a send, for a reader, and // for an attach that finds none open. // -// A damaged journal fails the open, which refuses it as unloadable. It marks what an earlier host -// process handed over and left unanswered as in doubt, and settles what it left running — the -// crash boundary. That -// needs no lease: provider history decides such a row later, under a won lease, in the attach. A -// row an earlier process accepted and never handed over (it quit or crashed first) is settled here -// too, before any reader, command or child sees it: a person's message is kept as a card, the rest -// rejected (`journal-unsent-send-hold.ts`). The cards then wait for the chat's next turn -// (`queued-message-pause.ts`). Nothing here starts a provider child. +// It only reads. A damaged journal fails the open, which refuses it as unloadable. What an earlier +// generation or host process left running or queued is settled by the ownership event that ended +// it — its exit, the next acquisition, or startup (`structured-agent-session-leftover-settlement.ts`, +// `structured-agent-session-startup-settlement.ts`) — never here. Nothing here starts a provider +// child. import type { JournalHostDatabase } from '../agent-session-journal/journal-host-database' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { @@ -22,12 +17,6 @@ import { type AgentSessionAttachParams } from './structured-agent-session-attach' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' -import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import { - markStructuredQueueReopen, - structuredAgentSessionHostInstance -} from './structured-agent-session-queued-pause' import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession @@ -38,17 +27,11 @@ export type OpenedStructuredAgentSessionConversation = { } export type StructuredAgentSessionConversationOpenDeps = { - store: Pick + store: Pick journalDatabase: JournalHostDatabase logger: StructuredAgentSessionHostDeps['logger'] } -/** An acquisition's own open: its reserve cleared the record's death evidence, so it settles - * what the gone generation left running itself, from what it read before. */ -export type StructuredAgentSessionConversationOpenOptions = { - acquisition?: boolean -} - export type StructuredAgentSessionConversationOpenContext = { deps: StructuredAgentSessionConversationOpenDeps sessions: Map @@ -63,8 +46,7 @@ export type StructuredAgentSessionConversationOpenContext = { * session's serialize, which is what makes "not open yet" exact. */ export async function openStructuredAgentSessionConversation( context: StructuredAgentSessionConversationOpenContext, - sessionId: string, - options: StructuredAgentSessionConversationOpenOptions = {} + sessionId: string ): Promise { const open = context.sessions.get(sessionId) if (open) { @@ -74,118 +56,27 @@ export async function openStructuredAgentSessionConversation( if (!record) { return null } - const opened = await openStructuredAgentSessionConversationJournal(context.deps, record, options) + const opened = await openStructuredAgentSessionConversationJournal(context.deps, record) await context.adoptOpened(sessionId, opened) return opened.session } -/** The open itself, indexed by nobody yet: the caller adopts the result. Any open that settles - * needs the store, which names the runtimes this one replaced. */ -export async function openStructuredAgentSessionConversationJournal( - deps: StructuredAgentSessionConversationOpenDeps, - record: AgentSessionRecord, - options?: StructuredAgentSessionConversationOpenOptions -): Promise -/** An acquisition's open settles nothing itself, so it needs no store. */ +/** The open itself, indexed by nobody yet: the caller adopts the result. */ export async function openStructuredAgentSessionConversationJournal( - deps: Omit, - record: AgentSessionRecord, - options: { acquisition: true } -): Promise -export async function openStructuredAgentSessionConversationJournal( - deps: Omit & - Partial>, - record: AgentSessionRecord, - options: StructuredAgentSessionConversationOpenOptions = {} + deps: Omit, + record: AgentSessionRecord ): Promise { const { sessionId } = record - const fence = record.lease.runtimeFence const params = attachParamsForRecord(record, { clientOperationId: `read-restore:${sessionId}`, - expectedRuntimeFence: fence + expectedRuntimeFence: record.lease.runtimeFence }) const identity = journalIdentityFor(record, params) const journal = await openAgentSessionJournal({ identity, database: deps.journalDatabase }) - try { - // A handed-over row found here is only doubt, which provider history decides under a won lease. - await journal.markPendingSubmissionsUnknown(fence) - } catch (error) { - deps.logger.warn('marking pending sends unknown on open failed', { - scope: 'open-pending-unknown', - sessionId, - error - }) - } - try { - // Before a Stop this open serves can withdraw one: a Stop never withdraws a card. - await holdUnsentSends(journal, { - fence, - hostInstance: structuredAgentSessionHostInstance(), - hold: { cause: 'hostRestarted' } - }) - } catch (error) { - // The row stays queued. The delivery loop's first step tries again before it hands anything - // over; if that fails too, the loop fails and rejects every queued send. - deps.logger.warn('settling sends an earlier process left queued failed on open', { - scope: 'open-leftover-sends', - sessionId, - error - }) - } - // After the leftovers became cards, so the mark follows every card this open found. Every open - // comes after the chat stopped running: the idle sweep never closes one with cards waiting. - await markStructuredQueueReopen(sessionId, journal, fence, deps.logger) - // No child in this process writes to a journal nobody had open, so whatever it shows running - // belongs to a generation that is gone, whatever the lease still claims. Settled before any - // reader or child sees it. - if (!options.acquisition) { - await settleGoneGeneration(deps, record, journal) - } + journal.holdReopenFromOpen() return { session: { journal, params, child: null } } } -/** - * The open's settle again, for a conversation already open: a proof of death written since it - * opened (the startup reconcile, a recovery) revises what the open could only call `unverifiable`. - * A record holds a proof only while released, so no child here is writing. A no-op once revised. - */ -export async function resettleOpenStructuredAgentSessionConversation( - deps: StructuredAgentSessionConversationOpenDeps, - sessionId: string, - session: StructuredAgentSessionHostSession | undefined -): Promise { - const record = deps.store.getRecord(sessionId) - if (session && record?.lease.deathEvidence) { - await settleGoneGeneration(deps, record, session.journal) - } -} - -async function settleGoneGeneration( - deps: Pick & - Partial>, - record: AgentSessionRecord, - journal: AgentSessionJournal -): Promise { - try { - await settleStaleStructuredAgentSessionState({ - journal, - sessionId: record.sessionId, - fence: record.lease.runtimeFence, - acquisitionGeneration: null, - deathEvidence: record.lease.deathEvidence ?? null, - replaced: deps.store?.replacedRuntime(record.sessionId), - failureTextContext: structuredAgentSessionFailureWordsContext(record) - }) - } catch (error) { - // Best effort: the next open or acquire re-derives it. - deps.logger.warn("settling a gone agent's work on open failed", { - scope: 'open-dead-generation', - sessionId: record.sessionId, - error - }) - } -} - export function attachParamsForRecord( record: AgentSessionRecord, input: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts index 94d08c054c51..e3b808989089 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts @@ -67,7 +67,7 @@ describe('a conversation delivers what its journal commits', () => { // Already delivered when the writer's await returns, as an explicit publish would have been. expect(deliver).toHaveBeenCalledOnce() - expect(deliver).toHaveBeenCalledWith('session-1', journal) + expect(deliver).toHaveBeenCalledWith('session-1', journal, { lowestFence: 0 }) }) it('binds a handle set through a plain map reference', async () => { @@ -84,7 +84,7 @@ describe('a conversation delivers what its journal commits', () => { await appendStatus(journal, 'through the plain map') - expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal) + expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal, expect.anything()) }) it('delivers an epoch replacement, which readers must reload from', async () => { @@ -99,7 +99,7 @@ describe('a conversation delivers what its journal commits', () => { await journal.replaceEpochItems('handle_forked', 0, []) - expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal) + expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal, expect.anything()) }) it('delivers nothing for a handle the conversation has replaced', async () => { @@ -118,7 +118,7 @@ describe('a conversation delivers what its journal commits', () => { expect(deliver).not.toHaveBeenCalled() await appendStatus(current, 'live') - expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', current) + expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', current, expect.anything()) }) it('delivers nothing once the conversation is dropped', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts index b4ee124597f6..0bff14727402 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts @@ -21,12 +21,17 @@ export class StructuredAgentSessionConversations extends Map< constructor( private readonly delivery: { - deliver: (sessionId: string, journal: AgentSessionJournal) => void + /** `lowestFence`: the lowest fence a row committed since the last delivery was written at. */ + deliver: ( + sessionId: string, + journal: AgentSessionJournal, + committed: { lowestFence: number | null } + ) => void /** A person's Stop settle opened or closed: no row, so neither a publish nor activity. */ deliverSettleEdge?: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger /** A conversation became held: state that waited on it (queued drafts) re-derives. */ - onOpened?: (sessionId: string) => void + onOpened?: (sessionId: string, journal: AgentSessionJournal) => void now: () => number } ) { @@ -36,18 +41,24 @@ export class StructuredAgentSessionConversations extends Map< override set(sessionId: string, session: StructuredAgentSessionHostSession): this { const { journal } = session let queued = false - journal.observeCommits(() => { + let lowestFence: number | null = null + journal.observeCommits((rows) => { + for (const row of rows ?? []) { + lowestFence = Math.min(lowestFence ?? row.fence, row.fence) + } if (queued) { return } queued = true queueMicrotask(() => { queued = false + const committed = { lowestFence } + lowestFence = null if (this.get(sessionId)?.journal !== journal) { return } try { - this.delivery.deliver(sessionId, journal) + this.delivery.deliver(sessionId, journal, committed) } catch (error) { this.delivery.logger.warn('delivering a journal commit failed', { scope: 'journal-delivery', @@ -75,7 +86,7 @@ export class StructuredAgentSessionConversations extends Map< }) this.activity.set(sessionId, this.delivery.now()) const adopted = super.set(sessionId, session) - this.delivery.onOpened?.(sessionId) + this.delivery.onOpened?.(sessionId, session.journal) return adopted } @@ -93,4 +104,9 @@ export class StructuredAgentSessionConversations extends Map< lastActivityAt(sessionId: string): number | undefined { return this.activity.get(sessionId) } + + /** Some conversation holds a running provider child. */ + hasLoadedProviders(): boolean { + return [...this.values()].some(({ child }) => child !== null) + } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index cd9ecc7e7418..4aa5548aa3f8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -32,18 +32,14 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { AgentSessionAcquisitionExitUnprovenError, type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' -import { resettleOpenStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { withNativeChatCutTurnNotices } from '../../../shared/native-chat-cut-turn-notice' import { latestNativeChatOrcaStopCut } from '../../../shared/native-chat-orca-stop-cut' import { beginAgentSessionRuntimeRecord } from '../../runtime/agent-session-runtime-end-record' -import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { hostTestAttachParams, @@ -54,6 +50,7 @@ import { createStructuredAgentSessionLogger } from './structured-agent-session-l import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const PROVIDER_SESSION = 'provider-session-alpha-1' /** The tool call's row: the last thing the provider wrote before the crash. */ @@ -183,9 +180,6 @@ function openHost(overrides: Partial): void { }) } -/** The turn ended where its owner was last proven alive, whatever proof ended it. */ -const INTERRUPTED_TURN = { turnId: 'turn-1', state: 'interrupted', completedAt: LAST_RENEWED_AT } - /** What a client reads over the wire. */ async function settledTurn() { return (await host.journalSnapshot(SESSION)).items @@ -200,7 +194,6 @@ beforeEach(async () => { }) afterEach(async () => { - vi.restoreAllMocks() await host?.flushAllStreamedEvents() closeTestJournalHostDatabases() await rm(root, { recursive: true, force: true }) @@ -211,6 +204,7 @@ describe('a turn a crash cut short mid-tool', () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) await host.restoreReadableSessions() + await reconciled() expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ kind: 'pid-absent', @@ -233,6 +227,7 @@ describe('a turn a crash cut short mid-tool', () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) await host.restoreReadableSessions() + await reconciled() const { items } = await host.journalSnapshot(SESSION) // As a reader's transcript shows it: the stored row is the explanation, so none is derived. @@ -269,6 +264,7 @@ describe('a turn a crash cut short mid-tool', () => { openHost({ probeOwner, stopOwnerProcess }) await host.restoreReadableSessions() + await reconciled() expect(stopOwnerProcess).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ @@ -303,16 +299,28 @@ function attach(fence: number) { ) } -/** Runs whatever this session's serialize already has queued. */ -function drainSession(): Promise { - return host.collaboratorsForTests().serialize(SESSION, async () => {}) +/** Runs whatever this session's serialize already has queued, then the chat's reconciliation. */ +async function drainSession(): Promise { + await host.collaboratorsForTests().serialize(SESSION, async () => {}) + await reconciled() +} + +/** Once the host's retry settled what a generation's end left. */ +function reconciled(): Promise { + return retryIdle(host.collaboratorsForTests().reconciliation, SESSION) +} + +/** Startup: the reconcile, then the scan it begins. */ +async function startUp(): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() + await reconciled() } // On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner. -// The Orca that held the owner is gone (this one replaced it), so the open itself ends the turn; the -// probe's later proof has nothing left to revise. +// The open writes nothing; the startup after the reconcile settles the turn by its proof. describe('a turn a read reached before the reconcile proved its owner dead', () => { - it('reads interrupted at the last renewal from the first open, and a subscriber is sent it once', async () => { + it('reads as the crash left it, then interrupted at the last renewal, and a subscriber is sent both', async () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) const events: AgentSessionSubscribeEvent[] = [] const unsubscribe = await host.subscribe({ @@ -320,12 +328,17 @@ describe('a turn a read reached before the reconcile proved its owner dead', () sessionId: SESSION, emit: (event) => events.push(event) }) - expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) + expect(await settledTurn()).toEqual(RUNNING_TURN) - await host.reconcileRestartLeases() - await drainSession() + await startUp() - expect(turnStatesSent(events)).toMatchObject([INTERRUPTED_TURN]) + // A paired or phone client learns of the revision through the ordinary journal publish. + await vi.waitFor(() => + expect(turnStatesSent(events)).toMatchObject([ + { state: 'running' }, + { state: 'interrupted', completedAt: LAST_RENEWED_AT } + ]) + ) const { items } = await host.journalSnapshot(SESSION) const [timing] = selectStructuredAgentTurnTimings(items).values() expect(completedStructuredAgentTurnSeconds(timing)).toBe(27) @@ -333,7 +346,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () unsubscribe() }) - it('reports the open to the status feed as an interruption, which the chat folds as worked', async () => { + it('reports the revision to the status feed as an interruption, which the chat folds as worked', async () => { const published: AgentSessionStatusSummary[] = [] openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }), @@ -344,13 +357,13 @@ describe('a turn a read reached before the reconcile proved its owner dead', () published .filter((summary) => summary.sessionId === SESSION && summary.turnOutcome) .map((summary) => summary.turnOutcome) - // The sidebar's red Failed until seen; the turn folds as "Worked for 27s" beside its notice row. + // Nothing written yet, but the runtime that held the turn was replaced: it reads interrupted. expect(outcomes()).toEqual(['interruption']) - await host.reconcileRestartLeases() - await drainSession() + await startUp() - expect(outcomes().at(-1)).toBe('interruption') + // The sidebar's red Failed until seen; the turn folds as "Worked for 27s" beside its notice row. + await vi.waitFor(() => expect(outcomes().at(-1)).toBe('interruption')) const [timing] = selectStructuredAgentTurnTimings( (await host.journalSnapshot(SESSION)).items ).values() @@ -363,7 +376,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ).toEqual({ key: 'workedFor', duration: '27s' }) }) - it('explains the turn it cut, so the cut reads once and offers Continue', async () => { + it('explains the turn its proof revised, so the cut reads once and offers Continue', async () => { // The Orca that crashed recorded that it started, and never that it ended. const crashed = crashedClaudeRecord() const ownerProcess = crashed.lease.ownerProcess @@ -381,10 +394,9 @@ describe('a turn a read reached before the reconcile proved its owner dead', () beginAgentSessionRuntimeRecord(root, 'runtime-crashed', TOOL_STARTED_AT - 60_000) store = await openTestAgentSessionRecordStore(root) openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) - expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) + expect(await settledTurn()).toEqual(RUNNING_TURN) - await host.reconcileRestartLeases() - await drainSession() + await startUp() const { items } = await host.journalSnapshot(SESSION) const turnItemId = items.find((item) => item.body.kind === 'turn')?.itemId @@ -401,25 +413,17 @@ describe('a turn a read reached before the reconcile proved its owner dead', () it('revises nothing twice, whoever re-runs the settle', async () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) await host.history({ sessionId: SESSION, direction: 'tail' }) - await host.reconcileRestartLeases() - await drainSession() + await startUp() const settled = await host.journalSnapshot(SESSION) - await host - .collaboratorsForTests() - .serialize(SESSION, () => - resettleOpenStructuredAgentSessionConversation( - host.deps, - SESSION, - host.collaboratorsForTests().sessions.get(SESSION) - ) - ) + // A second startup pass and the restore's share find nothing left to do. + await startUp() await host.restoreReadableSessions() expect(await host.journalSnapshot(SESSION)).toEqual(settled) }) - it('ends it at the last renewal although recovery stops a child that outlived Orca later', async () => { + it('revises it when recovery stops a child that outlived Orca, with no send', async () => { let alive = true const probeOwner = async (): Promise => alive @@ -442,7 +446,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () stopOwnerProcess }) await host.history({ sessionId: SESSION, direction: 'tail' }) - expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) + expect(await settledTurn()).toEqual(RUNNING_TURN) // The reconcile only parks the live orphan in recovery; recovery's stop is what proves it gone. await host.restoreReadableSessions() @@ -511,32 +515,30 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ]) }) - it('stays running when the open cannot write its settle, and a later open settles it', async () => { - let now = RELAUNCHED_AT + it('stays as it was when the startup settlement cannot be written, and the retry retries it', async () => { const log = recordingStructuredAgentSessionLogger() - const append = AgentSessionJournal.prototype.appendLifecycleBatch - let refused = false - vi.spyOn(AgentSessionJournal.prototype, 'appendLifecycleBatch').mockImplementation(function ( - this: AgentSessionJournal, - input - ) { - if (!refused && input.settlementId.startsWith(STALE_SESSION_ROW_PREFIX)) { - refused = true - return Promise.reject(new Error('disk full')) - } - return append.call(this, input) - }) - openHost({ now: () => now, logger: log.logger }) - + openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }), logger: log.logger }) await host.history({ sessionId: SESSION, direction: 'tail' }) - - expect(log.scopes()).toEqual(['open-dead-generation']) - expect(await settledTurn()).toMatchObject({ turnId: 'turn-1', state: 'running' }) - // Nothing is owed in between: the next open derives the same end again. - now += STRUCTURED_AGENT_SESSION_IDLE_MS + 1 - await host.collaboratorsForTests().lifetime.idleSweep.tick() - expect(host.hasSession(SESSION)).toBe(false) - expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) + const { journal } = host.collaboratorsForTests().sessions.get(SESSION)! + const full = vi + .spyOn(journal, 'appendPlannedLifecycleBatch') + .mockRejectedValue(new Error('disk full')) + + await startUp() + + expect(new Set(log.scopes())).toEqual(new Set(['reconciliation'])) + expect(await settledTurn()).toEqual(RUNNING_TURN) + full.mockRestore() + // Nothing stored says it is owed: the retry re-derives it after its backoff, from the proof + // the record holds. + await vi.waitFor( + async () => + expect(await settledTurn()).toMatchObject({ + state: 'interrupted', + completedAt: LAST_RENEWED_AT + }), + { timeout: 10_000 } + ) }) }) @@ -585,9 +587,24 @@ async function hostWithFailingFirstStart(failure: Error) { } } -// The relaunch proved the fence-13 owner gone, but a start reserving fence 15 clears that proof -// before the turn is settled; what the record holds afterwards is about the start's own child. The -// turn is judged by the runtime this one replaced, at its owner's last renewal, never by that death. +/** The turn as the crashed owner left it, before anything settled it. */ +const RUNNING_TURN = { + turnId: 'turn-1', + state: 'running', + startedAt: TOOL_STARTED_AT - 2_000 +} + +/** The turn the relaunch's proof of the fence-13 owner's death settled at startup. */ +const PROVEN_TURN = { + turnId: 'turn-1', + state: 'interrupted', + startedAt: TOOL_STARTED_AT - 2_000, + completedAt: LAST_RENEWED_AT +} + +// The relaunch proved the fence-13 owner gone, and the host's retry settles the turn by that proof, +// held across the starts that clear it from the record; what the record holds after a start +// reserving fence 15 fails is about the start's own child. describe('a turn a newer start could not settle before it failed', () => { it('is not judged by the death of the child it left for recovery', async () => { const { attach, advance } = await hostWithFailingFirstStart( @@ -608,8 +625,10 @@ describe('a turn a newer start could not settle before it failed', () => { lastProvenAliveAt: RELAUNCHED_AT }) await expect(attach(16)).resolves.toMatchObject({ ok: true }) + await host.startupSettled() + await reconciled() - expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) + expect(await settledTurn()).toEqual(PROVEN_TURN) }) it('is not judged by the watched exit of a start that failed', async () => { @@ -623,8 +642,10 @@ describe('a turn a newer start could not settle before it failed', () => { }) advance(60_000) await expect(attach(16)).resolves.toMatchObject({ ok: true }) + await host.startupSettled() + await reconciled() - // Main ended it at the failed start, an hour after the crash, with the start's reason. - expect(await settledTurn()).toMatchObject(INTERRUPTED_TURN) + // Never ended at the failed start, an hour after the crash, with the start's reason. + expect(await settledTurn()).toEqual(PROVEN_TURN) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts index c3f2a4855986..978cec42ccca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts @@ -381,7 +381,8 @@ it.each(['journal_rows', 'agent_session_operations'])( expect(rig.store.getSessionTabId(SESSION)).toBeNull() expect(rig.store.listOperationRows()).toEqual([]) expect((await readPersistedTestAgentSessionStore(rig.root)).records).toEqual({}) - expect(loadTestJournal(rig.root, SESSION)?.state.submissions.size).toBe(0) + // The chat's first epoch is written with its first row, so a refused create leaves no journal. + expect(loadTestJournal(rig.root, SESSION)).toBeNull() expect(rig.host.hasSession(SESSION)).toBe(false) expect(rig.adapter.acquire).not.toHaveBeenCalled() expect(publishTab).not.toHaveBeenCalled() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts new file mode 100644 index 000000000000..03dd59da8010 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts @@ -0,0 +1,329 @@ +// The host's one projection of current work (`structuredAgentSessionCurrentWork`), end to end +// against the real host, store and SQLite journal: what an ended generation left holds nothing and +// reads as nothing to every reader (the projection, the session list, a chat client, Stop, a prompt +// answer, the queue), whether or not its cleanup or its release landed, while a live owner's work +// holds exactly as before. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { + isActionableStructuredAgentSessionPrompt, + runningStructuredAgentSessionTurnId +} from '../../../shared/structured-agent-session-live-turn' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession, + type StructuredAgentSessionState +} from '../../../shared/structured-agent-session-reducer' +import { + liveTestJournalRows, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + APPROVAL, + currentJournal, + currentWork, + exitWhileSettlementFails, + leaveUnfinishedWork, + REJECT_RECOVERED_ROWS, + sendText, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { retryIdle } from './structured-agent-session-retry.test-fixture' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { scanRecord } from './structured-agent-session-startup-scan.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +const APPROVAL_ID = agentJournalItemKey(APPROVAL) +const FOLDER: AgentSessionExecutionLocation = { + executionHostId: 'local', + wslDistro: null, + workspaceId: 'folder-1', + workspaceKind: 'folder' +} + +/** A chat client and the session list, each fed by the host as a real one is. */ +function watch(current: QueuedMessageTestRig) { + let chat: StructuredAgentSessionState = EMPTY_STRUCTURED_AGENT_SESSION + let listed: AgentSessionStatusSummary | undefined + current.host.subscribe({ + id: 'chat-1', + sessionId: SESSION, + emit: (event) => { + chat = reduceStructuredAgentSession(chat, { type: 'event', event: structuredClone(event) }) + } + }) + current.host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status' && event.session.sessionId === SESSION) { + listed = structuredClone(event.session) + } else if (event.type === 'snapshot') { + listed = event.sessions.find((session) => session.sessionId === SESSION) ?? listed + } + } + }) + return { + chat: () => chat, + listed: () => listed, + /** Whether the chat would let the person answer the prompt. */ + answerable: () => + chat.items.some( + (item) => + item.itemId === APPROVAL_ID && + item.body.kind === 'approval' && + item.body.resolution.state === 'pending' && + isActionableStructuredAgentSessionPrompt(item.itemId, chat.actionablePromptIds) + ) + } +} + +/** The tail page a client hydrates from. */ +async function hydration(current: QueuedMessageTestRig) { + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + if (!page.ok) { + throw new Error('history refused') + } + return page.page +} + +async function deadGenerationWithFailedSettlement(location?: typeof FOLDER) { + rig = await createQueuedMessageTestRig({ + restartable: true, + ...(location ? { location } : {}) + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const view = watch(current) + const exited = await exitWhileSettlementFails(current) + return { current, view, ...exited } +} + +describe("a dead generation's leftovers, its settlement refused and its lease released", () => { + it.each([ + ['a git worktree', undefined], + ['a folder workspace', FOLDER] + ])('in %s, read as nothing by every reader, and hold nothing', async (_name, location) => { + const { current, view } = await deadGenerationWithFailedSettlement(location) + // Still saved as running and pending: only a settlement would end them. + expect(turnState(current)).toBe('running') + const work = currentWork(current) + + expect(work.liveFence).toBeNull() + expect(work.working()).toBe(false) + expect(work.actionablePromptIds()).toEqual([]) + expect(view.listed()?.status).not.toBe('working') + expect(current.host.readStatusSummary(SESSION)?.status).not.toBe('working') + // A client hydrating now, and the one already watching, read the same. + const page = await hydration(current) + // Read interrupted, as its settlement will write it: never running, never finished. + expect(page.latestTurn?.turn.state).toBe('interrupted') + expect(page.actionablePromptIds).toEqual([]) + await vi.waitFor(() => expect(view.chat().actionablePromptIds).toEqual([])) + expect(runningStructuredAgentSessionTurnId(view.chat())).toBeNull() + expect(view.answerable()).toBe(false) + }) + + it('gives Stop nothing to stop: the dead turn is not touched', async () => { + const { current } = await deadGenerationWithFailedSettlement() + + await current.stop() + + expect(current.cancelTurn).not.toHaveBeenCalled() + expect(turnState(current)).toBe('running') + }) +}) + +describe('a close Orca asked for, its settlement refused', () => { + it('tells a watching client the turn and prompt are over, with no row to carry it', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const view = watch(current) + await vi.waitFor(() => expect(view.answerable()).toBe(true)) + const database = openTestJournalHostDatabase(current.root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + const before = currentJournal(current).cursor().sequence + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'requested-close', + reason: 'closed by Orca', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + + expect(currentJournal(current).cursor().sequence).toBe(before) + expect(turnState(current)).toBe('running') + await vi.waitFor(() => expect(view.answerable()).toBe(false)) + expect(runningStructuredAgentSessionTurnId(view.chat())).toBeNull() + expect(view.listed()?.status).not.toBe('working') + }) +}) + +describe('a waiting card when the generation ahead of it ends', () => { + it.each([ + ['an exit of its own, its settlement refused', 'unexpected-exit' as const], + ['a close Orca asked for', 'requested-close' as const] + ])('drains after %s, with no journal write in between', async (_name, cause) => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const queued = await sendText(current, 'behind the turn').result + expect(queued).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + const journal = currentJournal(current) + const [card] = journal.queuedMessages.list() + expect(card?.state).toBe('waiting') + const database = openTestJournalHostDatabase(current.root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + const before = journal.cursor().sequence + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause, + reason: 'ended', + fence: child.fence, + acquisitionGeneration: child.generation + }) + + await vi.waitFor(() => + expect(journal.queuedMessages.get(card?.messageId ?? '')?.state).toBe('dispatched') + ) + // The first row after the end is the drain's own hand-off: nothing was written to wake it. + const first = liveTestJournalRows(database.db, SESSION).find((row) => row.seq > before) + expect(JSON.parse(first?.rowJson ?? '{}')).toMatchObject({ + kind: 'submission', + queuedMessageId: card?.messageId + }) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an observed exit whose release write failed', () => { + it('reads as not Working, and a send is delivered', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const transition = current.store.transitionHandoff.bind(current.store) + let refused = 0 + vi.spyOn(current.store, 'transitionHandoff').mockImplementation((sessionId, change) => + transition(sessionId, (record) => { + const next = change(record) + // The exit's own release, and the retry's first repair of it. + if (refused < 2 && next.lease.claimStatus === 'released') { + refused += 1 + throw new Error('disk full') + } + return next + }) + ) + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + await retryIdle(current.host.collaboratorsForTests().reconciliation, SESSION) + + expect(refused).toBe(2) + // The release never landed: the lease still names the gone child. + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'live', + runtimeFence: child.fence + }) + const work = currentWork(current) + expect(work.liveFence).toBeNull() + expect(work.working()).toBe(false) + expect(work.actionablePromptIds()).toEqual([]) + expect(current.host.readStatusSummary(SESSION)?.status).not.toBe('working') + + expect((await sendText(current, 'go on').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('a live owner', () => { + it('publishes its running turn and its prompt exactly as before', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const view = watch(current) + + const work = currentWork(current) + expect(work.liveFence).toBe(current.store.getRecord(SESSION)?.lease.runtimeFence) + expect(work.activeTurnId()).toBe('unfinished') + expect(work.actionablePromptIds()).toEqual([APPROVAL_ID]) + const page = await hydration(current) + expect(page.latestTurn?.turn).toMatchObject({ turnId: 'unfinished', state: 'running' }) + expect(page.actionablePromptIds).toEqual([APPROVAL_ID]) + await vi.waitFor(() => expect(view.chat().actionablePromptIds).toEqual([APPROVAL_ID])) + expect(runningStructuredAgentSessionTurnId(view.chat())).toBe('unfinished') + expect(view.answerable()).toBe(true) + expect(view.listed()?.status).not.toBe('idle') + }) +}) + +describe('an owner whose adjudication concludes with no row written', () => { + it('changes the published answer when it goes from unverifiable to released', () => { + const journal = { + runningTurn: () => null, + itemFence: () => undefined, + visitItems: () => undefined, + submissions: () => [], + wroteBeforeOpen: () => false + } + const record = scanRecord(SESSION, false) + const unverifiable = structuredAgentSessionCurrentWork(journal, { + record: { lease: { ...record.lease, unreconciled: true } }, + child: null + }) + const released = structuredAgentSessionCurrentWork(journal, { + record: scanRecord(SESSION, true), + child: null + }) + + expect(unverifiable.liveFence).toBe(released.liveFence) + expect(unverifiable.viewKey()).not.toBe(released.viewKey()) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts new file mode 100644 index 000000000000..2cee82ca63f9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -0,0 +1,268 @@ +// The host's one operational projection of a chat's current work: the live generation's running +// turn, the prompts it raised that still wait on the person, the sends it owes, and the queued +// sends this process may hand over. Every host reader asks here; none keeps a rule of its own. +// +// Derived on every read, never stored, from what the host already holds: +// - the lease: the live or reserved generation's fence; none once it is released; +// - this host's own sight of that generation's root exit, which ends it even when the release +// write failed (`lastEndedChild`); +// - the Orca runtimes this one replaced (`AgentSessionRecordStore.replacedRuntime`): a generation +// one of them held lost its pipes with it, so it is over before any startup write says so; +// - the journal's fold, where every item carries the generation that produced it (`ownerFence`). +// So work an ended generation left holds nothing and is not Working whether or not its cleanup +// landed: cleanup is bookkeeping and never gates what the person does next. Loss of contact is +// never an end: an unreconciled or conflicted lease keeps its fence, and its work stays current, unless +// a runtime this one replaced held it, which is this host's own proof. + +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem, + type AgentJournalSubmission, + type AgentJournalTurnScope +} from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionLatestTurn } from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' +import { isUnansweredStructuredAgentSessionDispatch } from '../../../shared/structured-agent-session-unanswered-dispatch' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' + +export type StructuredAgentSessionCurrentWorkJournal = Pick< + AgentSessionJournal, + 'runningTurn' | 'itemFence' | 'visitItems' | 'submissions' | 'wroteBeforeOpen' +> + +/** What the host knows of who runs the chat now. */ +export type StructuredAgentSessionWorkEvidence = { + record: Pick | null + /** The last child this host saw end (`StructuredAgentSessionHostSession.lastEndedChild`). */ + ended?: StructuredAgentSessionSeenEnd + /** `StructuredAgentSessionHostSession.operationalRevision`. */ + revision?: number + /** The runtimes this one replaced for the chat (`AgentSessionRecordStore.replacedRuntime`). */ + replaced?: StructuredAgentSessionReplacedEnd | undefined + /** This host's live child (`StructuredAgentSessionHostSession.child`); absent: not known. */ + child?: { fence: number } | null +} + +/** Every fence at or below `fence` was granted by a runtime this one replaced. */ +type StructuredAgentSessionReplacedEnd = { fence: number } + +type StructuredAgentSessionSeenEnd = { fence: number; rootGone?: boolean; cause?: string } + +/** The live or reserved generation's fence, or null when no generation is live. With no record + * nothing has ended: fence 0, under which every item is current, as before records scoped work. */ +export function structuredAgentSessionLiveFence( + evidence: StructuredAgentSessionWorkEvidence +): number | null { + const lease = evidence.record?.lease + if (!lease) { + return 0 + } + if (lease.claimStatus === 'released') { + return null + } + if ( + structuredAgentSessionReplacedRuntimeEnded(lease, evidence.replaced) || + structuredAgentSessionOwnerUnverifiable(evidence) + ) { + return null + } + const { ended } = evidence + return ended?.rootGone === true && ended.fence === lease.runtimeFence ? null : lease.runtimeFence +} + +/** An owner not yet reconciled, or latched in recovery, that this process holds no child for: it + * may still be running, so its turn is unverifiable, and nothing it left is current work. */ +function structuredAgentSessionOwnerUnverifiable(evidence: StructuredAgentSessionWorkEvidence) { + const lease = evidence.record?.lease + return ( + (lease?.unreconciled === true || lease?.handoffStage === 'recovering') && + evidence.child !== undefined && + evidence.child?.fence !== lease.runtimeFence + ) +} + +/** Whether a runtime this one replaced granted the lease's generation, which is then over. A + * conflicted claim's agent is a terminal's, which no runtime's pipes held. */ +export function structuredAgentSessionReplacedRuntimeEnded( + lease: Pick, + replaced: StructuredAgentSessionReplacedEnd | undefined +): boolean { + return ( + replaced !== undefined && + lease.runtimeFence <= replaced.fence && + lease.claimStatus !== 'conflicted' + ) +} + +/** `StructuredAgentSessionCurrentWork.handsOver`, for a reader that needs no other answer. */ +export function structuredAgentSessionHandsOver( + journal: Pick, + submission: Pick< + AgentJournalSubmission, + 'handoverRecorded' | 'dispatchState' | 'handedOverAt' | 'acceptedSequence' + > +): boolean { + return ( + isQueuedAgentJournalSubmission(submission) && + !journal.wroteBeforeOpen(submission.acceptedSequence) + ) +} + +/** One read of a chat's current work; build a fresh one per decision, since nothing is cached. */ +export class StructuredAgentSessionCurrentWork { + /** The shared projections' form of this answer (`StructuredAgentSessionWorkScope`). */ + readonly scope: StructuredAgentSessionWorkScope = { + isCurrentItem: (itemId) => this.isCurrentItem(itemId), + owesSend: (submission) => this.owesSend(submission) + } + + constructor( + private readonly journal: StructuredAgentSessionCurrentWorkJournal, + /** `structuredAgentSessionLiveFence`. */ + readonly liveFence: number | null, + /** Moves with every generation end this host saw, lease write or not: a reader that caches + * what it derived from this answer keys it here and on the journal's cursor. */ + readonly revision = 0, + private readonly ended?: StructuredAgentSessionSeenEnd, + private readonly replaced?: StructuredAgentSessionReplacedEnd, + /** `structuredAgentSessionOwnerUnverifiable`. */ + private readonly ownerUnverifiable = false + ) {} + + /** What a reader that published this answer compares to learn it changed with no row: the live + * generation, since every answer here is the journal read through it. Not the revision, which + * moves on every generation-end signal: an end that changed nothing is published once. */ + viewKey(): string { + return `${this.liveFence}${this.ownerUnverifiable ? ':unverifiable' : ''}` + } + + /** Whether the live generation's execution produced this item. */ + isCurrentItem(itemId: string): boolean { + if (this.liveFence === null) { + return false + } + const provenance = this.journal.itemFence(itemId) + return provenance === undefined || provenance >= this.liveFence + } + + /** Whether this process may hand this send over: accepted here, never by an earlier host process. + * Derived from where the row sits, so nothing has to be written first; keeping an earlier + * process's send as a card is the reconciliation pass's bookkeeping (`holdUnsentSends`). */ + handsOver(submission: AgentJournalSubmission): boolean { + return structuredAgentSessionHandsOver(this.journal, submission) + } + + /** A send still owed: one this process may hand over, or one handed to the live generation that + * it has neither answered nor refused (`isUnansweredStructuredAgentSessionDispatch`). */ + owesSend(submission: AgentJournalSubmission): boolean { + if (isQueuedAgentJournalSubmission(submission)) { + return this.handsOver(submission) + } + return ( + this.liveFence !== null && + isUnansweredStructuredAgentSessionDispatch(submission, this.liveFence) + ) + } + + /** The live generation's running turn, with its record's item. */ + activeTurn(): { item: AgentJournalRenderItem; turnId: string } | null { + const running = this.journal.runningTurn() + return running && this.isCurrentItem(running.item.itemId) ? running : null + } + + activeTurnId(): string | null { + return this.activeTurn()?.turnId ?? null + } + + /** Where a row written now joins: the live generation's running turn, else the conversation. */ + turnScope(): AgentJournalTurnScope { + const running = this.activeTurn() + return running ? { kind: 'turn', turnItemId: running.item.itemId } : AGENT_JOURNAL_THREAD_SCOPE + } + + /** The approvals and questions the live generation raised that wait on the person, a + * subagent's included. */ + actionablePromptIds(): string[] { + const ids: string[] = [] + this.journal.visitItems((itemId, _sequence, body) => { + if ( + (body.kind === 'approval' || body.kind === 'question') && + body.resolution.state === 'pending' && + this.isCurrentItem(itemId) + ) { + ids.push(itemId) + } + }) + return ids + } + + hasActionablePrompt(): boolean { + return this.actionablePromptIds().length > 0 + } + + owesAnySend(): boolean { + return this.journal.submissions().some((submission) => this.owesSend(submission)) + } + + /** The session's own agent is working: its running turn, or a send it owes. */ + working(): boolean { + return this.activeTurnId() !== null || this.owesAnySend() + } + + /** The running turn of a generation this host saw die on its own (its root gone, nobody asked), + * or that a runtime this one replaced held: its settlement writes it interrupted, so it reads so + * before that lands. */ + diedTurn(): { item: AgentJournalRenderItem; turnId: string } | null { + const running = this.journal.runningTurn() + if (!running || this.isCurrentItem(running.item.itemId)) { + return null + } + const fence = this.journal.itemFence(running.item.itemId) + const { ended, replaced } = this + const died = ended?.rootGone === true && ended.cause === 'exit' && fence === ended.fence + return died || (replaced !== undefined && fence !== undefined && fence <= replaced.fence) + ? running + : null + } + + /** The newest turn record as a client is told it: a running one an ended generation opened is + * none, interrupted when it died on its own (`diedTurn`), or unverifiable while its owner is, so + * no client reads it as working, steers into it or reads it finished. */ + publishedLatestTurn(latest: AgentSessionLatestTurn | null): AgentSessionLatestTurn | null { + if (latest?.turn.state !== 'running' || this.isCurrentItem(latest.itemId)) { + return latest + } + if (this.diedTurn()?.item.itemId === latest.itemId) { + return { ...latest, turn: { ...latest.turn, state: 'interrupted' } } + } + return this.ownerUnverifiable + ? { ...latest, turn: { ...latest.turn, state: 'unverifiable' } } + : null + } +} + +export function structuredAgentSessionCurrentWork( + journal: StructuredAgentSessionCurrentWorkJournal, + evidence: StructuredAgentSessionWorkEvidence +): StructuredAgentSessionCurrentWork { + return new StructuredAgentSessionCurrentWork( + journal, + structuredAgentSessionLiveFence(evidence), + evidence.revision, + evidence.ended, + evidence.replaced, + structuredAgentSessionOwnerUnverifiable(evidence) + ) +} + +/** An operation context's current work: the host projection it was built with, or, for one built + * without (a test's), the generation at its fence read as live. */ +export function contextStructuredAgentSessionCurrentWork(ctx: { + journal: StructuredAgentSessionCurrentWorkJournal + fence: number + currentWork?: () => StructuredAgentSessionCurrentWork | null +}): StructuredAgentSessionCurrentWork { + return ctx.currentWork?.() ?? new StructuredAgentSessionCurrentWork(ctx.journal, ctx.fence) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index 07510c754e1c..21d59549faa0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -4,14 +4,13 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import { agentSessionFailureFact, MAX_PROVIDER_DIAGNOSTIC_CHARS, providerDiagnostic } from '../../../shared/agent-session-failure' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import { captureUnfinishedStructuredAgentSessionWork, unfinishedStructuredAgentSessionWorkWasInterrupted @@ -93,7 +92,7 @@ describe('dead structured-session generation settlement', () => { await seedUnfinishedWork() await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, @@ -102,7 +101,7 @@ describe('dead structured-session generation settlement', () => { verdict: { state: 'unverifiable' }, showUnexpectedExitOutcome: false }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) const snapshot = journal.snapshot() expect(snapshot.submissions).toEqual([ @@ -141,9 +140,9 @@ describe('dead structured-session generation settlement', () => { showUnexpectedExitOutcome: true } - await expect(settleStructuredAgentSessionDeadGeneration(input)).resolves.toEqual({ ok: true }) + await expect(settleObservedExitForTest(input)).resolves.toMatchObject({ ok: true }) const settledCursor = journal.cursor() - await expect(settleStructuredAgentSessionDeadGeneration(input)).resolves.toEqual({ ok: true }) + await expect(settleObservedExitForTest(input)).resolves.toMatchObject({ ok: true }) expect(journal.cursor()).toEqual(settledCursor) expect( @@ -158,7 +157,7 @@ describe('dead structured-session generation settlement', () => { it('cuts short a call the proven death interrupted', async () => { await seedUnfinishedWork() - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -193,7 +192,7 @@ describe('dead structured-session generation settlement', () => { verdict: { state: 'interrupted' as const, completedAt: 1_000 }, showUnexpectedExitOutcome: true } - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) const settled = journal.snapshot() expect(settled.items).toContainEqual(terminal) expect(settled.items.map((item) => item.body)).toContainEqual({ @@ -203,7 +202,7 @@ describe('dead structured-session generation settlement', () => { state: 'failed', endedAs: 'interrupted' }) - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) expect(journal.snapshot()).toEqual(settled) }) @@ -211,7 +210,7 @@ describe('dead structured-session generation settlement', () => { await seedUnfinishedWork() await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -223,7 +222,7 @@ describe('dead structured-session generation settlement', () => { detail: providerDiagnostic('stack frame '.repeat(4_000), 'log') }) }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) const statuses = journal .snapshot() @@ -238,7 +237,7 @@ describe('dead structured-session generation settlement', () => { it("words Orca's own fault as Orca's, never as the provider stopping", async () => { await seedUnfinishedWork() - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -263,51 +262,6 @@ describe('dead structured-session generation settlement', () => { ]) }) - it('retries an already settled expected close without writing through a closed journal gate', async () => { - const settledItem: AgentJournalRenderItem = { - itemId: 'codex:thread-1:turn-1:0', - revision: 2, - sequence: 2, - observedAt: 1_000, - body: { - kind: 'turn', - turnId: 'turn-1', - state: 'interrupted', - completedAt: 1_000 - } - } - const settledSnapshot = journal.snapshot() - const closedJournal: Pick< - AgentSessionJournal, - 'snapshot' | 'submissions' | 'rejectQueuedSubmissions' | 'appendLifecycleBatch' | 'itemFence' - > = { - itemFence: () => undefined, - snapshot: () => ({ - ...settledSnapshot, - items: [settledItem] - }), - submissions: () => [], - rejectQueuedSubmissions: async () => { - throw new Error('journal_closed') - }, - appendLifecycleBatch: async () => { - throw new Error('journal_closed') - } - } - - await expect( - settleStructuredAgentSessionDeadGeneration({ - journal: closedJournal, - sessionId: SESSION, - fence: 7, - settlementId: `expected-close:${SESSION}:7:generation-1`, - pendingSubmissionReason: 'provider_closed_before_acknowledgement', - verdict: { state: 'interrupted', completedAt: 1_000 }, - showUnexpectedExitOutcome: false - }) - ).resolves.toEqual({ ok: true }) - }) - it('settles a live unknown submission even when no unfinished item remains', async () => { await journal.appendSubmission({ clientMessageId: 'client-unknown', @@ -323,7 +277,7 @@ describe('dead structured-session generation settlement', () => { }) await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -332,7 +286,7 @@ describe('dead structured-session generation settlement', () => { verdict: { state: 'interrupted', completedAt: 1_000 }, showUnexpectedExitOutcome: false }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) expect(journal.submissions()).toEqual([ expect.objectContaining({ @@ -352,7 +306,7 @@ describe('dead structured-session generation settlement', () => { fence: 7 }) - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -413,7 +367,7 @@ describe('dead structured-session generation settlement', () => { { fence: 7, ...child, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, @@ -497,7 +451,7 @@ describe('whether a dead generation interrupted anything', () => { await seedIdlePendingApproval() await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -510,7 +464,7 @@ describe('whether a dead generation interrupted anything', () => { 1_000 ) }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) const snapshot = journal.snapshot() expect(snapshot.items.some((item) => item.body.kind === 'status')).toBe(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 0a32aa14d6fd..41890413e13c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -3,7 +3,6 @@ import { MAX_PROVIDER_DIAGNOSTIC_CHARS, type SubmissionRejectionFact } from '../../../shared/agent-session-failure' -import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { AGENT_JOURNAL_THREAD_SCOPE, @@ -20,7 +19,8 @@ import { runningCallEnd, terminalAgentJournalBody } from '../agent-session-journal/journal-terminal-settlement' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { ResolveDispatchInput } from '../agent-session-journal/journal-store-contracts' +import { lostLiveWorkJournalBody } from '../agent-session-journal/journal-subagent-liveness' import { agentSessionFailureWords, type AgentSessionFailureWordsContext @@ -30,48 +30,29 @@ import { hasStructuredAgentSessionStartFailureRow, structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' -import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { - endedByPersonsStop, - provenUnverifiableTurnRevisions, - provenUnverifiedToolCallRevisions, runningTurnLifecycleRevisions, - lastProvenTurnLiveAt, - turnVerdictFromDeathEvidence, - turnVerdictFromReplacedRuntime, watchedExitRevisions, type StructuredAgentSessionTurnVerdict, type StructuredAgentSessionWatchedExit } from './structured-agent-session-stale-turn-verdict' -import { - exitedRootTurnScope, - settledRootTurnScope -} from './structured-agent-session-exit-turn-scope' +import { exitedRootTurnScope } from './structured-agent-session-exit-turn-scope' import { hasUnfinishedStructuredAgentSessionWork, - isInProgressStructuredAgentSessionItem, type DeadGenerationJournal } from './structured-agent-session-unfinished-work' import { codexUnopenedSendResolutions } from './structured-agent-session-unopened-send-withdrawal' -import type { AgentSessionReplacedRuntime } from '../../runtime/agent-session-replaced-runtime' -import { - isUnverifiableTurn, - staleStopRow, - type StaleCut -} from './structured-agent-session-stale-stop-row' + +export { settleStaleStructuredAgentSessionState } from './structured-agent-session-stale-state-settlement' /** Bounds the exit reason the lease keeps as log evidence; a provider diagnostic is held to the * same cap. */ export const MAX_UNEXPECTED_EXIT_REASON_CHARS = MAX_PROVIDER_DIAGNOSTIC_CHARS -/** Whether the settlement was written, and what stopped it when it was not. */ -export type StructuredAgentSessionDeadGenerationSettlement = - | { ok: true } - | { ok: false; error: unknown } - -export async function settleStructuredAgentSessionDeadGeneration(input: { +export type StructuredAgentSessionDeadGenerationInput = { journal: DeadGenerationJournal sessionId: string + /** The fence the rows are written at: the lease's current one once the exit released it. */ fence: number settlementId: string verdict: StructuredAgentSessionTurnVerdict @@ -89,146 +70,83 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { exit?: StructuredAgentSessionWatchedExit /** A person's Stop ended a starting child before what it was handed could run: each is rejected so. */ unrunRejection?: SubmissionRejectionFact -}): Promise { - try { - const hasUnfinishedWork = hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) - const showUnexpectedExitOutcome = input.showUnexpectedExitOutcome ?? hasUnfinishedWork - if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { - return { ok: true } - } - // A queued message is the delivery loop's to settle: it was never handed to this child. A send - // a child still starting was handed and never echoed did not run, and its root is gone: it is - // rejected, with the child's own diagnostic or as the Stop that ended it. A proven child's - // handed-over sends stay in doubt. - const startupFailure = input.exitedDuringStartup - ? structuredAgentSessionStartFailure({ exit: input.exitFailure }, input.failureTextContext) - : null - const closed = input.unrunRejection - const unrun = - startupFailure ?? (closed && agentSessionFailureWords(closed, { surface: 'rejection' })) - const withdrawn = unrun ? [] : codexUnopenedSendResolutions(input.journal, input.fence) - const withdrawnIds = new Set(withdrawn.map((entry) => entry.clientMessageId)) - const dispatches = [ - ...withdrawn, - ...journalPendingSubmissionResolutions( - (input.journal.submissions?.() ?? input.journal.pendingSubmissions?.() ?? []).filter( - (entry) => !withdrawnIds.has(entry.clientMessageId) - ), - input.fence, - unrun ? { rejection: unrun } : { reason: input.pendingSubmissionReason } - ) - ] - const items = input.journal.snapshot().items - const proven = watchedExitRevisions(items, input.exit, input.journal) - const mutations: JournalLifecycleMutationInput[] = [] - if (showUnexpectedExitOutcome && input.exitedDuringStartup && startupFailure) { - const startKey = input.exitedDuringStartup.generation ?? input.settlementId - // A start a message waited on is the delivery loop's to record, before or after this exit, - // in the words it rejected the message with; this row is for a command, goal or rewind start. - // A row already written stays: rejected is terminal, so its words are not reworded. - const recordedByDeliveryLoop = - input.journal.submissions?.().some(isQueuedAgentJournalSubmission) || - hasStructuredAgentSessionStartFailureRow(items, startKey) - if (!recordedByDeliveryLoop) { - mutations.push(structuredAgentSessionStartFailureRow(startKey, startupFailure)) - } - } else if (showUnexpectedExitOutcome) { - // The turn the exit ended, and an error so no fold ever hides why it stopped. - mutations.push({ - kind: 'item', - identity: { provider: 'orca', clientMessageId: input.settlementId }, - body: { - kind: 'status', - ...agentSessionFailureWords( - input.exitFailure ?? agentSessionFailureFact('providerExited'), - { - ...input.failureTextContext, - surface: 'row' - } - ), - tone: 'error' - }, - turnScope: exitedRootTurnScope(withRevisions(items, proven), input.verdict) - }) - } - const bodies = new Map(items.map((item) => [item.itemId, item.body])) - for (const item of items) { - // Ended as its turn is: a proven death cuts a running call short. An open reasoning row is - // ended too, but is not unfinished work: its running turn already says so. - const end = runningCallEnd(item.turnScope, (id) => bodies.get(id), input.verdict.state) - const body = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) - if (body) { - mutations.push({ - kind: 'item', - itemId: item.itemId, - body, - turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE - }) - } - } - mutations.push(...runningTurnLifecycleRevisions(items, input.verdict), ...proven) - await input.journal.appendLifecycleBatch({ - settlementId: `dead-generation:${input.settlementId}`, - fence: input.fence, - recovered: true, - mutations, - dispatches - }) - return { ok: true } - } catch (error) { - // Returned rather than logged: each caller logs it under its own scope. - return { ok: false, error } - } } -/** - * Settles whatever a generation with no child in this process left running: found when a new child - * is acquired, or when a chat is reopened for reading. Derived from the journal, the lease's death - * evidence and the runtimes this one replaced each time, so nothing is owed in between. Either proof - * ends the turn interrupted, and a proof written after an earlier settle revises what that settle - * left `unverifiable`, the turn and the calls it closed alike. Must run before a new child's - * buffered events land, or a live turn would be judged. - */ -export async function settleStaleStructuredAgentSessionState(input: { - journal: AgentSessionJournal - sessionId: string - fence: number - acquisitionGeneration: string | null - deathEvidence: AgentSessionDeathEvidence | null - /** The runtimes this one replaced for this chat: every owner they held lost its pipes with them. */ - replaced?: AgentSessionReplacedRuntime - /** Who the exit row names. */ - failureTextContext?: AgentSessionFailureWordsContext -}): Promise { - const { journal, replaced } = input - const items = journal.snapshot().items - // Each turn is judged by the evidence only if it names that turn's owner. - const deathVerdictFor = (item: AgentJournalRenderItem) => - turnVerdictFromDeathEvidence( - input.deathEvidence, - journal.itemFence(item.itemId), - lastProvenTurnLiveAt(journal, item) - ) - const replacedVerdictFor = (item: AgentJournalRenderItem) => - turnVerdictFromReplacedRuntime( - replaced, - journal.itemFence(item.itemId), - lastProvenTurnLiveAt(journal, item) +/** The exit's rows, read from the journal as it stands. `owns`: which fences are the exited + * generation's (and later); the rest are an earlier generation's, settled by its own rule. */ +export function planStructuredAgentSessionDeadGeneration( + input: StructuredAgentSessionDeadGenerationInput, + owns: (fence: number | undefined) => boolean = () => true +): { mutations: JournalLifecycleMutationInput[]; dispatches: ResolveDispatchInput[] } { + const hasUnfinishedWork = hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) + const showUnexpectedExitOutcome = input.showUnexpectedExitOutcome ?? hasUnfinishedWork + if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { + return { mutations: [], dispatches: [] } + } + // A queued message is the delivery loop's to settle: it was never handed to this child. A send + // a child still starting was handed and never echoed did not run, and its root is gone: it is + // rejected, with the child's own diagnostic or as the Stop that ended it. A proven child's + // handed-over sends stay in doubt. + const startupFailure = input.exitedDuringStartup + ? structuredAgentSessionStartFailure({ exit: input.exitFailure }, input.failureTextContext) + : null + const closed = input.unrunRejection + const unrun = + startupFailure ?? (closed && agentSessionFailureWords(closed, { surface: 'rejection' })) + const withdrawn = unrun ? [] : codexUnopenedSendResolutions(input.journal, input.fence) + const withdrawnIds = new Set(withdrawn.map((entry) => entry.clientMessageId)) + const dispatches = [ + ...withdrawn, + ...journalPendingSubmissionResolutions( + (input.journal.submissions?.() ?? input.journal.pendingSubmissions?.() ?? []).filter( + (entry) => !withdrawnIds.has(entry.clientMessageId) && owns(entry.fence) + ), + input.fence, + unrun ? { rejection: unrun } : { reason: input.pendingSubmissionReason } ) - const verdictFor = (item: AgentJournalRenderItem) => { - const byDeath = deathVerdictFor(item) - return byDeath.state === 'interrupted' ? byDeath : replacedVerdictFor(item) + ] + const allItems = input.journal.snapshot().items + const items = allItems.filter((item) => owns(input.journal.itemFence(item.itemId))) + const proven = watchedExitRevisions(items, input.exit, input.journal) + const mutations: JournalLifecycleMutationInput[] = [] + if (showUnexpectedExitOutcome && input.exitedDuringStartup && startupFailure) { + const startKey = input.exitedDuringStartup.generation ?? input.settlementId + // A start a message waited on is the delivery loop's to record, before or after this exit, + // in the words it rejected the message with; this row is for a command, goal or rewind start. + // A row already written stays: rejected is terminal, so its words are not reworded. + const recordedByDeliveryLoop = + input.journal.submissions?.().some(isQueuedAgentJournalSubmission) || + hasStructuredAgentSessionStartFailureRow(allItems, startKey) + if (!recordedByDeliveryLoop) { + mutations.push(structuredAgentSessionStartFailureRow(startKey, startupFailure)) + } + } else if (showUnexpectedExitOutcome) { + // The turn the exit ended, and an error so no fold ever hides why it stopped. + mutations.push({ + kind: 'item', + identity: { provider: 'orca', clientMessageId: input.settlementId }, + body: { + kind: 'status', + ...agentSessionFailureWords( + input.exitFailure ?? agentSessionFailureFact('providerExited'), + { + ...input.failureTextContext, + surface: 'row' + } + ), + tone: 'error' + }, + turnScope: exitedRootTurnScope(withRevisions(items, proven), input.verdict) + }) } - // A successful settlement leaves terminal items; a failed transaction leaves the same prefix. - const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` - const settlementId = `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}` - // Calls an earlier settle closed with no proof, revised once a proof names their owner. - const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal, replaced) + const bodies = new Map(allItems.map((item) => [item.itemId, item.body])) for (const item of items) { - // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running - // leaves them to the evidence. - const end = runningCallEnd(item.turnScope, (id) => journal.itemBody(id), verdictFor(item).state) - const body = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) + // Ended as its turn is: a proven death cuts a running call short. An open reasoning row is + // ended too, but is not unfinished work: its running turn already says so. + const end = runningCallEnd(item.turnScope, (id) => bodies.get(id), input.verdict.state) + const terminal = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) + // One revision per item: its live subagents and background tasks end with it. + const body = lostLiveWorkJournalBody(terminal ?? item.body) ?? terminal if (body) { mutations.push({ kind: 'item', @@ -238,42 +156,8 @@ export async function settleStaleStructuredAgentSessionState(input: { }) } } - const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal, replaced) - const turnEnds = [ - ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), - ...proven - ] - mutations.push(...turnEnds) - const stopRow = staleStopRow(input, items, (item): StaleCut => { - const unverifiable = isUnverifiableTurn(item) - if (!unverifiable && !isInProgressStructuredAgentSessionItem(item)) { - return null - } - const fence = journal.itemFence(item.itemId) - // An earlier settle's `unverifiable` turn is revised by a death naming its owner or by its replaced runtime. - if ( - unverifiable - ? fence !== undefined && fence === input.deathEvidence?.ownerFence - : deathVerdictFor(item).state === 'interrupted' - ) { - return { by: 'death' } - } - return fence !== undefined && replacedVerdictFor(item).state === 'interrupted' - ? { by: 'replaced', fence } - : null - }) - if (stopRow && !endedByPersonsStop(journal, turnEnds)) { - mutations.unshift({ ...stopRow, turnScope: settledRootTurnScope(items, turnEnds) }) - } - if (mutations.length > 0) { - await journal.appendLifecycleBatch({ - settlementId, - fence: input.fence, - recovered: true, - mutations - }) - } - return mutations.length + mutations.push(...runningTurnLifecycleRevisions(items, input.verdict), ...proven) + return { mutations, dispatches } } function withRevisions( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts new file mode 100644 index 000000000000..1e02f9b1ff25 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts @@ -0,0 +1,106 @@ +// An agent that dies while another connection holds the database: its exit's settlement fails at +// once and waits for the retry, so the journal still says the turn runs. Until the settlement +// lands, every frame a reader gets reads that turn as the settlement will write it, interrupted: +// the chat's latest turn, the session list's row (which the tab and sidebar show), and the host's +// working answer that decides a queued card's Send. Never Working, never a finish with no verdict. + +import { afterEach, expect, it, vi } from 'vitest' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { runningStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-live-turn' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession, + type StructuredAgentSessionState +} from '../../../shared/structured-agent-session-reducer' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + exitChild, + leaveUnfinishedWork, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + holdWriteLock, + releaseWriteLocks +} from './structured-agent-session-write-lock.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await releaseWriteLocks() + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Every frame a chat client and the session list receive, as each reduces it. */ +function recordFrames(current: QueuedMessageTestRig) { + const chats: StructuredAgentSessionState[] = [] + const rows: AgentSessionStatusSummary[] = [] + let chat = EMPTY_STRUCTURED_AGENT_SESSION + current.host.subscribe({ + id: 'chat-1', + sessionId: SESSION, + emit: (event) => { + chat = reduceStructuredAgentSession(chat, { type: 'event', event: structuredClone(event) }) + chats.push(chat) + } + }) + current.host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + const row = + event.type === 'status' + ? event.session + : event.type === 'snapshot' + ? event.sessions.find((session) => session.sessionId === SESSION) + : undefined + if (row?.sessionId === SESSION) { + rows.push(structuredClone(row)) + } + } + }) + return { chats, rows } +} + +it('reads the dead turn interrupted in every frame until its settlement lands', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await current.send('after this', 'queue-if-active').result + await leaveUnfinishedWork(current) + const frames = recordFrames(current) + await vi.waitFor(() => expect(frames.chats.at(-1)?.working).toBe(true)) + const { released } = await holdWriteLock(current.root, 1_200) + const before = { chats: frames.chats.length, rows: frames.rows.length } + + await exitChild(current) + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + await released + // Nothing landed: what the frames read is derived from the death alone. + expect(turnState(current)).toBe('running') + + const chats = frames.chats.slice(before.chats) + const rows = frames.rows.slice(before.rows) + expect(chats.length).toBeGreaterThan(0) + expect(rows.length).toBeGreaterThan(0) + for (const chat of [...chats, page.ok ? page.page : null]) { + expect(chat?.latestTurn?.turn.state).toBe('interrupted') + expect(chat?.working).toBe(false) + // So no turn runs to steer into: the card's control is Send. + expect(chat?.nextQueuedMessageId ?? null).toBeNull() + } + for (const chat of chats) { + expect(runningStructuredAgentSessionTurnId(chat)).toBeNull() + } + for (const row of rows) { + expect(row).toMatchObject({ status: 'idle', turnOutcome: 'interruption' }) + } + + // Once the lock lifts, the retry writes what the frames already said. + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 8_000 }) +}, 20_000) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts index 1858eaeddf8d..25307a2e5d31 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -9,8 +9,10 @@ // that arrives while a start holds the queue withdraw the queued messages before the handover that // would have written them. Stop and the conversation's close are the only other writers of a // queued message: a child's exit only ends the child, and this loop reads why. A message an -// earlier host process left queued is never handed over: the open, or this loop's first step, -// settles it first (`journal-unsent-send-hold.ts`). +// earlier host process left queued is never handed over: the loop hands over only what this +// process accepted (`structuredAgentSessionHandsOver`), read off the journal, so nothing has to be +// written first. Keeping that message as a card (`journal-unsent-send-hold.ts`) is the startup +// share's bookkeeping, and its failure holds up no one. import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -37,7 +39,7 @@ import { handOverSubmission } from './structured-agent-session-turns' import { structuredAgentSessionNextHandover } from './structured-agent-session-opening-send' import { structuredAgentSessionCommandRunning } from './structured-agent-session-command-turn' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import { holdRestartedStructuredAgentSessionSends } from './structured-agent-session-host-lifetime' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { structuredAgentSessionEndedChildFailure } from './structured-agent-session-ended-child-failure' export type StructuredAgentSessionDeliveryLoopDeps = { @@ -64,6 +66,8 @@ export type StructuredAgentSessionDeliveryLoopDeps = { failureTextContext: (sessionId: string) => AgentSessionFailureWordsContext logger: StructuredAgentSessionLogger record: (sessionId: string) => AgentSessionRecord | null + /** The host's projection of the chat's current work (`hostStructuredAgentSessionCurrentWork`). */ + currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null readChildWork: (sessionId: string) => readonly AgentChildWorkView[] | undefined /** A person's Stop is still ending the session's work: the status feed's own reading. */ stopping: (sessionId: string) => boolean @@ -167,21 +171,12 @@ export class StructuredAgentSessionDeliveryLoop { } } - /** Settles what an earlier host process left queued, then makes the session ready. */ + /** Makes the session ready for the oldest send this process may hand over. */ private async prepare(sessionId: string): Promise { const session = this.deps.sessions.get(sessionId) if (!session || this.disposed) { return this.stop(sessionId) } - // The open already did, unless its write failed: a row an earlier handle wrote is never handed - // over, whether it outlived a quit or a crash. A failure here throws, so none is: this run then - // fails, which rejects every queued send, this process's own too. - await holdRestartedStructuredAgentSessionSends( - this.deps.logger, - sessionId, - session.journal, - this.deps.conversationFence(sessionId) - ) if (!(await this.closeWhatTheUserClosed(sessionId, session))) { // Never start an agent for a message the user closed; the next wake re-derives and retries. return this.stop(sessionId) @@ -189,7 +184,8 @@ export class StructuredAgentSessionDeliveryLoop { const oldest = oldestQueuedSubmission(session) // A running command takes no input while its child carries it; its end is a commit, which // wakes the loop again. With no child it is a gone generation's, which the start below settles. - if (!oldest || (session.child && structuredAgentSessionCommandRunning(session.journal))) { + const work = this.deps.currentWork(sessionId) + if (!oldest || (session.child && work && structuredAgentSessionCommandRunning(work))) { return this.stop(sessionId) } const failedStart = startThatFailedWhileQueued(session, oldest) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts new file mode 100644 index 000000000000..351d48db55d8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts @@ -0,0 +1,226 @@ +// A Cancel that names a card an ended generation raised, from a client that still shows it. As on +// main: the card is dismissed in the journal alone, and a provider whose card Cancel routes +// (Claude, Pi) stops nothing. One with no route gets the turn's own cancel with the card, never the +// chat's Stop, so queued messages stay and no Stop event pauses the queue: ACP interrupts the live +// turn the request names; Codex finds no such card in its live child and interrupts nothing. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { claudePromptCancelRoute } from '../../claude/claude-structured-prompt-replies' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import type { StructuredAgentSessionChatStopRun } from './structured-agent-session-chat-stop' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { cancelStructuredAgentSessionPrompt } from './structured-agent-session-prompt-cancel' +import { performCancel, type AgentSessionTurnContext } from './structured-agent-session-turns' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: codexProviderHandle('thread-1') +} +const ENDED = 1 +const LIVE = 2 + +const journals = createTrackedJournalOpener() +let root: string | null = null + +afterEach(async () => { + await journals.closeAll() + if (root) { + await rm(root, { recursive: true, force: true }) + root = null + } +}) + +/** Generation 1 raised a card and ended without its cleanup; generation 2 runs `turn-2`, with a + * message queued behind it. */ +async function endedCardAndLiveTurn( + kind: 'approval' | 'question' = 'approval' +): Promise<{ journal: AgentSessionJournal; itemId: string }> { + root = await mkdtemp(join(tmpdir(), 'orca-ended-prompt-cancel-')) + const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) + const turn = (turnId: string, fence: number) => + journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId, ordinal: 0 }, + { kind: 'turn', turnId, state: 'running', startedAt: fence }, + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + const old = await turn('turn-1', ENDED) + const pending = { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null + } + const card = await journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 1 }, + kind === 'approval' + ? { + kind: 'approval', + title: 'Approve?', + detail: null, + options: [{ id: 'allow', label: 'Allow' }], + resolution: pending + } + : { + kind: 'question', + question: 'Which?', + options: [{ id: 'a', label: 'A' }], + resolution: pending + }, + { fence: ENDED, turnScope: { kind: 'turn', turnItemId: old.itemId } } + ) + await turn('turn-2', LIVE) + await journal.queuedMessages.insert({ + messageId: 'queued-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, + fingerprint: 'queued-1', + hostInstance: 'host-1' + }) + return { journal, itemId: card.itemId } +} + +/** The queue as the live turn left it: its message waiting, and nothing pausing it. */ +function queueUntouched(journal: AgentSessionJournal): void { + expect(journal.queuedMessages.list().map((card) => [card.messageId, card.state])).toEqual([ + ['queued-1', 'waiting'] + ]) + expect(structuredQueuePauses(journal)).toEqual([]) +} + +function provider(routePromptCancel?: StructuredAgentSessionAdapter['routePromptCancel']) { + return { + cancelTurn: vi.fn(async (_input: { turnId?: string; prompt?: unknown }) => ({ + cancelled: true + })), + dismissPrompt: vi.fn(async () => undefined), + ...(routePromptCancel ? { routePromptCancel } : {}) + } +} + +function context( + journal: AgentSessionJournal, + adapter: ReturnType +): AgentSessionTurnContext { + return { + logger: createStructuredAgentSessionLogger(), + sessionId: 'session-1', + journal, + fence: LIVE, + agents: NO_STRUCTURED_AGENTS, + agent: 'codex', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a Cancel reaches only cancelTurn, dismissPrompt and routePromptCancel, all supplied. + adapter: adapter as unknown as StructuredAgentSessionAdapter, + persistOptions: async () => undefined, + resolvedBy: 'client-1', + publish: vi.fn(), + now: () => 1 + } +} + +function chatStop() { + return vi.fn(async (): Promise => ({ + outcome: { ok: true, value: { turnId: 'turn-2', cancelled: true } }, + endsSession: false + })) +} + +function resolution(journal: AgentSessionJournal, itemId: string) { + const body = journal.snapshot().items.find((item) => item.itemId === itemId)?.body + return body?.kind === 'approval' || body?.kind === 'question' ? body.resolution.state : undefined +} + +describe('a card an ended generation raised, cancelled from a client that still shows it', () => { + const pi = () => ({ kind: 'dismiss' as const }) + it.each([ + ['Claude approval', 'approval', claudePromptCancelRoute], + ['Claude question', 'question', claudePromptCancelRoute], + ['Pi (every card dismisses)', 'approval', pi] + ] as const)( + '%s: dismissed only, even when the request names the live turn', + async (_name, kind, route) => { + const { journal, itemId } = await endedCardAndLiveTurn(kind) + const agent = provider(route) + const stop = chatStop() + const interrupt = vi.fn() + + const result = await cancelStructuredAgentSessionPrompt( + context(journal, agent), + { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } }, + { stop, interrupt } + ) + + expect(result).toMatchObject({ ok: true, value: { turnId: 'turn-2', cancelled: true } }) + expect(resolution(journal, itemId)).toBe('cancelled') + expect(stop).not.toHaveBeenCalled() + expect(interrupt).not.toHaveBeenCalled() + expect(agent.cancelTurn).not.toHaveBeenCalled() + expect(agent.dismissPrompt).not.toHaveBeenCalled() + queueUntouched(journal) + } + ) + + it.each([ + ['ACP: the named live turn is interrupted', true], + ['Codex: its live child holds no such card, so nothing is interrupted, as on main', false] + ] as const)("no card route, through the turn's own cancel. %s", async (_name, interrupts) => { + const { journal, itemId } = await endedCardAndLiveTurn() + const agent = provider() + agent.cancelTurn.mockResolvedValueOnce({ cancelled: interrupts }) + const stop = chatStop() + const ctx = context(journal, agent) + const input = { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } } + + const result = await cancelStructuredAgentSessionPrompt(ctx, input, { + stop, + // What the host's plan runs: the turn's cancel, with the card, as on main. + interrupt: () => performCancel(ctx, { clientOperationId: 'cancel-1', ...input }) + }) + + expect(result).toMatchObject({ ok: true, value: { cancelled: interrupts } }) + expect(agent.cancelTurn).toHaveBeenCalledOnce() + expect(agent.cancelTurn.mock.calls[0]?.[0]).toMatchObject({ + turnId: 'turn-2', + prompt: { itemId } + }) + expect(stop).not.toHaveBeenCalled() + expect(resolution(journal, itemId)).toBe('cancelled') + queueUntouched(journal) + }) + + it.each([ + ['naming no turn', undefined], + ["naming the ended generation's turn", 'turn-1'] + ])('Codex or ACP, %s: dismissed only, and the live turn runs on', async (_name, turnId) => { + const { journal, itemId } = await endedCardAndLiveTurn() + const agent = provider() + const stop = chatStop() + const interrupt = vi.fn() + + const result = await cancelStructuredAgentSessionPrompt( + context(journal, agent), + { ...(turnId ? { turnId } : {}), prompt: { itemId, expectedRevision: 1 } }, + { stop, interrupt } + ) + + expect(result).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(resolution(journal, itemId)).toBe('cancelled') + expect(interrupt).not.toHaveBeenCalled() + expect(stop).not.toHaveBeenCalled() + expect(agent.cancelTurn).not.toHaveBeenCalled() + queueUntouched(journal) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 51894a9ff8d7..a2b30d57c301 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -18,7 +18,8 @@ import { endExitedStructuredAgentSessionChildUnderSerialize, settleStructuredAgentSessionChildExit, type StructuredAgentSessionChildExit, - type StructuredAgentSessionChildExitContext + type StructuredAgentSessionChildExitContext, + type StructuredAgentSessionGenerationEnded } from './structured-agent-session-child-exit' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' @@ -31,8 +32,8 @@ export class StructuredAgentSessionEventRecovery { store: StructuredAgentSessionHostDeps['store'] sessions: Map flushLifecycle: (sessionId: string) => Promise - publishFence: (sessionId: string, session: StructuredAgentSessionHostSession) => void publishStatus?: (sessionId: string) => void + generationEnded: StructuredAgentSessionGenerationEnded serialize: (sessionId: string, task: () => Promise) => Promise now: () => number runtimeState: StructuredAgentSessionHostRuntimeState diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts index 476089292a93..3a443559e2e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts @@ -577,7 +577,7 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } + { fence: 5, ownerFence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } ]) deferred.close() } @@ -601,7 +601,7 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } + { fence: 5, ownerFence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } ]) deferred.close() } @@ -637,7 +637,10 @@ describe('producer linkage reaches the journal through every append path', () => // A control, not a pin. Absence is the claim, so the keys must be missing // rather than present-and-undefined: a reader holding this options object // would read `agentId: undefined` as a key that exists. - expect(journalAppendOptions).toEqual([{ fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE }]) + // A provider observation names the generation that produced it: the bound fence. + expect(journalAppendOptions).toEqual([ + { fence: 5, ownerFence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ]) deferred.close() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index 1e2d2c64d70a..d375bc9cddd7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -246,7 +246,9 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { settlementId, mutations, // No row-level linkage: each mutation names its own (see the batch row builder). - fence: bound.fence + fence: bound.fence, + // The provider's own observation: its child's generation produced every item. + ownerFence: bound.fence }) }, { ...options, lifecycle: true } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts new file mode 100644 index 000000000000..373ca6a3da37 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts @@ -0,0 +1,224 @@ +// A queued card when the agent ahead of it exits: every frame from the exit on reads the dead +// generation as ended (not Working, none of its prompts answerable), and the card is sent at once, +// whether the exit's release write lands, waits or keeps failing: an acquisition then replaces the +// owner once it is proven dead, and never one that is alive or unproven. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../../shared/agent-session-record.test-fixture' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession +} from '../../../shared/structured-agent-session-reducer' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + exitChild, + leaveUnfinishedWork +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** A card queued behind a running turn that raised a prompt. */ +async function queuedBehindPrompt( + probeOwner?: StructuredAgentSessionHostDeps['probeOwner'] +): Promise<{ current: QueuedMessageTestRig; draftId: string }> { + rig = await createQueuedMessageTestRig({ + restartable: true, + ...(probeOwner ? { probeOwner } : {}) + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + return { current, draftId: queued.value.queued.messageId } +} + +/** Every release write refused, as storage that keeps failing would. */ +function refuseReleases(current: QueuedMessageTestRig): () => number { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const transition = current.store.transitionHandoff.bind(current.store) + let refused = 0 + vi.spyOn(current.store, 'transitionHandoff').mockImplementation((sessionId, change, options) => + transition( + sessionId, + (record) => { + const next = change(record) + if (next.lease.claimStatus === 'released') { + refused += 1 + throw new Error('disk full') + } + return next + }, + options + ) + ) + return () => refused +} + +/** What a client reads after each frame, from the host's own fields. */ +async function watchChat(current: QueuedMessageTestRig) { + let state = EMPTY_STRUCTURED_AGENT_SESSION + const reads: { working: boolean; settled: boolean; prompts: number; cardSent: boolean }[] = [] + await current.host.subscribe({ + id: 'chat', + sessionId: SESSION, + emit: (event) => { + state = reduceStructuredAgentSession(state, { type: 'event', event }) + reads.push({ + working: state.working === true, + settled: state.latestTurn?.turn.state !== 'running', + prompts: state.actionablePromptIds?.length ?? 0, + cardSent: state.submissions.some((entry) => entry.queuedMessageId !== undefined) + }) + } + }) + return reads +} + +const provenDead: StructuredAgentSessionHostDeps['probeOwner'] = async () => ({ + outcome: 'pid-absent' +}) + +describe('an exit whose settlement lands while its release keeps failing', () => { + it('reads the dead generation as ended in every frame, and the card is sent over the proven-dead owner', async () => { + const { current, draftId } = await queuedBehindPrompt(provenDead) + const refused = refuseReleases(current) + const reads = await watchChat(current) + const before = reads.length + + await exitChild(current) + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + await eventually(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + + expect(refused()).toBeGreaterThan(0) + const untilSent = reads.slice(before).filter((read) => !read.cardSent) + expect(untilSent.length).toBeGreaterThan(0) + // Not Working, and the dead generation's prompt answerable, in no frame before the card's send. + expect(untilSent.filter((read) => read.working)).toEqual([]) + expect(untilSent.filter((read) => read.prompts > 0)).toEqual([]) + // Its turn was settled before the send, and the lease belongs to the generation that took it. + expect(untilSent.at(-1)?.settled).toBe(true) + expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + }) +}) + +describe('an exit that settles and releases cleanly', () => { + it('sends the card with no added delay', async () => { + const { current, draftId } = await queuedBehindPrompt() + const started = Date.now() + + await exitChild(current) + await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + + // Inside one turn of the chat's lane, no backoff; the retry's visit once the lane frees finds + // nothing left. + expect(Date.now() - started).toBeLessThan(250) + await eventually(() => + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(false) + ) + }) +}) + +/** What a client shows, without the host clock every frame carries. */ +function readView(state: typeof EMPTY_STRUCTURED_AGENT_SESSION): string { + return JSON.stringify(state, (key, value: unknown) => (key === 'hostNow' ? undefined : value)) +} + +describe('each exit', () => { + // Two generation-end signals reach readers per exit: its death at once, and its outcome once + // settled. The second may restate; it never repeats a frame the first published. + it.each([ + ['its release lands', false], + ['its release keeps failing', true] + ])('publishes each frame once when %s', async (_name, refuse) => { + // No card behind it: a card's own start is a later generation, with frames of its own. + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + if (refuse) { + refuseReleases(current) + } + const events: AgentSessionSubscribeEvent[] = [] + await current.host.subscribe({ id: 'chat', sessionId: SESSION, emit: (e) => events.push(e) }) + const before = events.length + + await exitChild(current) + await current.host.collaboratorsForTests().reconciliation.attempted(SESSION) + + // Every frame changes what a client reads: none repeats a view it already has. + let state = EMPTY_STRUCTURED_AGENT_SESSION + const repeats: string[] = [] + for (const [index, event] of events.entries()) { + const next = reduceStructuredAgentSession(state, { type: 'event', event }) + if (index >= before && readView(next) === readView(state)) { + repeats.push(event.type) + } + state = next + } + expect(repeats).toEqual([]) + }) +}) + +describe('an owner this host cannot prove dead', () => { + it.each([ + ['alive', { outcome: 'identity-matched', matchedOn: ['process-start-time'] }], + ['unproven', { outcome: 'indeterminate', reason: 'no start time' }] + ] satisfies [string, AgentSessionOwnerProbe][])( + 'is never replaced when %s: no agent starts over it, and the card comes back', + async (_name, probe) => { + const { current, draftId } = await queuedBehindPrompt(async () => probe) + refuseReleases(current) + const owner = current.store.getRecord(SESSION)?.lease + + await exitChild(current) + await eventually(async () => + expect((await current.handoff(draftId))?.dispatchState).toBe('rejected') + ) + + expect(current.dispatch).toHaveBeenCalledTimes(1) + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'live', + runtimeFence: owner?.runtimeFence, + ownerProcess: owner?.ownerProcess + }) + expect(await current.drafts()).toEqual([{ messageId: draftId, state: 'returned' }]) + } + ) + + it('widens nothing for a lease this host never saw end: another fence, or no root proof', () => { + const record = agentSessionRecordFixture( + agentSessionLeaseFixture({ claimStatus: 'live', handoffStage: null, runtimeFence: 7 }) + ) + expect(isResumableStructuredAgentSessionRecord(record)).toBe(false) + expect(isResumableStructuredAgentSessionRecord(record, { fence: 6, rootGone: true })).toBe( + false + ) + expect(isResumableStructuredAgentSessionRecord(record, { fence: 7, rootGone: false })).toBe( + false + ) + expect(isResumableStructuredAgentSessionRecord(record, { fence: 7, rootGone: true })).toBe(true) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts index 49fb06e83bad..882e7f940629 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts @@ -8,7 +8,8 @@ import type { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { readAgentSessionHistory, - type AgentSessionHistoryScope + type AgentSessionHistoryScope, + type AgentSessionPublishedWork } from './agent-session-history-page' export function structuredAgentSessionProviderSessionMetadata( @@ -23,8 +24,16 @@ export function readStructuredAgentSessionHistoryResult(input: { record: AgentSessionRecord | null request: AgentSessionHistoryRequest scope?: AgentSessionHistoryScope + /** The host's current-work projection, which the page's turn and prompt fields state. */ + work?: AgentSessionPublishedWork }): AgentSessionHistoryResult { - const result = readAgentSessionHistory(input.journal, input.request, undefined, input.scope) + const result = readAgentSessionHistory( + input.journal, + input.request, + undefined, + input.scope, + input.work + ) const fence = input.record?.lease.runtimeFence const providerSession = structuredAgentSessionProviderSessionMetadata(input.record) if (fence === undefined) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts new file mode 100644 index 000000000000..5c5ddaccea16 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts @@ -0,0 +1,57 @@ +// The host's current-work projection (`structured-agent-session-current-work.ts`) for a chat it +// holds, read from its sessions and lease store. Apart from the projection itself, which stays free +// of host types so shared and client code can import it. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkJournal +} from './structured-agent-session-current-work' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' + +/** The host's sessions and lease store, as the projection reads them. */ +export type StructuredAgentSessionCurrentWorkHost = { + store: Pick + sessions: { + get( + sessionId: string + ): + | Pick< + StructuredAgentSessionHostSession, + 'journal' | 'lastEndedChild' | 'operationalRevision' | 'child' + > + | undefined + } +} + +/** A held chat's current work; null when this host holds no conversation for it. */ +export function hostStructuredAgentSessionCurrentWork( + host: StructuredAgentSessionCurrentWorkHost, + sessionId: string +): StructuredAgentSessionCurrentWork | null { + const session = host.sessions.get(sessionId) + return session + ? heldStructuredAgentSessionCurrentWork(host.store, sessionId, session.journal, session) + : null +} + +/** The one current work every host reader decides and displays by: this host's child is always + * evidence (none held reads as none), so an action never reads work the display does not show. */ +export function heldStructuredAgentSessionCurrentWork( + store: StructuredAgentSessionCurrentWorkHost['store'], + sessionId: string, + journal: StructuredAgentSessionCurrentWorkJournal, + session: + | Pick + | undefined +): StructuredAgentSessionCurrentWork { + const ended = session?.lastEndedChild + return structuredAgentSessionCurrentWork(journal, { + record: store.getRecord(sessionId), + replaced: store.replacedRuntime(sessionId), + child: session?.child ?? null, + ...(ended ? { ended } : {}), + revision: session?.operationalRevision ?? 0 + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 9ff8543d514e..524a370b4a5b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -2,14 +2,12 @@ // messages to a provider child. Bundled because they share one invariant — a conversation open // with a message queued has a delivery loop — and the open is where a loop for leftovers wakes. -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { holdClosedStructuredAgentSessionSends } from './structured-agent-session-host-lifetime' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openStructuredAgentSessionConversation, - resettleOpenStructuredAgentSessionConversation, - type OpenedStructuredAgentSessionConversation, - type StructuredAgentSessionConversationOpenOptions + type OpenedStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' import type { StructuredAgentSessionClientDelivery } from './structured-agent-session-client-delivery' import { StructuredAgentSessionDeliveryLoop } from './structured-agent-session-delivery-loop' @@ -21,16 +19,12 @@ import type { } from './structured-agent-session-host-types' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import { recoverStructuredRewind } from './structured-rewind-recovery' import { retireSignedOutStructuredAgentSessionChild } from './structured-agent-session-signed-out-child' export type StructuredAgentSessionConversationDelivery = { loop: StructuredAgentSessionDeliveryLoop /** For a caller inside the session's serialize. */ - open: ( - sessionId: string, - options?: StructuredAgentSessionConversationOpenOptions - ) => Promise + open: (sessionId: string) => Promise /** Every commit a conversation's journal makes: one may have ended the command that held its * queue. Enqueued through the session's serialize, never read here, so a commit that lands while * a step is deciding to stop wakes the loop after that step rather than being lost to it. */ @@ -38,7 +32,7 @@ export type StructuredAgentSessionConversationDelivery = { /** A person's Stop settle opened or closed. It writes no row, so only what reads the settle * moves: the session's status row and the steer hold's handover. Never activity. */ afterSettleEdge: (sessionId: string, journal: AgentSessionJournal) => void - /** Stops the loop and the resettle on a proof of death; quit's first step. */ + /** Stops the loop; quit's first step. */ dispose: () => void /** Indexes a conversation some other open produced, as `open` would have. */ adoptOpened: ( @@ -78,6 +72,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { : sessions.get(sessionId) await retireSignedOutStructuredAgentSessionChild(sessionId, session, { work: { + currentWork: () => + hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId), childWork: () => input.clientDelivery.readChildWork(sessionId), hasOpenDispatch: () => { const record = deps.store.getRecord(sessionId) @@ -109,6 +105,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { ), logger: deps.logger, record: (sessionId) => deps.store.getRecord(sessionId), + currentWork: (sessionId) => + hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId), readChildWork: input.clientDelivery.readChildWork, stopping: input.clientDelivery.readStopping, now: () => deps.now?.() ?? Date.now() @@ -117,20 +115,19 @@ export function createStructuredAgentSessionConversationDelivery(input: { sessionId: string, opened: OpenedStructuredAgentSessionConversation ): Promise => { - const { session } = opened - sessions.set(sessionId, session) + // Indexing a chat writes nothing: the loop never hands over a send an earlier process left + // (`StructuredAgentSessionCurrentWork.handsOver`), and keeping one as a card is startup's. + sessions.set(sessionId, opened.session) input.clientDelivery.publishRestored(sessionId) - await settleInterruptedCommands(deps, sessionId, session) - if (session.journal.submissions().some(isQueuedAgentJournalSubmission)) { - loop.wake(sessionId) - } } const wakesQueued = new Set() const afterCommit = (sessionId: string, journal: AgentSessionJournal): void => { + const work = hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId) if ( wakesQueued.has(sessionId) || - structuredAgentSessionCommandRunning(journal) || - !journal.submissions().some(isQueuedAgentJournalSubmission) + !work || + structuredAgentSessionCommandRunning(work) || + !journal.submissions().some((submission) => work.handsOver(submission)) ) { return } @@ -149,25 +146,6 @@ export function createStructuredAgentSessionConversationDelivery(input: { }) }) } - // A chat open before its owner's death was proven revises what its open settled. Queued, never - // awaited: the writer can hold this session's serialize (an attach recovering its lease). - const stopResettling = deps.store.onDeathEvidence((sessionId) => { - if (sessions.has(sessionId)) { - void input - .trackStart( - input.serialize(sessionId, () => - resettleOpenStructuredAgentSessionConversation(deps, sessionId, sessions.get(sessionId)) - ) - ) - .catch((error: unknown) => - deps.logger.warn('resettling an open chat after its owner died failed', { - scope: 'death-evidence-resettle', - sessionId, - error - }) - ) - } - }) return { loop, afterCommit, @@ -176,34 +154,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { afterCommit(sessionId, journal) }, adoptOpened, - dispose: () => { - loop.dispose() - stopResettling() - }, - open: (sessionId, options) => - openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) - } -} - -/** - * A rewind found prepared when the conversation opens was started under a child this process no - * longer has — the open runs only when none is indexed — so nothing will finish it, and left alone - * it refuses every send until a view attaches. Settled here instead of by a start inside - * acceptance. A Codex rewind only its provider can prove stays for the attach. - */ -async function settleInterruptedCommands( - deps: StructuredAgentSessionHostDeps, - sessionId: string, - session: StructuredAgentSessionHostSession -): Promise { - const fence = structuredAgentSessionConversationFence(deps.store, sessionId) - try { - await recoverStructuredRewind(deps, sessionId, session.journal, fence) - } catch (error) { - deps.logger.warn('settling an interrupted rewind on open failed', { - scope: 'rewind-recovery', - sessionId, - error - }) + dispose: () => loop.dispose(), + open: (sessionId) => + openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 8e072e714b6d..76a37199f5d3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -49,6 +49,8 @@ export type StructuredAgentSessionLifetimeContext = { now: () => number /** Re-projects the session's status after its agent stopped and the chat stays. */ publishStatus?: (sessionId: string) => void + /** A generation ended (`StructuredAgentSessionClientDelivery.publishGenerationEnded`). */ + generationEnded?: (sessionId: string) => void /** Hands the delivery loop what is queued; for a caller inside the session's serialize. */ wakeDelivery?: (sessionId: string) => void /** The one exit handler (`structured-agent-session-child-exit`), for a caller inside the @@ -71,11 +73,11 @@ type ConversationCloseDeps = Pick & { /** What is still queued when the chat closes will not be handed over: a person's message is kept * as a card that waits for the chat's next turn, the rest rejected (`journal-unsent-send-hold.ts`). - * A quit is not a close: the next open settles what it left. The chat stops running, so the - * reopen mark follows (`markStructuredQueueReopen`): on every `close`, or only once it settled a - * send, for a later re-check of the same close, which must never mark past a new send. `which` narrows it to the messages a close that did not complete - * closed. Best effort, so a close never waits on it: resolves false when it failed, reported and - * never thrown. */ + * A quit is not a close: what it left, the next process's reconciliation pass keeps as cards. The chat + * stops running, so the reopen mark follows (`markStructuredQueueReopen`): on every `close`, or + * only once it settled a send, for a later re-check of the same close, which must never mark past + * a new send. `which` narrows it to the messages a close that did not complete closed. Best + * effort, so a close never waits on it: resolves false when it failed, reported and never thrown. */ export async function holdClosedStructuredAgentSessionSends( deps: ConversationCloseDeps, sessionId: string, @@ -108,26 +110,6 @@ export async function holdClosedStructuredAgentSessionSends( return settled.ok } -/** What an earlier host process left queued and the open could not settle (its write failed): - * kept or rejected now, never handed over. A failure throws. A send that woke this came after the - * ones it settled, so the reopen mark starts where they did. */ -export async function holdRestartedStructuredAgentSessionSends( - logger: StructuredAgentSessionHostDeps['logger'], - sessionId: string, - journal: StructuredAgentSessionHostSession['journal'], - fence: number -): Promise { - const hostInstance = structuredAgentSessionHostInstance() - const settled = await holdUnsentSends(journal, { - fence, - hostInstance, - hold: { cause: 'hostRestarted' } - }) - if (settled !== null) { - await markStructuredQueueReopen(sessionId, journal, fence, logger, settled + 1) - } -} - /** * The agent goes to rest; the conversation stays. Begins the child's close, or joins the one * already begun, and waits for the exit's proof as long as a caller may. Still unproven, it throws @@ -195,6 +177,7 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( journal: session.journal, sessionId, fence: child.fence, + rowFence: Math.max(child.fence, record?.lease.runtimeFence ?? child.fence), generation: child.generation ?? 'unknown', turnItemId: quitCuts, trigger: ending.quit, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts index d604b805eced..201d057bbd3f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts @@ -14,6 +14,7 @@ import { } from './structured-agent-session-provider-child' import { StructuredAgentSessionOptionRevisions } from './structured-agent-session-option-revisions' import { resolveStructuredSessionRecovery } from './structured-agent-session-recovery-resolution' +import type { StructuredAgentSessionLeaseWrites } from './structured-agent-session-lease-release' import { recordAgentSessionStartup } from '../../observability/agent-session-instrumentation' import { StructuredAgentSessionStartupAttempts, @@ -154,10 +155,14 @@ export class StructuredAgentSessionHostRuntimeState { } /** Exit from a latched recovery stage when present-time evidence permits one. */ - resolveRecovery(sessionId: string): Promise<'resolved' | 'unresolved' | 'not-applicable'> { + resolveRecovery( + sessionId: string, + writes?: StructuredAgentSessionLeaseWrites + ): Promise<'resolved' | 'unresolved' | 'not-applicable'> { return resolveStructuredSessionRecovery( { store: this.deps.store, + ...(writes ? { writes } : {}), probeRecord: (record) => this.probeRecord(record), now: () => this.deps.now?.() ?? Date.now(), ...(this.deps.stopOwnerProcess ? { stopOwnerProcess: this.deps.stopOwnerProcess } : {}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-status-feed.ts index 25d475abb460..3ad17339753f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-status-feed.ts @@ -1,4 +1,4 @@ -import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { deferredStructuredAgentSessionLogger, type StructuredAgentSessionLogger @@ -16,7 +16,7 @@ export function createStructuredAgentSessionHostStatusFeed(args: { sessions: StructuredAgentSessionStatusFeedDeps['sessions'] now: () => number deps: () => { - store: { getRecord: (sessionId: string) => AgentSessionRecord | null } + store: Pick logger: StructuredAgentSessionLogger onSessionStatusChanged?: StructuredAgentSessionStatusFeedDeps['onStatusChanged'] statusSink?: StructuredAgentSessionStatusSink @@ -27,6 +27,7 @@ export function createStructuredAgentSessionHostStatusFeed(args: { return new StructuredAgentSessionStatusFeed({ sessions: args.sessions, getRecord: (sessionId) => args.deps().store.getRecord(sessionId), + replacedRuntime: (sessionId) => args.deps().store.replacedRuntime(sessionId), now: args.now, logger: deferredStructuredAgentSessionLogger(() => args.deps().logger), onStatusChanged: (summary, options) => args.deps().onSessionStatusChanged?.(summary, options), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts index d9b8e068c37e..308aa76035e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts @@ -1,7 +1,8 @@ // The Stop event a host stop writes (`JournalStopEvent`): whether it ends work its event must // record, and the write itself, issued before the kill. -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { heldStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { StructuredAgentSessionStopCause } from './structured-agent-session-adapter' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -62,22 +63,27 @@ export async function stopEndsWork( STOP_EVENT_DRAIN_TIMEOUT_MS, 'slow' as const ) + // The child is live: the work its stop ends is its generation's. + const work = heldStructuredAgentSessionCurrentWork( + context.deps.store, + sessionId, + journal, + session + ) const working = - (drain === 'slow' && journal.stopMarks.latestAcceptedSendUnopened()) || - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), - journal.submissions(), - child.fence - ) - return working && (ending.cause === 'user-close' || !defersToPersonsStop(session)) + (drain === 'slow' && journal.stopMarks.latestAcceptedSendUnopened()) || work.working() + return working && (ending.cause === 'user-close' || !defersToPersonsStop(session, work)) } /** A host stop of work a person's Stop is already ending must not supersede that Stop's reason: * the turn it decides, or, with no turn running, the work under the queue pause it still holds, * with nothing sent since, which a host's event would lift. */ -function defersToPersonsStop(session: StructuredAgentSessionHostSession): boolean { +function defersToPersonsStop( + session: StructuredAgentSessionHostSession, + work: StructuredAgentSessionCurrentWork +): boolean { const { journal } = session - const live = journal.activeTurnId() + const live = work.activeTurnId() if (live !== null) { return journal.stopMarks.personStopDecides(live) } @@ -101,12 +107,15 @@ export function recordStopEvent( if ('recorded' in ending) { return Promise.resolve(null) } - const turnId = session.journal.activeTurnId() + const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) + const turnId = heldStructuredAgentSessionCurrentWork( + context.deps.store, + sessionId, + session.journal, + session + ).activeTurnId() return session.journal - .appendStopEvent( - { reason: ending.cause, ...(turnId !== null ? { turnId } : {}) }, - structuredAgentSessionConversationFence(context.deps.store, sessionId) - ) + .appendStopEvent({ reason: ending.cause, ...(turnId !== null ? { turnId } : {}) }, fence) .then( () => (ending.cause === 'user-close' ? session.journal.stopMarks.beginSettle() : null), (error: unknown) => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index 2fc2bc28d735..d007029296c3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -1,6 +1,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import type { JournalWriteOptions } from '../agent-session-journal/journal-database' /** * Chat-tab visibility is the deletion funnel: every path that removes a chat as a user-facing @@ -97,7 +98,8 @@ export function createStructuredAgentSessionTabSurface( getPersistedVisibleSessionTabIndex: () => host.deps.store.getVisibleSessionTabIndex(), getSessionTabId: (sessionId: string): string | null => host.deps.store.getSessionTabId(sessionId), - showSessionTabs: (sessionIds: readonly string[]) => host.deps.store.showSessionTabs(sessionIds), + showSessionTabs: (sessionIds: readonly string[], options?: JournalWriteOptions) => + host.deps.store.showSessionTabs(sessionIds, options), notifySessionTabHidden: (sessionId: string): void => notifyTabHidden(host, sessionId), setSessionTabVisibility: async ( sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index d6372878b2bd..35fa639691df 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -110,6 +110,9 @@ export type StructuredAgentSessionHostSession = { * observed exit. */ child: StructuredAgentSessionProviderChild | null lastEndedChild?: StructuredAgentSessionEndedChild + /** Bumped by every signal that a generation ended, which can change what is current with no + * journal row or lease write: readers key what they cache of current work on it. */ + operationalRevision?: number /** Owned by one live restart action, so another caller cannot replace its progress. */ restartResume?: NonNullable & { operationId: string } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 8f9a97671a07..cf8caa406618 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -30,7 +30,7 @@ import { type StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' import { settleStructuredAgentSessionLateDispatch } from './structured-agent-session-late-dispatch' -import { releaseStructuredAgentSessionUnansweredDispatches } from './structured-agent-session-unanswered-dispatch-release' +import { releaseStructuredAgentSessionUnansweredDispatches as releaseUnanswered } from './structured-agent-session-unanswered-dispatch-release' import { flushStructuredAgentSessionHost } from './structured-agent-session-host-teardown' import type { StructuredAgentSessionCaller, @@ -47,8 +47,7 @@ import { } from './structured-agent-session-restart-resume-host' import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' import * as conversation from './structured-agent-session-host-delivery' -import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -import { wireStructuredAgentSessionQueuedMessages } from './structured-agent-session-queued-wiring' +import { wireStructuredAgentSessionQueuedMessages as wireQueuedMessages } from './structured-agent-session-queued-wiring' import * as sessionLogger from './structured-agent-session-logger' export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' @@ -58,24 +57,26 @@ export class StructuredAgentSessionHost { this ) private readonly sessions = new StructuredAgentSessionConversations({ - deliver: (sessionId, journal) => { + deliver: (sessionId, journal, committed) => { this.subscribers.publish(sessionId, journal) this.conversationDelivery.afterCommit(sessionId, journal) + this.restore.reconciliation.observeCommit(sessionId, committed.lowestFence) }, deliverSettleEdge: (id, journal) => this.conversationDelivery.afterSettleEdge(id, journal), logger: sessionLogger.deferredStructuredAgentSessionLogger(() => this.deps.logger), - onOpened: (sessionId) => this.queued.onOpened(sessionId), + onOpened: (sessionId, journal) => { + this.queued.onOpened(sessionId) + this.restore.reconciliation.opened(sessionId, journal) + }, now: () => this.now() }) - private readonly queued = wireStructuredAgentSessionQueuedMessages(this.sessions, () => - this.mutationContext() - ) + private readonly queued = wireQueuedMessages(this.sessions, () => this.mutationContext()) // Every journal publish is activity: the one renewal the idle sweep reads. private readonly clientDelivery = new StructuredAgentSessionClientDelivery( this.sessions, () => this.now(), () => this.deps, - (sessionId) => this.queued.onJournalActivity(sessionId), + this.queued, (sessionId) => this.restartResume.onAgentStarted(sessionId), (sessionId) => this.backgroundTasks.publish(sessionId), (sessionId) => this.backgroundTasks.read(sessionId) @@ -129,26 +130,27 @@ export class StructuredAgentSessionHost { }) this.restore = reveal.createStructuredAgentSessionHostRestore(deps, { reconcileLeases: this.reconcileLeases, - resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), + resolveRecovery: (sessionId, writes) => this.runtimeState.resolveRecovery(sessionId, writes), serialize: (sessionId, task) => this.serialize(sessionId, task), hasSession: this.hasSession, // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened, - onUnopened: (sessionId) => this.tabs.markUnopened(sessionId) + onUnopened: (sessionId) => this.tabs.markUnopened(sessionId), + startup: { + sessions: this.sessions, + tasks: this.tasks, + clientDelivery: this.clientDelivery, + queue: this.queued.drain + } }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, store: deps.store, sessions: this.sessions, flushLifecycle: (sessionId) => this.runtimeState.lifecycleBarrier(sessionId), - publishFence: (sessionId, session) => - this.subscribers.snapshot( - sessionId, - session.journal, - structuredAgentSessionConversationFence(deps.store, sessionId) - ), publishStatus: this.clientDelivery.publishStatusAndSettlement, + generationEnded: this.restore.reconciliation.signal, serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), now: () => this.now(), runtimeState: this.runtimeState, @@ -175,8 +177,7 @@ export class StructuredAgentSessionHost { private now = (): number => this.deps.now?.() ?? Date.now() hasSession = (sessionId: string): boolean => this.sessions.has(sessionId) - hasLoadedProviders = (): boolean => - [...this.sessions.values()].some(({ child }) => child !== null) + hasLoadedProviders = (): boolean => this.sessions.hasLoadedProviders() sessionAgent = (sessionId: string) => this.deps.store.getRecord(sessionId)?.provider ?? null handleAdapterEvent = (event: Parameters[0]) => @@ -190,6 +191,7 @@ export class StructuredAgentSessionHost { sessions: this.sessions, now: () => this.now(), publishStatus: this.clientDelivery.publishStatus, + generationEnded: this.restore.reconciliation.signal, wakeDelivery: (sessionId: string) => this.conversationDelivery.loop.wake(sessionId), endExitedChild: this.eventRecovery.endExitedChildUnderSerialize } satisfies StructuredAgentSessionLifetimeContext @@ -208,8 +210,7 @@ export class StructuredAgentSessionHost { } /** Releases a session's resources without ending the conversation; see the lifetime's close. * `user-close` makes a turn it cuts short the user's cancellation; an `evict` leaves it news. */ - close: StructuredAgentSessionConversationLifetime['close'] = (sessionId, cause) => - this.lifetime.close(sessionId, cause) + close: StructuredAgentSessionConversationLifetime['close'] = (...a) => this.lifetime.close(...a) supportsCreate = (location: AgentSessionExecutionLocation, agent: string): boolean => providerSupport.adapterSupportsCreate(this.deps.adapter, location, agent) @@ -235,6 +236,8 @@ export class StructuredAgentSessionHost { onSessionsHeld = (listener: () => void): (() => void) => this.deps.store.onFirstRecord(listener) reconcileRestartLeases = (): Promise => this.restore.reconcileRestartLeases() + /** Once the settlement the last startup reconcile began has run for every chat. */ + startupSettled = (): Promise => this.restore.startupSettled() restoreReadableSessions = (sessionIds?: readonly string[]): Promise => this.restore.restoreReadableSessions(sessionIds) @@ -257,7 +260,9 @@ export class StructuredAgentSessionHost { /** Quit: no exit or recovery settled after this starts a child or hands a message over, and the * queue hands no card over. */ stopDelivery = (): void => - [this.conversationDelivery, this.queued.drain].forEach((d) => d.dispose()) + [this.conversationDelivery, this.queued.drain, this.restore.reconciliation].forEach((d) => + d.dispose() + ) async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { this.stopDelivery() @@ -287,6 +292,7 @@ export class StructuredAgentSessionHost { wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), stopAgent: (sessionId, ending) => this.lifetime.stopAgent(sessionId, ending), wakeQueuedDrain: (sessionId) => this.queued.drain.schedule(sessionId), + retry: this.restore.reconciliation, acquireAborts: this.runtimeState.acquireAborts, optionRevisions: this.runtimeState.optionRevisions, now: () => this.now() @@ -313,8 +319,7 @@ export class StructuredAgentSessionHost { rewind = (caller: StructuredAgentSessionCaller, params: AgentSessionRewindParams) => rewindStructuredAgentSession(this.mutationContext(), this.attachContext(), caller, params) - conversationCommand = (...args: Parameters) => - this.conversationCommands.run(...args) + conversationCommand = this.conversationCommands.run /** Undefined means unavailable; an empty array is an authoritative catalog. */ readCommands = (sessionId: string) => ({ commands: this.clientDelivery.readCommands(sessionId) }) @@ -330,14 +335,16 @@ export class StructuredAgentSessionHost { journalSnapshot = async (sessionId: string): Promise => (await this.lifetime.conversation(sessionId)).journal.snapshot() + /** The host's projection of a held chat's current work (`structuredAgentSessionCurrentWork`). */ + currentWork = this.clientDelivery.readCurrentWork + subscribe = (input: AgentSessionSubscribeInput) => this.backgroundTasks.subscribe(input) settleLateDispatch = (input: Parameters[1]) => settleStructuredAgentSessionLateDispatch(this.mutationContext(), input) - releaseUnansweredDispatches = ( - input: Parameters[1] - ) => releaseStructuredAgentSessionUnansweredDispatches(this.mutationContext(), input) + releaseUnansweredDispatches = (input: Parameters[1]) => + releaseUnanswered(this.mutationContext(), input) publishChildWorkEvidence = this.clientDelivery.publishChildWork unsubscribe = (sessionId: string, id: string): void => this.subscribers.close(sessionId, id) @@ -357,6 +364,7 @@ export class StructuredAgentSessionHost { runtimeState: this.runtimeState, conversationDelivery: this.conversationDelivery, lifetime: this.lifetime, - serialize: this.serialize + serialize: this.serialize, + reconciliation: this.restore.reconciliation }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts index 7de8ad28581f..f3adb18b2a2d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts @@ -225,7 +225,7 @@ describe('the idle sweep', () => { await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) }) - it('never stops a worker whose orchestration dispatch is open, and stops it once it settles (P2-19 i)', async () => { + it('never stops the retry whose orchestration dispatch is open, and stops it once it settles (P2-19 i)', async () => { await rig.dispose() let open = true const hasOpenDispatch = vi.fn(() => open) @@ -320,6 +320,7 @@ describe('the idle sweep with no child running (P2-22 ii)', () => { now: () => IDLE_MS + 1, isDisposed: () => false, deliveryActive: () => false, + store: { getRecord: () => null, replacedRuntime: () => undefined }, childWork: () => undefined, hasOpenDispatch: () => false, providerHoldsDispatch: () => false, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts index 1fd7fc937568..2821be3b2f7b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts @@ -9,11 +9,17 @@ // Owed work is derived on every tick, never stored, so there is nothing to disagree with it. import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + structuredAgentSessionHandsOver, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' +import { + heldStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkHost +} from './structured-agent-session-host-current-work' export const STRUCTURED_AGENT_SESSION_IDLE_SWEEP_INTERVAL_MS = 5 * 60_000 export const STRUCTURED_AGENT_SESSION_IDLE_MS = 30 * 60_000 @@ -33,6 +39,8 @@ export type StructuredAgentSessionIdleSweepDeps = { hasOpenDispatch: (sessionId: string) => boolean /** The provider holds a send it took and has not answered; see `holdsDispatch`. */ providerHoldsDispatch: (sessionId: string) => boolean + /** The lease store the current-work projection reads (`heldStructuredAgentSessionCurrentWork`). */ + store: StructuredAgentSessionCurrentWorkHost['store'] /** Each of these runs inside the session's serialize and never takes it again. */ stopAgent: (sessionId: string) => Promise stopStartingAgent: (sessionId: string) => Promise @@ -42,35 +50,34 @@ export type StructuredAgentSessionIdleSweepDeps = { idleMs?: number } -/** The host's reads of what a running child still serves besides its journal. */ +/** The host's reads of what a running child still serves. */ export type StructuredAgentSessionChildWorkReads = { + /** The host's projection of current work: an ended generation's turn or prompt holds nothing. + * Null when the host holds no conversation, which has none. */ + currentWork: () => Pick< + StructuredAgentSessionCurrentWork, + 'activeTurnId' | 'hasActionablePrompt' + > | null childWork: () => readonly AgentChildWorkView[] | undefined hasOpenDispatch: () => boolean providerHoldsDispatch: () => boolean } /** Work the running child owes apart from queued sends: a turn, a child agent, an open - * orchestration dispatch, a send the provider holds, or a prompt nobody answered. Whatever owes - * any of it keeps its child; it builds no snapshot. */ + * orchestration dispatch, a send the provider holds, or a prompt nobody answered (a subagent's + * too, which the lead turn cannot see). Whatever owes any of it keeps its child; it builds no + * snapshot. */ export function structuredAgentSessionChildHasOpenWork( - journal: Pick, reads: StructuredAgentSessionChildWorkReads ): boolean { - if ( - journal.activeTurnId() !== null || + const work = reads.currentWork() + return ( + (work !== null && work.activeTurnId() !== null) || agentChildWorkLiveness(reads.childWork()) !== null || reads.hasOpenDispatch() || - reads.providerHoldsDispatch() - ) { - return true - } - let pendingPrompt = false - journal.visitItems((_itemId, _sequence, body) => { - // A subagent can raise a prompt the lead turn cannot see. - pendingPrompt ||= - (body.kind === 'approval' || body.kind === 'question') && body.resolution.state === 'pending' - }) - return pendingPrompt + reads.providerHoldsDispatch() || + (work !== null && work.hasActionablePrompt()) + ) } export class StructuredAgentSessionIdleSweep { @@ -151,7 +158,10 @@ export class StructuredAgentSessionIdleSweep { private queuedOrDelivering(sessionId: string, session: StructuredAgentSessionHostSession) { return ( this.deps.deliveryActive(sessionId) || - session.journal.submissions().some(isQueuedAgentJournalSubmission) + // Only a send this process may hand over: an earlier one's is bookkeeping, owed to no child. + session.journal + .submissions() + .some((submission) => structuredAgentSessionHandsOver(session.journal, submission)) ) } @@ -160,7 +170,14 @@ export class StructuredAgentSessionIdleSweep { private owesWork(sessionId: string, session: StructuredAgentSessionHostSession): boolean { return ( this.queuedOrDelivering(sessionId, session) || - structuredAgentSessionChildHasOpenWork(session.journal, { + structuredAgentSessionChildHasOpenWork({ + currentWork: () => + heldStructuredAgentSessionCurrentWork( + this.deps.store, + sessionId, + session.journal, + session + ), childWork: () => this.deps.childWork(sessionId), hasOpenDispatch: () => this.deps.hasOpenDispatch(sessionId), providerHoldsDispatch: () => this.deps.providerHoldsDispatch(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts index e7ce667ed2b6..4aa4d0f646da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts @@ -3,6 +3,9 @@ // Every such path calls this, so a row-level field added to the sink's options // reaches the durable row through one edit rather than through whichever spread // the next change remembers. Lifecycle batches carry producers per mutation. +// +// The sink carries only the provider's own observations, bound at its child's fence: that fence is +// the generation whose execution produced each item (`JournalItemRow.ownerFence`). import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import type { JournalItemAppendOptions } from '../agent-session-journal/journal-store-contracts' @@ -14,6 +17,7 @@ export function structuredAgentSessionJournalAppendOptions( ): JournalItemAppendOptions { return { fence, + ownerFence: fence, ...(options.observedAt === undefined ? {} : { observedAt: options.observedAt }), turnScope: options.turnScope, ...agentJournalLinkageFields(options) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts index b9093d850994..3ba3e3043892 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts @@ -12,10 +12,8 @@ import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-re import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from './structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' const SESSION = 'session-late-proof' @@ -153,10 +151,11 @@ describe('a proof written after an unverifiable settle', () => { it('corrects a call an unverifiable restart eviction closed', async () => { await seedTurn('turn-1', 7) - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, + ownerFence: 7, settlementId: `restart-eviction:${SESSION}:8`, pendingSubmissionReason: 'provider_exited_before_acknowledgement', verdict: { state: 'unverifiable' }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts index 63e3add288e4..99bd45df8406 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts @@ -17,10 +17,15 @@ export type StructuredAgentSessionLeaseStore = Pick< 'getRecord' | 'transitionHandoff' > +/** What the release writes through: the store itself, or bookkeeping's background handle + * (`BackgroundLeaseWrites`). */ +export type StructuredAgentSessionLeaseWrites = Pick + /** Releases the lease of the child whose exit this host observed, with that exit's evidence. * Throws `agent_session_checkpoint_stale` when the record no longer names that child. */ export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { - store: StructuredAgentSessionLeaseStore + store: Pick + writes: StructuredAgentSessionLeaseWrites sessionId: string expectedFence: number now: number @@ -35,7 +40,7 @@ export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { ) { throw new Error('agent_session_checkpoint_stale') } - return releaseStoredAgentSessionOwnerAfterSurfaceClose(input.store, { + return releaseStoredAgentSessionOwnerAfterSurfaceClose(input.writes, { sessionId: input.sessionId, expectedFence: input.expectedFence, now: input.now, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts new file mode 100644 index 000000000000..69ba41d34b97 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -0,0 +1,169 @@ +// What the leftover settlement never touches: an open or a re-attach to the live child writes +// nothing, a release nothing proved settles only as `unverifiable`, and an owner that may still run +// keeps its current work, which still holds the chat. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueueHold } from './structured-agent-session-queued-messages' +import { + APPROVAL, + currentJournal, + currentWork, + exitWhileSettlementFails, + leaveUnfinishedWork, + logEveryWrite, + recoveredRows, + sendText, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Every row of every table, as stored. */ +function everyRow(db: ReturnType['db']): unknown[] { + return db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + .all() + .map((table) => [table.name, db.prepare(`SELECT * FROM main."${String(table.name)}"`).all()]) +} + +/** Whether the chat reads Working and holds a queue-if-active send, by the host's projection. */ +function holds(current: QueuedMessageTestRig): { working: boolean; queueHeld: boolean } { + const work = currentWork(current) + return { + working: work.working(), + queueHeld: structuredQueueHold({ record: current.store.getRecord(SESSION), work }) !== null + } +} + +describe('reading a chat', () => { + it('opens a chat whose leftovers are owed with zero writes, by every reader', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + // Answered, so no send left pending keeps the conversation open. + await current.settleAccepted(await current.workingSend(), 'work') + await leaveUnfinishedWork(current, { prompt: true }) + // The storage fault stays: the retry keeps retrying in the background, and every attempt + // rolls back, so any write the readers made would be the only one logged. + const { database } = await exitWhileSettlementFails(current) + await current.host.close(SESSION, 'evict') + expect(current.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) + const writes = logEveryWrite(database.db) + + await current.host.revealSession(SESSION) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + await current.host.journalSnapshot(SESSION) + const unsubscribe = await current.host.subscribe({ + id: 'reader', + sessionId: SESSION, + emit: () => undefined + }) + unsubscribe() + + expect(writes()).toEqual([]) + // Still owed, and still as the dead generation left it: only the host's retry settles it. + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(true) + expect(turnState(current)).toBe('running') + expect(holds(current)).toEqual({ working: false, queueHeld: false }) + }) + + it('re-attaches to the live child with zero writes', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { db } = openTestJournalHostDatabase(current.root) + const before = everyRow(db) + const writes = logEveryWrite(db) + + expect(await current.host.attach(CALLER, current.firstAttach)).toMatchObject({ ok: true }) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + + // The replayed operation's ledger row is upserted with the same bytes; nothing else is touched. + expect(writes().filter((entry) => entry !== 'UPDATE agent_session_operations')).toEqual([]) + expect(everyRow(db)).toEqual(before) + expect(turnState(current)).toBe('running') + }) +}) + +describe('a release nothing proved', () => { + it('settles what it left as unverifiable at startup, and the next send is delivered', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + + // The rig's probe answers indeterminate: the restart releases the owner with no proof. + await current.crashRestartHostProcess() + + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'released', + deathEvidence: null + }) + expect(turnState(current)).toBe('unverifiable') + expect(currentJournal(current).itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ + resolution: { state: 'cancelled' } + }) + expect(holds(current)).toEqual({ working: false, queueHeld: false }) + + expect((await sendText(current, 'carry on').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an owner that may still run', () => { + it('reads an unreconciled owner’s work as unverifiable, holding nothing, and settles nothing', async () => { + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => { + throw new Error('the probe could not run') + } + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + + // The restart cannot reconcile: no proof, no release. + await current.crashRestartHostProcess() + + expect(current.store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') + expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual([]) + expect(turnState(current)).toBe('running') + // No child here holds it, and it may still run: not working, its prompt not actionable. + expect(holds(current)).toEqual({ working: false, queueHeld: false }) + }) + + it('leaves a live owner’s work unchanged at every trigger but its own exit', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const database = openTestJournalHostDatabase(current.root) + + // A startup pass and a second attach both find the owner live. + await current.host.reconcileRestartLeases() + await current.host.startupSettled() + expect(await current.host.attach(CALLER, current.firstAttach)).toMatchObject({ ok: true }) + const queued = await sendText(current, 'behind the live turn').result + + expect(queued).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + expect(recoveredRows(database.db)).toEqual([]) + expect(turnState(current)).toBe('running') + expect(holds(current)).toEqual({ working: true, queueHeld: true }) + expect(current.dispatch).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts new file mode 100644 index 000000000000..4a97303d40ee --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -0,0 +1,407 @@ +// The retry of background bookkeeping, driven by every way a generation ends: what it settles, that +// it settles in one commit, and that nothing a person does ever waits on it. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import { + agentSessionWriteNoticeEnglish, + agentSessionWriteNoticeParts +} from '../../../shared/agent-session-refusal-notice' +import { agentSessionRefusalFailure } from '../../../shared/agent-session-write-failure' +import { isSubagentGroupBlock } from '../../../shared/native-chat-types' +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { codexSubagentGroupIdentity } from '../../codex/codex-subagent-roster' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueueHold } from './structured-agent-session-queued-messages' +import { isMainAgentWorking } from './structured-agent-session-turns-cancel' +import { + APPROVAL, + currentJournal, + currentWork, + exitChild, + exitWhileSettlementFails, + leaveUnfinishedWork, + REASONING, + reconciled, + recoveredRows, + REJECT_RECOVERED_ROWS, + rowFence, + ROSTER_GROUP, + sendText, + startUp, + TASK, + TOOL, + turnState, + watchSettlementCommits +} from './structured-agent-session-leftover-settlement.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +describe('an observed exit whose settlement the database refused', () => { + it('is settled by the retry in one commit, while the next send starts and delivers', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const firstSend = await current.workingSend() + await leaveUnfinishedWork(current, { everything: true }) + const { session, database, deadFence } = await exitWhileSettlementFails(current) + + // The fault holds: the send is accepted, starts its agent and is delivered all the same. + const { clientOperationId, result } = sendText(current, 'continue while storage fails') + expect(await result).toMatchObject({ ok: true }) + expect(session.journal.queuedMessages.list()).toEqual([]) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(session.child?.fence).toBe(deadFence + 2) + expect(turnState(current)).toBe('running') + + const commits = watchSettlementCommits() + database.db.exec('DROP TRIGGER reject_recovered') + await reconciled(current) + + // One transaction, at the lease's current fence (the successor's), judged by the exit's proof + // though the successor's reservation cleared it from the lease. + expect(commits()).toHaveLength(1) + const settled = recoveredRows(database.db) + expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 2)) + expect(turnState(current)).toBe('interrupted') + const body = (identity: AgentJournalItemIdentity) => + session.journal.itemBody(agentJournalItemKey(identity)) + expect(body(APPROVAL)).toMatchObject({ resolution: { state: 'cancelled' } }) + expect(body(TOOL)).toMatchObject({ state: 'failed', endedAs: 'interrupted' }) + expect(body(REASONING)).not.toMatchObject({ state: 'running' }) + const roster = body(codexSubagentGroupIdentity(ROSTER_GROUP)) + expect( + roster?.kind === 'message' ? roster.blocks.find(isSubagentGroupBlock)?.agents : undefined + ).toMatchObject([{ id: 'a', state: 'unverifiable' }]) + expect(body(TASK)).toMatchObject({ + blocks: [ + { text: 'Background command "sleep 20" stopped reporting' }, + { state: 'unverifiable' } + ] + }) + // Handed over and never answered: doubt, never "not delivered". + expect(session.journal.submission(firstSend)).toMatchObject({ dispatchState: 'unknown' }) + // The successor's send is its own, untouched. + expect(session.journal.submission(clientOperationId)?.dispatchState).not.toBe('unknown') + }) + + it('settles at the exit itself when nothing refuses it, once', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const database = openTestJournalHostDatabase(current.root) + const commits = watchSettlementCommits() + + await exitChild(current) + await reconciled(current) + + expect(turnState(current)).toBe('interrupted') + expect(commits()).toHaveLength(1) + const settled = recoveredRows(database.db) + // The next start finds nothing left: the retry its end signals writes no row. + expect((await sendText(current, 'next').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + await reconciled(current) + expect(recoveredRows(database.db)).toEqual(settled) + }) +}) + +describe('a settlement that cannot commit never gates the user', () => { + it('delivers the queue-if-active send the STOP reproduction stranded', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { session } = await exitWhileSettlementFails(current) + + const { clientOperationId, result } = sendText(current, 'still stranded?') + + expect(await result).toMatchObject({ ok: true }) + expect((await result).ok && 'queued' in (await result)).toBe(false) + expect(session.journal.queuedMessages.list()).toEqual([]) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) + + it('reads a dead generation’s open turn as not Working, holds nothing, and delivers', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + await exitWhileSettlementFails(current) + const journal = currentJournal(current) + const record = current.store.getRecord(SESSION) + const fence = record?.lease.runtimeFence ?? 0 + const work = currentWork(current) + + // Still saved as running: only its settlement would end it. + expect(turnState(current)).toBe('running') + expect(work.working()).toBe(false) + expect(structuredQueueHold({ record, work })).toBeNull() + expect(isMainAgentWorking({ journal, fence, currentWork: () => work })).toBe(false) + const { items, submissions } = journal.snapshot() + expect( + projectStructuredAgentSessionStatusState(items, submissions, fence, work.scope).summary.status + ).not.toBe('working') + + expect((await sendText(current, 'go on').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) + + it('drains a card that waited on the dead generation, with no journal write', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const queued = await current.send('after this turn', 'queue-if-active').result + expect(queued).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + expect(currentJournal(current).queuedMessages.list()).toHaveLength(1) + const database = openTestJournalHostDatabase(current.root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + + // The end alone woke the drain: the card went out though nothing settled the turn. + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(recoveredRows(database.db)).toEqual([]) + expect(turnState(current)).toBe('running') + }) +}) + +describe('the retry', () => { + /** Lets every attempt a fired timer began run to its end, with no fake time passing. */ + async function settleTurns(): Promise { + for (let turn = 0; turn < 50; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } + } + + it('backs off from 1 s, doubling; a new signal adds no round to the backoff; settles once the fault clears', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + try { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { database } = await exitWhileSettlementFails(current) + const reconciliation = current.host.collaboratorsForTests().reconciliation + const attempts = vi.spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') + expect(retryOwes(reconciliation, SESSION)).toBe(true) + + await vi.advanceTimersByTimeAsync(990) + await settleTurns() + expect(attempts).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(10) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(1) + // The second failure doubled the wait. + await vi.advanceTimersByTimeAsync(1_990) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(10) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(2) + + // A new signal while it backs off adds no round: the timer's next step runs it. + reconciliation.signal(SESSION) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(2) + expect(turnState(current)).toBe('running') + database.db.exec('DROP TRIGGER reject_recovered') + await vi.advanceTimersByTimeAsync(1_990) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(10) + await settleTurns() + expect(retryOwes(reconciliation, SESSION)).toBe(false) + expect(turnState(current)).toBe('interrupted') + } finally { + vi.useRealTimers() + } + }) +}) + +describe('a stale prompt', () => { + it('holds no send, refuses a stale answer in plain words, and ends cancelled once settled', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { session, database } = await exitWhileSettlementFails(current) + const prompt = session.journal.item(agentJournalItemKey(APPROVAL)) + if (!prompt) { + throw new Error('expected the saved prompt') + } + const answer = () => { + const fields = { itemId: prompt.itemId, expectedRevision: prompt.revision, optionId: 'yes' } + return current.host.respondToPrompt(CALLER, { + envelope: { + ...current.envelope(fields, 'agentSession.respondTo:approval', hostTestOperationId()), + expectedRuntimeFence: current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + }, + kind: 'approval', + ...fields + }) + } + + // Unsettled, it holds nothing and takes no answer: the agent that asked is gone. + expect((await sendText(current, 'go on without it').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(session.journal.itemBody(prompt.itemId)).toMatchObject({ + resolution: { state: 'pending' } + }) + const refused = await answer() + expect(refused).toMatchObject({ + ok: false, + refusal: { details: { reason: 'promptOwnerEnded' } } + }) + if (refused.ok) { + throw new Error('expected the answer refused') + } + // One sentence: the agent stopped, the answer was not sent, and how to go on. + const words = agentSessionWriteNoticeEnglish( + agentSessionWriteNoticeParts(agentSessionRefusalFailure(refused.refusal), 'answer', { + agentName: 'Codex' + }) + ) + expect(words).toBe( + 'Codex has stopped, so your answer was not sent. Send a message to continue.' + ) + expect(current.answerPrompt).not.toHaveBeenCalled() + + database.db.exec('DROP TRIGGER reject_recovered') + await reconciled(current) + + expect(session.journal.itemBody(prompt.itemId)).toMatchObject({ + resolution: { state: 'cancelled' } + }) + expect((await answer()).ok).toBe(false) + expect(current.answerPrompt).not.toHaveBeenCalled() + }) +}) + +describe('a late write from an ended generation', () => { + it('is settled by the retry its commit wakes', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + // Answered, so the exit leaves nothing to settle and writes no row at the moved fence. + await current.settleAccepted(await current.workingSend(), 'work') + const deadFence = current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await exitChild(current) + await reconciled(current) + const database = openTestJournalHostDatabase(current.root) + expect(recoveredRows(database.db)).toEqual([]) + const lateCall: AgentJournalItemIdentity = { + provider: 'codex', + threadId: THREAD, + turnId: 'late', + ordinal: 1 + } + + // An operation the closed sink had already taken lands at the generation's own fence. + await currentJournal(current).appendItem( + lateCall, + { kind: 'tool-call', name: 'shell', input: { command: 'make' }, state: 'running' }, + { fence: deadFence, ownerFence: deadFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + + await vi.waitFor(() => + expect(currentJournal(current).itemBody(agentJournalItemKey(lateCall))).toMatchObject({ + state: 'failed' + }) + ) + await reconciled(current) + expect(recoveredRows(database.db).map(rowFence)).toEqual([deadFence + 1]) + }) +}) + +describe('startup', () => { + it('settles a failed exit settlement with no stored mark, starting nothing', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { deadFence } = await exitWhileSettlementFails(current) + // A new process loads the database and the record store back from disk, and starts up. + await current.crashReloadHostProcess() + await reconciled(current) + const database = openTestJournalHostDatabase(current.root) + // The retry opened the chat itself, so it closes it again: nothing else had it open. + await vi.waitFor(() => + expect(current.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) + ) + await current.host.journalSnapshot(SESSION) + + expect(turnState(current)).toBe('interrupted') + expect(currentJournal(current).itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ + resolution: { state: 'cancelled' } + }) + const settled = recoveredRows(database.db) + expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 1)) + expect(current.starts).toHaveBeenCalledOnce() + // A second pass finds nothing owed. + await startUp(current) + expect(recoveredRows(database.db)).toEqual(settled) + }) +}) + +describe('a folder workspace', () => { + it('settles a folder chat’s dead generation the same way, and delivers', async () => { + rig = await createQueuedMessageTestRig({ + restartable: true, + location: { + executionHostId: 'local', + wslDistro: null, + workspaceId: 'folder-1', + workspaceKind: 'folder' + } + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { session, database } = await exitWhileSettlementFails(current) + + const { clientOperationId, result } = sendText(current, 'continue in the folder') + + expect(await result).toMatchObject({ ok: true }) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + database.db.exec('DROP TRIGGER reject_recovered') + await reconciled(current) + expect(turnState(current)).toBe('interrupted') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts new file mode 100644 index 000000000000..300958ee3b5e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts @@ -0,0 +1,306 @@ +// A dead generation's leftover work, for the leftover-settlement tests: a child that leaves a running +// turn (and what else a test asks for), an exit whose settlement the database refuses, and the +// database instruments the tests read. + +import { expect, vi } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + codexSubagentGroupBody, + codexSubagentGroupIdentity +} from '../../codex/codex-subagent-roster' +import { + liveTestJournalRows, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type Database from '../../sqlite/sync-database' +import { JournalRowWriter } from '../agent-session-journal/journal-row-writer' +import type { JournalRow } from '../agent-session-journal/journal-row-schema' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' +import { + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestMessage, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { retryIdle, retryOwes } from './structured-agent-session-retry.test-fixture' + +export const TURN: AgentJournalItemIdentity = { + provider: 'codex', + threadId: THREAD, + turnId: 'unfinished', + ordinal: 1 +} +export const TURN_KEY = agentJournalItemKey(TURN) +const IN_TURN = { kind: 'turn' as const, turnItemId: TURN_KEY } +export const APPROVAL: AgentJournalItemIdentity = { ...TURN, ordinal: 2 } +export const TOOL: AgentJournalItemIdentity = { ...TURN, ordinal: 3 } +export const REASONING: AgentJournalItemIdentity = { ...TURN, ordinal: 4 } +export const ROSTER_GROUP = `${THREAD}:unfinished` +export const TASK: AgentJournalItemIdentity = { + provider: 'orca', + clientMessageId: 'claude-background-task:task-1' +} +/** Refuses every row a settlement writes: its rows are all `recovered`. */ +export const REJECT_RECOVERED_ROWS = ` + CREATE TEMP TRIGGER reject_recovered BEFORE INSERT ON journal_rows + WHEN json_extract(NEW.row_json, '$.recovered') = 1 + BEGIN SELECT RAISE(ABORT, 'temporary storage failure'); END; +` + +/** The live child leaves a running turn, and what else the test asks for, in the journal. */ +export async function leaveUnfinishedWork( + current: QueuedMessageTestRig, + options: { prompt?: true; everything?: true } = {} +): Promise { + const events = current.providerEvents() + events.appendItem( + TURN, + { kind: 'turn', turnId: 'unfinished', state: 'running' }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + if (options.prompt || options.everything) { + events.appendItem( + APPROVAL, + { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }, + { turnScope: IN_TURN } + ) + } + if (options.everything) { + events.appendItem( + TOOL, + { kind: 'tool-call', name: 'shell', input: { command: 'pnpm test' }, state: 'running' }, + { turnScope: IN_TURN } + ) + events.appendItem( + REASONING, + { + kind: 'message', + role: 'assistant', + state: 'running', + blocks: [{ type: 'text', text: 'thinking' }] + }, + { turnScope: IN_TURN } + ) + events.appendItem( + codexSubagentGroupIdentity(ROSTER_GROUP), + codexSubagentGroupBody(ROSTER_GROUP, [ + { id: 'a', label: 'read', state: 'working', startedAt: 10 } + ]), + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + events.appendItem( + TASK, + { + kind: 'message', + role: 'system', + blocks: [ + { type: 'text', text: 'Started background command "sleep 20"' }, + { + type: 'background-task', + taskId: 'task-1', + kind: 'command', + label: 'sleep 20', + state: 'working', + startedAt: 10 + } + ] + }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + } + await current.host.flushStreamedEvents(SESSION) +} + +/** The child dies on its own while the database refuses every settlement row: the exit releases + * its lease (proof written, fence moved) and its work stays saved as running. The host's + * retry is left backing off. */ +export async function exitWhileSettlementFails(current: QueuedMessageTestRig) { + const context = current.host.collaboratorsForTests() + const session = context.sessions.get(SESSION) + const child = session?.child + if (!session || !child?.generation) { + throw new Error('expected the original child') + } + const database = openTestJournalHostDatabase(current.root) + const before = liveTestJournalRows(database.db, SESSION) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await context.serialize(SESSION, async () => {}) + // The retry's first visit waited for the exit's lane; it fails too, and backs off. + await context.reconciliation.attempted(SESSION) + expect(session.child).toBeNull() + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'released', + runtimeFence: child.fence + 1, + deathEvidence: { kind: 'exit-observed', ownerFence: child.fence } + }) + expect(liveTestJournalRows(database.db, SESSION)).toEqual(before) + return { session, database, deadFence: child.fence } +} + +/** The child exits on its own with nothing refused. */ +export async function exitChild(current: QueuedMessageTestRig): Promise { + const context = current.host.collaboratorsForTests() + const child = context.sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected a child') + } + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await context.serialize(SESSION, async () => {}) +} + +/** A person's ordinary text send from the composer, with queueing on, at the current fence. */ +export function sendText(current: QueuedMessageTestRig, text: string) { + const fence = current.store.getRecord(SESSION)?.lease.runtimeFence + if (fence === undefined) { + throw new Error('expected the record') + } + const body = hostTestMessage(text) + const clientOperationId = hostTestOperationId() + const fields = { body, delivery: 'queue-if-active' as const } + const result = current.host.send(CALLER, { + ...fields, + userSend: true, + envelope: { + ...current.envelope(fields, 'agentSession.send', clientOperationId), + expectedRuntimeFence: fence + } + }) + return { clientOperationId, result } +} + +export function currentJournal(current: QueuedMessageTestRig) { + const journal = current.host.collaboratorsForTests().sessions.get(SESSION)?.journal + if (!journal) { + throw new Error('expected the open conversation') + } + return journal +} + +/** The host's projection of the chat's current work, as every host reader asks it. */ +export function currentWork(current: QueuedMessageTestRig): StructuredAgentSessionCurrentWork { + const work = hostStructuredAgentSessionCurrentWork( + { store: current.store, sessions: current.host.collaboratorsForTests().sessions }, + SESSION + ) + if (!work) { + throw new Error('expected the open conversation') + } + return work +} + +export function turnState(current: QueuedMessageTestRig): string | undefined { + return readAgentJournalTurn(currentJournal(current).itemBody(TURN_KEY) ?? undefined)?.state +} + +/** The rows settlements wrote: each is `recovered`. */ +export function recoveredRows(db: Database.Database) { + return liveTestJournalRows(db, SESSION).filter((row) => { + const parsed: unknown = JSON.parse(row.rowJson) + return typeof parsed === 'object' && parsed !== null && 'recovered' in parsed + }) +} + +export function rowFence(row: { rowJson: string }): number { + const parsed: unknown = JSON.parse(row.rowJson) + return typeof parsed === 'object' && parsed !== null && 'fence' in parsed + ? Number(parsed.fence) + : Number.NaN +} + +/** Each journal transaction that wrote settlement rows, by the rows it wrote: one entry per + * commit. */ +export function watchSettlementCommits(): () => JournalRow[][] { + const writes = vi.spyOn(JournalRowWriter.prototype, 'writeRows') + return () => + writes.mock.results.flatMap((result) => + result.type === 'return' && result.value.some((row) => row.recovered === true) + ? [result.value] + : [] + ) +} + +/** Once the host's retry owes the chat nothing. */ +export function reconciled(current: QueuedMessageTestRig, timeout = 10_000): Promise { + return vi.waitFor( + () => + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(false), + { timeout } + ) +} + +/** Logs every INSERT, UPDATE and DELETE on every table of the file, from this connection on. */ +export function logEveryWrite(db: Database.Database): () => string[] { + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + .all() + .map((row) => String(row.name)) + db.exec('CREATE TEMP TABLE test_write_log (entry TEXT)') + for (const table of tables) { + for (const op of ['INSERT', 'UPDATE', 'DELETE']) { + db.exec(`CREATE TEMP TRIGGER test_log_${op}_${table} AFTER ${op} ON main.${table} + BEGIN INSERT INTO test_write_log VALUES ('${op} ${table}'); END;`) + } + } + return () => + db + .prepare('SELECT entry FROM test_write_log') + .all() + .map((row) => String(row.entry)) +} + +/** Startup as the runtime runs it, on any host: the restart reconcile, then the scan of every chat + * it begins, with no stored mark of what is owed, until the host's retry is idle. */ +export async function startUpHost( + host: Pick< + StructuredAgentSessionHost, + 'reconcileRestartLeases' | 'startupSettled' | 'collaboratorsForTests' + > +): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) +} + +/** Startup as the runtime runs it: the restart reconcile, then the scan of every chat it begins, + * until the host's retry owes it nothing. */ +export async function startUp( + current: QueuedMessageTestRig, + host: Pick = current.host +): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() + await reconciled(current) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts new file mode 100644 index 000000000000..edb75ed42c6e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts @@ -0,0 +1,134 @@ +// The one settlement of what ended generations left unfinished. An observed exit runs it first, +// with what it saw; the host's retry runs it after every generation end and at startup +// (`structured-agent-session-reconciliation-retry.ts`). Opening a chat never runs it. Bookkeeping +// no person waits on, so it writes only through a background handle (`BackgroundSettlementWrites`). +// +// Re-derived each time from the journal, the lease and this host's sight of an exit: every turn, +// call, prompt, reasoning row, subagent roster entry and background task whose execution belongs +// to a generation that is not live (`structuredAgentSessionLiveFence`), and every send handed to +// one and never answered. Its turns end `interrupted` only when death evidence names that +// generation or a runtime this one replaced held it, else `unverifiable`, and a later proof revises +// an earlier `unverifiable`. Rows land at +// the lease's current fence in ONE transaction; a run with nothing left plans no row and writes +// nothing. A failure is the caller's to retry: its rows hold nothing, since the projection counts +// only the live generation's work. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import { + planStructuredAgentSessionDeadGeneration, + type StructuredAgentSessionDeadGenerationInput +} from './structured-agent-session-dead-generation-settlement' +import { + planStaleStructuredAgentSessionState, + type StaleStructuredAgentSessionStateJournal +} from './structured-agent-session-stale-state-settlement' +import type { DeadGenerationJournal } from './structured-agent-session-unfinished-work' +import type { BackgroundSettlementWrites } from './structured-agent-session-background-writes' +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' +import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' +import { + structuredAgentSessionLiveFence, + type StructuredAgentSessionWorkEvidence +} from './structured-agent-session-current-work' + +/** Without `replacedRuntime`, a store whose runtime replaced none. */ +export type StructuredAgentSessionLeftoverStore = Pick & + Partial> + +/** An observed exit's settlement: what that child's generation left, settled as the exit says. */ +export type StructuredAgentSessionExitSettlement = Omit< + StructuredAgentSessionDeadGenerationInput, + 'journal' | 'sessionId' | 'fence' +> & { ownerFence: number } + +export type StructuredAgentSessionLeftoverSettlementInput = { + store: StructuredAgentSessionLeftoverStore + sessionId: string + /** Read for the plan; the settlement's write goes through `writes`. */ + journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal + writes: BackgroundSettlementWrites + exit?: StructuredAgentSessionExitSettlement + /** This host's sight of a child's root exit, which ends its generation even when the release + * write failed (`StructuredAgentSessionHostSession.lastEndedChild`). */ + ended?: StructuredAgentSessionWorkEvidence['ended'] + /** Proof a generation end carried that the lease no longer holds (a reservation that replaced a + * proven-dead owner clears it, and a later release writes its own): only what that generation + * and earlier ones left is settled, judged by it. Absent: the lease's own evidence judges. */ + proof?: AgentSessionDeathEvidence & { ownerFence: number } +} + +export type StructuredAgentSessionLeftoverSettlement = + | { ok: true; planned: number } + | { ok: false; error: unknown } + +export async function settleStructuredAgentSessionLeftovers( + input: StructuredAgentSessionLeftoverSettlementInput +): Promise { + try { + const record = input.store.getRecord(input.sessionId) + if (!record) { + return { ok: true, planned: 0 } + } + const fence = record.lease.runtimeFence + const { journal, sessionId } = input + const replaced = input.store.replacedRuntime?.(sessionId) + // Everything below the live generation is ended; with none live, all of it is. + const liveFence = structuredAgentSessionLiveFence({ + record, + replaced, + ...(input.ended ? { ended: input.ended } : {}) + }) + // An exit's own account holds only until a later generation is live: its rows and sends at + // this fence would be that generation's. The general rule settles what the exit left instead. + const exit = liveFence === null && !input.proof ? input.exit : undefined + const { proof } = input + const below = proof + ? Math.min(proof.ownerFence + 1, liveFence ?? fence + 1) + : exit + ? exit.ownerFence + : (liveFence ?? fence + 1) + const failureTextContext = structuredAgentSessionFailureWordsContext(record) + let planned = 0 + await input.writes.appendPlannedLifecycleBatch({ + settlementId: exit + ? `dead-generation:${exit.settlementId}` + : `${STALE_SESSION_ROW_PREFIX}${sessionId}:${fence}:${proof ? `proof-${proof.ownerFence}:` : ''}seq-${journal.cursor().sequence}`, + fence, + recovered: true, + plan: () => { + const exited = exit + ? planStructuredAgentSessionDeadGeneration( + { ...exit, journal, sessionId, fence }, + (itemFence) => itemFence === undefined || itemFence >= exit.ownerFence + ) + : { mutations: [], dispatches: [] } + const settledByExit = new Set(exited.dispatches.map((entry) => entry.clientMessageId)) + const stale = planStaleStructuredAgentSessionState({ + journal, + sessionId, + fence, + acquisitionGeneration: null, + // Read with the rows: the proof the lease holds now. On an observed exit this settlement + // runs before the release, so the exit's own account (`exit`) judges instead. + deathEvidence: proof ?? input.store.getRecord(sessionId)?.lease.deathEvidence ?? null, + replaced, + failureTextContext, + below, + // With an exit's account, nothing is live: every entry of what is settled is ended. + ...(exit ? {} : { entriesBelow: below }) + }) + const mutations = [...exited.mutations, ...stale.mutations] + const dispatches = [ + ...exited.dispatches, + ...stale.dispatches.filter((entry) => !settledByExit.has(entry.clientMessageId)) + ] + planned = mutations.length + dispatches.length + return { mutations, dispatches } + } + }) + return { ok: true, planned } + } catch (error) { + return { ok: false, error } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts index 75c88bcf480e..895f4f39d18b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts @@ -116,7 +116,7 @@ describe('a running turn whose record is older than the loaded page', () => { await runLongTurn('turn-1', 250) const state = hydrate() - expect(journal.activeTurnId()).toBe('turn-1') + expect(journal.runningTurn()?.turnId ?? null).toBe('turn-1') expect(state.hasOlder).toBe(true) // What every client read before: the record is off the page, so the loaded rows said idle. expect(activeStructuredAgentSessionTurnId(state.items)).toBeNull() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index 54a1d769a001..39a3934bd26f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -40,6 +40,7 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { MutationPlan } from './structured-agent-session-mutation-plans' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { runSettledAgentSessionMutation } from './structured-agent-session-operation-settlement' import { agentSessionOperationOutcomeUnknown, @@ -74,6 +75,8 @@ export type AgentSessionMutationRequest = { plan: MutationPlan /** Journal of the attached session, read after `prepareSession`; absent when this host holds none. */ journal: () => AgentSessionJournal | undefined + /** The host's projection of the chat's current work (`hostStructuredAgentSessionCurrentWork`). */ + currentWork?: () => StructuredAgentSessionCurrentWork | null /** Between the ledger's answer and the lease check, for a call that may first have to make the * session ready for itself. Answers with the refusal that ends the call, if any. */ prepareSession?: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index f9b83c3b4f3a..e91d81e03c7b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -1,6 +1,7 @@ // The context every client mutation of a session runs with, and the one path each takes: admit the // envelope against the lease, then run its plan inside the session's serialize. +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' import type { AgentSessionMutationEnvelope, AgentSessionMutationResult @@ -15,6 +16,7 @@ import type { MutationPlan } from './structured-agent-session-mutation-plans' import type { StructuredAgentSessionStopEnding } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionAcquireAborts } from './structured-agent-session-acquire-aborts' import type { StructuredAgentSessionOptionRevisions } from './structured-agent-session-option-revisions' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import type { StructuredAgentSessionCaller, StructuredAgentSessionHostDeps, @@ -39,6 +41,8 @@ export type StructuredAgentSessionMutationContext = { joinChildClose: (sessionId: string) => Promise /** A message was accepted: the session's delivery loop hands it over. */ wakeDelivery: (sessionId: string) => void + /** The host's retry of background bookkeeping (`StructuredAgentSessionRetry`). */ + retry: Pick /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. * Each caller names why (`ending`). */ stopAgent: (sessionId: string, ending: StructuredAgentSessionStopEnding) => Promise @@ -72,6 +76,11 @@ export function mutateStructuredAgentSession( envelope, plan, journal: () => context.sessions.get(envelope.sessionId)?.journal, + currentWork: () => + hostStructuredAgentSessionCurrentWork( + { store: context.deps.store, sessions: context.sessions }, + envelope.sessionId + ), prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), wakeDelivery: (sessionId) => context.wakeDelivery(sessionId), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts index 776937662eb0..40ecfb34c6c1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts @@ -11,11 +11,13 @@ export function mutationTurnContext( record: AgentSessionRecord ): AgentSessionTurnContext { const fence = record.lease.runtimeFence - const persistedOptions = request.store.getRecord(request.envelope.sessionId)?.options + const { sessionId } = request.envelope + const persistedOptions = request.store.getRecord(sessionId)?.options return { - sessionId: request.envelope.sessionId, + sessionId, journal, fence, + ...(request.currentWork ? { currentWork: request.currentWork } : {}), adapter: request.adapter, agents: request.agents, agent: record.provider, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts new file mode 100644 index 000000000000..1cedf8da3543 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts @@ -0,0 +1,27 @@ +// An observed exit's settlement as the host's retry writes it (`settleStructuredAgentSessionLeftovers` +// with the exit's own account), for a chat whose lease that exit released. + +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { backgroundSettlementWrites } from './structured-agent-session-background-writes' +import type { StructuredAgentSessionDeadGenerationInput } from './structured-agent-session-dead-generation-settlement' +import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' +import { scanRecord } from './structured-agent-session-startup-scan.test-fixture' + +/** `ownerFence`: the exited generation's; by default every fence, as a chat's lone generation. */ +export function settleObservedExitForTest( + input: StructuredAgentSessionDeadGenerationInput & { + journal: AgentSessionJournal + ownerFence?: number + } +) { + const { journal, sessionId, fence, ownerFence = 0, ...exit } = input + const released = scanRecord(sessionId, true) + const record = { ...released, lease: { ...released.lease, runtimeFence: fence } } + return settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => record }, + sessionId, + journal, + writes: backgroundSettlementWrites(journal), + exit: { ...exit, ownerFence } + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts index 1778f7e43977..c4d7333b9eb6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts @@ -80,7 +80,10 @@ function cutByNobody(item: AgentJournalRenderItem | undefined): boolean { export async function recordStructuredAgentSessionShutdownCut(input: { journal: OrcaStopRowJournal sessionId: string + /** The stopped child's fence, which keys the row. */ fence: number + /** The fence the row is written at: the lease's, which the child's end moved past `fence`. */ + rowFence: number generation: string turnItemId: string | null trigger: AgentSessionResumeTrigger @@ -108,7 +111,7 @@ export async function recordStructuredAgentSessionShutdownCut(input: { await input.journal.appendItem( { provider: 'orca', clientMessageId }, orcaStopRowBody(input.failureTextContext, input.trigger), - { fence: input.fence, turnScope: { kind: 'turn', turnItemId } } + { fence: input.rowFence, turnScope: { kind: 'turn', turnItemId } } ) } catch (error) { input.logger.warn('recording why a quit cut a reply failed', { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-answer-receipts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-answer-receipts.test.ts index 87ad1fc95a11..8cefd4c0d9ae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-answer-receipts.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-answer-receipts.test.ts @@ -115,7 +115,13 @@ async function reviseFromProvider(itemId: string): Promise { async function rewindKeepingPrompt(itemId: string): Promise { const fence = store.getRecord(SESSION)!.lease.runtimeFence await journal().replaceEpochItems('handle_forked', fence, [ - { identity: PROMPT, body: promptBody(itemId), turnScope: AGENT_JOURNAL_THREAD_SCOPE } + // The prompt keeps the generation that raised it (this one), as a rewind carries it. + { + identity: PROMPT, + body: promptBody(itemId), + turnScope: AGENT_JOURNAL_THREAD_SCOPE, + ownerFence: fence + } ]) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts index b4281859aba0..aaf7885b6326 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts @@ -10,10 +10,13 @@ import { AgentSessionPromptUnavailableError } from './structured-agent-session-a import type { StructuredAgentSessionChatStopRun } from './structured-agent-session-chat-stop' import { answerCancelOfSettledPrompt, + dismissEndedGenerationPrompt, + isEndedGenerationPrompt, validatePendingPrompt, type PendingPromptValidation } from './structured-agent-session-prompt-state' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' type CancelOutcome = TurnOutcome type PendingPrompt = Extract @@ -34,6 +37,9 @@ export async function cancelStructuredAgentSessionPrompt( sessionId: ctx.sessionId, prompt: validated.prompt }) + if (isEndedGenerationPrompt(ctx, validated)) { + return cancelEndedGenerationPrompt(ctx, input, validated, route ? null : routes.interrupt) + } if (!route) { return routes.interrupt() } @@ -63,8 +69,47 @@ export async function cancelStructuredAgentSessionPrompt( return dismissed.ok ? cancelled : dismissed } +/** A card an ended generation raised: no live provider holds it, so it is dismissed in the journal + * alone. As on main, a provider whose card Cancel routes (a dismissal, or a Stop of the turn that + * raised it) stops nothing, since the live turn did not raise it. One with no route gets the + * named live turn's own cancel with the card, as main's plan does, never the chat's Stop, so + * queued messages stay and the queue is not paused: ACP interrupts that turn, and Codex, finding + * no such card in its live child, interrupts nothing. */ +async function cancelEndedGenerationPrompt( + ctx: AgentSessionTurnContext, + input: { turnId?: string }, + pending: PendingPrompt, + interrupt: (() => Promise) | null +): Promise { + const liveTurnId = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() + const interrupted = + interrupt && input.turnId !== undefined && input.turnId === liveTurnId + ? await interrupt() + : null + if (interrupted && !interrupted.ok) { + return interrupted + } + // The interrupt's provider may have let go of it already; what is still pending is dismissed. + const still = validatePendingPrompt(ctx, { + itemId: pending.item.itemId, + expectedRevision: pending.item.revision + }) + if (still.ok) { + const dismissed = await dismissEndedGenerationPrompt(ctx, still) + if (!dismissed.ok) { + return dismissed + } + } + return ( + interrupted ?? { + ok: true, + value: { ...(input.turnId ? { turnId: input.turnId } : {}), cancelled: true } + } + ) +} + function raisedByLiveTurn(ctx: AgentSessionTurnContext, pending: PendingPrompt): boolean { - const live = ctx.journal.liveTurnScope() + const live = contextStructuredAgentSessionCurrentWork(ctx).turnScope() const raised = pending.item.turnScope return live.kind === 'turn' && raised?.kind === 'turn' && raised.turnItemId === live.turnItemId } @@ -102,7 +147,7 @@ async function dismissPrompt( } }, // A revision: the prompt keeps the turn it was raised in. - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) committed = true } @@ -128,7 +173,7 @@ async function dismissPrompt( surface: 'row' }) }, - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts index 8cc9b7e4bc92..288e99521eed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts @@ -7,7 +7,9 @@ import { type AgentSessionCancelResult, type AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' type PendingPromptBody = Extract @@ -91,3 +93,45 @@ export function validatePendingPrompt( } return { ok: true, item, prompt } } + +/** A pending prompt an ended generation raised (`StructuredAgentSessionCurrentWork.isCurrentItem`): + * no live provider holds it, so a Cancel of it reaches no provider. */ +export function isEndedGenerationPrompt( + ctx: Pick, + pending: Extract +): boolean { + return !contextStructuredAgentSessionCurrentWork(ctx).isCurrentItem(pending.item.itemId) +} + +/** Records an ended generation's prompt as cancelled by the caller, in the journal only. */ +export async function dismissEndedGenerationPrompt( + ctx: Pick, + pending: Extract +): Promise> { + const identity = parseAgentJournalItemKey(pending.item.itemId) + if (!identity) { + return { + ok: false, + refusal: refuse( + 'agent_session_operation_invalid', + { reason: 'requestMalformed' }, + `Item id ${pending.item.itemId} is not a well-formed item key.` + ) + } + } + await ctx.journal.appendItem( + identity, + { + ...pending.prompt, + resolution: { + state: 'cancelled', + selectedOptionId: null, + resolvedBy: ctx.resolvedBy, + resolvedAt: ctx.now() + } + }, + // A revision: the prompt keeps the turn it was raised in. + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } + ) + return { ok: true, value: null } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts index c7a9997117e3..194c219cf7dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -44,6 +44,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-ho import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const CALLER = { callerKey: 'client-1' } const CHAT_CLOSED = agentSessionFailureWords(agentSessionFailureFact('chatClosed'), { @@ -130,6 +131,10 @@ beforeEach(async () => { startHost() expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) await host.close(SESSION, 'evict') + // The stop's release wakes the host's retry, which opens the chat and closes it again; the + // tests below write its journal directly, so it must be shut first. + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) }) afterEach(async () => { @@ -359,10 +364,16 @@ describe('settling an earlier child before the next one takes its message', () = { fence: releasedFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await journal.close() + // The process died between that release and its settlement: the next startup settles it, in + // the background, from the evidence the release kept. + await restartHost() + await host.reconcileRestartLeases() + await host.startupSettled() + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) const id = await accept('for the next child') await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) - // The exit was observed, so its receipt ends the turn, and the chat says why it stopped. + // The exit was observed, so its evidence ends the turn, and the chat says why it stopped. const items = conversation()!.journal.snapshot().items expect(items.map((item) => readAgentJournalTurn(item.body)).filter(Boolean)).toContainEqual( expect.objectContaining({ @@ -617,10 +628,13 @@ describe('a quit with a message still queued', () => { surface: 'rejection' }) - /** Read by the next launch, through the same open any reader takes. */ + /** Read after the next launch's startup: its reconcile and settlement, then the restore of the + * chat's tab, whose share holds what the quit left unsent. */ async function afterRelaunch(id: string): Promise { startHost() - await host.revealSession(SESSION) + await host.reconcileRestartLeases() + await host.startupSettled() + await host.restoreReadableSessions([SESSION]) return await submission(id) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts index 978d65d055bf..8f9d2e42ea42 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -85,12 +85,26 @@ export function endProviderChild( ended: Omit & { endedAt?: AgentJournalCursor } +): boolean { + if (!recordProviderChildEnd(session, ended)) { + return false + } + session.child = null + return true +} + +/** Records how the child on record ended, leaving it on record for its own teardown: every reader + * projects its generation as ended from now on (`StructuredAgentSessionCurrentWork`). */ +export function recordProviderChildEnd( + session: ChildBearer, + ended: Omit & { + endedAt?: AgentJournalCursor + } ): boolean { const child = matchingChild(session, ended) if (!child) { return false } - session.child = null session.lastEndedChild = { ...ended, ...(child.startedFor === undefined ? {} : { startedFor: child.startedFor }), @@ -99,6 +113,13 @@ export function endProviderChild( return true } +/** The child's exit is still settling: it ended, and its exit has not let it go (`endChild`). */ +export function providerChildExitSettling( + session: Pick | undefined +): boolean { + return session?.child != null && session.lastEndedChild?.fence === session.child.fence +} + /** The conversation's last start died before it proved itself, and nothing started since. Only a * send retries it: a view or an exit recovery would respawn into the same failure, adding a row. */ export function failedProviderChildStart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts new file mode 100644 index 000000000000..b7197d8379c2 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts @@ -0,0 +1,40 @@ +// A queued card whose automatic send failed for good: the retry gave up on it +// (`StructuredAgentSessionRetry`), or the journal refused it. It reads as not sent at once, from +// memory, while its stored hold is tried once, in the background. Memory only: a restart's +// earlier-process settlement converts the card. + +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { backgroundJournalWrites } from './structured-agent-session-background-writes' + +export class QueuedSendAbandonment { + /** By chat, until a person moves the card, its hold lands or a later attempt sends it. */ + private readonly cards = new Map() + + /** `publish`: re-sends what clients read, so the card shows as not sent and is not named next. */ + constructor( + private readonly publish: (sessionId: string, journal: AgentSessionJournal) => void + ) {} + + card(sessionId: string): string | undefined { + return this.cards.get(sessionId) + } + + clear(sessionId: string): void { + this.cards.delete(sessionId) + } + + /** Marks the card, then tries its stored hold once; should that fail too, the mark still shows. */ + mark(sessionId: string, messageId: string, journal: AgentSessionJournal): void { + this.cards.set(sessionId, messageId) + void backgroundJournalWrites(journal) + .queuedMessages.hold({ messageIds: [messageId], reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED }) + .then(() => { + if (this.cards.get(sessionId) === messageId) { + this.cards.delete(sessionId) + } + }) + .catch(() => undefined) + this.publish(sessionId, journal) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts index 915e781401ab..8e8184bae182 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts @@ -42,6 +42,7 @@ import { QUEUED_RIG_CALLER, type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' let rig: QueuedMessageTestRig @@ -235,7 +236,7 @@ describe('after a restart, nothing sends by itself', () => { await rig.workingSend() const first = await queuedDraft('A') const second = await queuedDraft('B') - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() // The new host opens the chat for its first reader. await published() return { first, second } @@ -395,7 +396,7 @@ describe('after a restart, nothing sends by itself', () => { await eventually(async () => expect(await rig.handoff(first)).toBeDefined()) const cutShort = await rig.handoffId(first) expect((await rig.handoff(first))?.handedOverAt).toBeUndefined() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() release() await published() // The new host refuses the leftover hand-off, and A waits again in its own place. @@ -555,6 +556,13 @@ describe('where the host would refuse the send', () => { if (!journal) { throw new Error('expected the conversation open') } + // The clear's generation end wakes the drain, which would send the card; holding the session's + // lane keeps it waiting while the test reads the gate (`release` lets the drain run). + // The gate exists before the task runs: the clear's end may have queued the chat's + // reconciliation pass ahead of it on the lane. + let release = (): void => undefined + const gate = new Promise((resolve) => (release = resolve)) + const held = rig.host.collaboratorsForTests().serialize(HOST_TEST_SESSION, () => gate) const card = 'left-behind' await journal.queuedMessages.insert({ messageId: card, @@ -566,20 +574,33 @@ describe('where the host would refuse the send', () => { if (!record) { throw new Error('expected the record') } - return { card, journal, record, fence: record.lease.runtimeFence } + const done = async (): Promise => { + release() + await held + } + return { card, journal, record, fence: record.lease.runtimeFence, done } } it('a card queued after clear is sendable with the completed clear record still present', async () => { - const { card, journal, record, fence } = await cardAddedAfterClear() - const gate = structuredQueueSendGate(rig.store, HOST_TEST_SESSION) + const { card, journal, record, fence, done } = await cardAddedAfterClear() + const gate = structuredQueueSendGate( + { store: rig.store, sessions: rig.host.collaboratorsForTests().sessions }, + HOST_TEST_SESSION + ) expect(readQueuePublication(journal, gate).nextQueuedMessageId).toBe(card) const { conversationCommand: _cleared, ...unblocked } = record - const next = readQueuePublication(journal, () => ({ record: unblocked, fence })) + const next = readQueuePublication(journal, () => ({ + record: unblocked, + work: new StructuredAgentSessionCurrentWork(journal, fence) + })) expect(next.nextQueuedMessageId).toBe(card) + await done() + // Released, the drain the clear's end woke sends it. + await vi.waitFor(() => expect(journal.queuedMessages.get(card)?.state).toBe('dispatched')) }) it('a rewind whose outcome is unknown names no next card', async () => { - const { journal, record, fence } = await cardAddedAfterClear() + const { journal, record, fence, done } = await cardAddedAfterClear() const { conversationCommand: _cleared, ...unblocked } = record const rewind = { operationId: hostTestOperationId(), @@ -590,9 +611,13 @@ describe('where the host would refuse the send', () => { retained: [] } const next = (gateRecord: typeof record) => - readQueuePublication(journal, () => ({ record: gateRecord, fence })).nextQueuedMessageId + readQueuePublication(journal, () => ({ + record: gateRecord, + work: new StructuredAgentSessionCurrentWork(journal, fence) + })).nextQueuedMessageId expect(next({ ...unblocked, rewind })).toBeNull() expect(next({ ...unblocked, rewind: { ...rewind, phase: 'provider-succeeded' } })).toBeNull() expect(next({ ...unblocked, rewind: { ...rewind, phase: 'completed' } })).not.toBeNull() + await done() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts index ee7a26af161e..76aa19dddb19 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts @@ -409,7 +409,7 @@ describe('a /compact that waits in line', () => { expect(await rig.sendNow(compactId)).toMatchObject({ ok: true }) } await eventually(() => expect(rig.starts.mock.calls.length).toBeGreaterThan(startsBefore)) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() release() return compactId } @@ -488,7 +488,7 @@ describe('a /compact that waits in line', () => { const startsBefore = rig.starts.mock.calls.length const { id } = compact() await eventually(() => expect(rig.starts.mock.calls.length).toBeGreaterThan(startsBefore)) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() release() // A command in flight is not resumed: the person runs it again. No card, kept or otherwise. expect(await rig.drafts()).toEqual([]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-edit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-edit.test.ts index 9e4280fb54f7..694482b418b0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-edit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-edit.test.ts @@ -75,8 +75,12 @@ function hold( : { ...common, action } ) } +function gate() { + const host = { store: rig.store, sessions: rig.host.collaboratorsForTests().sessions } + return structuredQueueSendGate(host, HOST_TEST_SESSION) +} function publication() { - return readQueuePublication(journal(), structuredQueueSendGate(rig.store, HOST_TEST_SESSION)) + return readQueuePublication(journal(), gate()) } function published(id: string) { return publication().queuedMessages.find((message) => message.messageId === id) @@ -198,7 +202,7 @@ describe('edit leases and automatic delivery', () => { expect( shouldQueueStructuredAgentSessionSend({ journal: journal(), - ...structuredQueueSendGate(rig.store, HOST_TEST_SESSION)() + ...gate()(journal()) }) ).toBe(true) // An explicit Send still overrides, and spends the lease with the card. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts index b9f6e3167920..742bf19a91f0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts @@ -18,6 +18,7 @@ import { hostTestMessage, hostTestOperationId } from './structured-agent-session-host-test-data' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' let rig: QueuedMessageTestRig let host: QueuedMessageTestRig['host'] @@ -87,9 +88,12 @@ describe('the one queue gate', () => { }) // Read in place: the gate runs on every admission and drain step. const snapshot = vi.spyOn(journal, 'snapshot') - expect(structuredQueueHold({ journal, record: store.getRecord(SESSION), fence: 1 })).toBe( - 'prompt' - ) + expect( + structuredQueueHold({ + record: store.getRecord(SESSION), + work: new StructuredAgentSessionCurrentWork(journal, 1) + }) + ).toBe('prompt') expect(snapshot).not.toHaveBeenCalled() snapshot.mockRestore() await journal.appendItem( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts index ef7e36953bd3..d8fc1a359cbc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts @@ -11,11 +11,15 @@ import { agentSessionFailureWords } from '../../../shared/agent-session-failure- import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionQueuePause } from '../../../shared/agent-session-wire' +import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' import type { AgentMessageSource } from '../../../shared/agent-session-message-source' import { agentSessionMessagePayload } from '../../../shared/structured-agent-session-send-mutation' import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' import { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' -import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + StructuredAgentSessionHost, + type StructuredAgentSessionHostDeps +} from './structured-agent-session-host' import { rotateStructuredAgentSessionHostInstanceForTests } from './structured-agent-session-queued-pause' import { HOST_TEST_NOW as NOW, @@ -26,13 +30,17 @@ import { hostTestOperationId, resetHostTestOperationIds } from './structured-agent-session-host-test-data' -import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + closeTestJournalHostDatabase, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { createQueuedRigProvider, type QueuedRigProviderOptions } from './structured-agent-session-queued-message-rig-provider.test-fixture' import { claudeAndCodexDeclared } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' export const QUEUED_RIG_CALLER = { callerKey: 'client-1' } type RigSendOptions = { internal?: true; from?: AgentMessageSource } @@ -49,12 +57,16 @@ export async function createQueuedMessageTestRig( idleSweep?: { idleMs: number; intervalMs: number } /** Teardown records its restart offers, which `restartOffers` lists. */ recoveryCapsule?: true + /** Where the chat runs, when not the default worktree. */ + location?: AgentSessionExecutionLocation + /** How the host probes a recorded owner; absent answers indeterminate. */ + probeOwner?: StructuredAgentSessionHostDeps['probeOwner'] } = {} ) { const root = await mkdtemp(join(tmpdir(), 'orca-queued-messages-')) resetHostTestOperationIds() - const store = await openTestAgentSessionRecordStore(root) - const provider = createQueuedRigProvider(store, options) + let store = await openTestAgentSessionRecordStore(root) + const provider = createQueuedRigProvider({ getRecord: (id) => store.getRecord(id) }, options) const { dispatch, compact, cancelTurn, closeSession } = provider const makeHost = () => new StructuredAgentSessionHost({ @@ -66,13 +78,17 @@ export async function createQueuedMessageTestRig( claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-1', now: () => NOW, + ...(options.probeOwner ? { probeOwner: options.probeOwner } : {}), ...(options.idleSweep ? { idleSweep: options.idleSweep } : {}), ...(options.recoveryCapsule ? { recoveryCapsule: new AgentSessionRecoveryCapsule(root) } : {}) }) let host = makeHost() - expect(await host.attach(QUEUED_RIG_CALLER, hostTestAttachParams(null))).toMatchObject({ - ok: true - }) + // Kept, so a test can re-attach under the same operation: a reconnect to the live child. + const firstAttach = hostTestAttachParams( + null, + options.location ? { location: options.location } : {} + ) + expect(await host.attach(QUEUED_RIG_CALLER, firstAttach)).toMatchObject({ ok: true }) function envelope( fields: Record, @@ -212,20 +228,38 @@ export async function createQueuedMessageTestRig( /** A host-process restart: the app quits (its own teardown runs) and a new host opens the same * state. A quit writes no close's Stop event, so a person's Stop pause survives it. */ - async function restartHostProcess(): Promise { - await quitRestartHostProcess() + async function restartHostProcess(beforeStartup?: () => unknown): Promise { + await quitRestartHostProcess(beforeStartup) } - /** A host process that dies with no close: a new host opens the same state directory. */ - function crashRestartHostProcess(): void { + /** A host process that dies with no close: a new process loads the same state directory, its + * records with it, and runs its startup as the runtime does before any client reaches it: the + * restart reconcile and its settlement, then the restore of the chat's tab, which decides a + * lease latched in recovery; that decision's release wakes the retry once more. */ + async function crashRestartHostProcess(beforeStartup?: () => unknown): Promise { + await crashReloadHostProcess(beforeStartup) + await host.restoreReadableSessions([SESSION]) + await host.startupSettled() + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) + } + + /** The same crash, with only the restart reconcile and its settlement: no tab is restored. + * `beforeStartup` sees the state directory between the two processes. */ + async function crashReloadHostProcess(beforeStartup?: () => unknown): Promise { + // Its connection closes with it, and every temporary trigger a test added with it. + closeTestJournalHostDatabase(root) + await beforeStartup?.() + store = await openTestAgentSessionRecordStore(root) rotateStructuredAgentSessionHostInstanceForTests() host = makeHost() + await host.reconcileRestartLeases() + await host.startupSettled() } /** The app quits — the host's own teardown runs — and a new host opens the same state. */ - async function quitRestartHostProcess(): Promise { + async function quitRestartHostProcess(beforeStartup?: () => unknown): Promise { await host.flushAllStreamedEvents() - crashRestartHostProcess() + await crashRestartHostProcess(beforeStartup) } /** The queue's published pause: null when it sends on its own. */ @@ -276,7 +310,10 @@ export async function createQueuedMessageTestRig( return { root, - store, + firstAttach, + get store() { + return store + }, get host() { return host }, @@ -285,6 +322,7 @@ export async function createQueuedMessageTestRig( closeSession, compact, starts: provider.starts, + answerPrompt: provider.adapter.answerPrompt, holdNextStart: provider.holdNextStart, failNextStart: provider.failNextStart, finishCompact: provider.finishCompact, @@ -303,6 +341,7 @@ export async function createQueuedMessageTestRig( settleRejected, restartHostProcess, crashRestartHostProcess, + crashReloadHostProcess, quitRestartHostProcess, queuePause, restartOffers, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index ac85a5ed8bfb..cc1c623c0d24 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -8,10 +8,9 @@ import { randomUUID } from 'node:crypto' import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' -import { - QUEUED_MESSAGE_PAUSED_SEND_FAILED, - type AgentSessionSendResult, - type AgentSessionWireRefusal +import type { + AgentSessionSendResult, + AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { createStructuredAgentSessionOperationId } from '../../../shared/structured-agent-session-mutation' import { agentSessionSendBodyFingerprint } from '../../../shared/structured-agent-session-send-mutation' @@ -19,12 +18,19 @@ import { queuedSendAnswer } from './structured-agent-session-queued-send-answer' import { structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { queuedMessagesPublishedBytesRefusal } from './structured-agent-session-queued-published-bytes' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' import type { AgentSessionTurnContext } from './structured-agent-session-turns' import { QueuedMessageNotConsumableError } from '../agent-session-journal/queued-message-consume-error' import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' +import { QueuedSendAbandonment } from './structured-agent-session-queued-abandonment' +import { backgroundJournalWrites } from './structured-agent-session-background-writes' import { structuredAgentSessionHostInstance, structuredQueuePauses @@ -39,22 +45,6 @@ export function queuedMessageBodyIsTextOnly(body: AgentJournalMessageItem): bool return body.blocks.every((block) => block.type === 'text') } -/** Walks the reduced items in place: the gate runs on every admission and - * drain step, so it must not render a snapshot of the whole journal. */ -export function pendingPromptExists(journal: Pick): boolean { - let pending = false - journal.visitItems((_itemId, _sequence, body) => { - if ( - !pending && - (body.kind === 'approval' || body.kind === 'question') && - body.resolution.state === 'pending' - ) { - pending = true - } - }) - return pending -} - /** Waiting, not held on its own, and not positioned behind a returned card or a * card the queue's pause holds: the queue never reorders. The admission rule * (§accept) and the drain's selection both read it; only the drain's (`automatic`) stops at a @@ -82,7 +72,8 @@ function oldestActionableQueuedMessage( * drain step: any hold returns early; whatever clears it publishes or * commits, which re-derives. * Send-now: overrides only `working` (plus FIFO order and the stored hold), - * never for a command card; `blocked` and `prompt` refuse readably. + * never for a command card while a turn runs; `blocked` and `prompt` refuse + * readably. * * `blocked` is whatever refuses any send (an uncertain rewind, a cleared source); * the rest are waits. A /compact is a queued message and then a turn, @@ -92,53 +83,43 @@ function oldestActionableQueuedMessage( */ export type StructuredQueueHold = 'blocked' | 'working' | 'prompt' -export function structuredQueueHold(input: { - journal: AgentSessionJournal +/** What the queue's gate reads: the record, and the host's projection of current work + * (`structuredAgentSessionCurrentWork`), so an ended generation's leftovers hold nothing. */ +export type StructuredQueueGateInput = { record: AgentSessionRecord | null - fence: number -}): StructuredQueueHold | null { + work: StructuredAgentSessionCurrentWork + /** The drain's own waits past the gate (`StructuredAgentSessionQueuedMessageDrain.waits`): a + * card it holds back is not one a client is told it sends next. */ + drainWaits?: (messageId: string) => boolean +} + +export function structuredQueueHold(input: StructuredQueueGateInput): StructuredQueueHold | null { // Whatever refuses any send refuses the queue too: an uncertain rewind or a source a // clear superseded. One rule, the immediate path's own. if (structuredAgentSessionSendBlock(input.record)) { return 'blocked' } - const { journal } = input // `prompt` outranks `working`: it is the one wait Send-now may not override, // so a prompt raised mid-turn must not read as merely `working`. - if (pendingPromptExists(journal)) { + if (input.work.hasActionablePrompt()) { return 'prompt' } - if ( - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), - journal.submissions(), - input.fence - ) - ) { - return 'working' - } - return null + return input.work.working() ? 'working' : null } /** The card the drain sends next, or null while anything holds the queue: the drain's own pick * through the one gate, so a client told this reads what the drain acts on. Live facts only; the * backlog is never a gate, so a lone draft drains. */ -export function nextStructuredQueuedMessage(input: { - journal: AgentSessionJournal - record: AgentSessionRecord | null - fence: number -}): QueuedMessageRow | null { +export function nextStructuredQueuedMessage( + input: StructuredQueueGateInput & { journal: AgentSessionJournal } +): QueuedMessageRow | null { const next = oldestActionableQueuedMessage(input.journal, true) - const { journal, fence } = input // The gate's cheap `working` first: publication asks on every streamed frame, and the gate's // prompt check walks the whole fold. - if ( - next === null || - isStructuredAgentSessionMainAgentWorking(journal.activeTurnId(), journal.submissions(), fence) - ) { + if (next === null || input.work.working()) { return null } - return structuredQueueHold(input) === null ? next : null + return structuredQueueHold(input) === null && !input.drainWaits?.(next.messageId) ? next : null } /** @@ -149,11 +130,9 @@ export function nextStructuredQueuedMessage(input: { * turn starting is what lifts the pause — Orca's own queue policy, a stated * deviation from held-head backlog counting. */ -export function shouldQueueStructuredAgentSessionSend(input: { - journal: AgentSessionJournal - record: AgentSessionRecord | null - fence: number -}): boolean { +export function shouldQueueStructuredAgentSessionSend( + input: StructuredQueueGateInput & { journal: AgentSessionJournal } +): boolean { const hold = structuredQueueHold(input) if (hold === 'blocked') { // The immediate path's own refusal (`structuredAgentSessionSendBlock`) @@ -176,7 +155,10 @@ export async function maybeQueueStructuredAgentSessionSend( context: { deps: { store: { getRecord: (sessionId: string) => AgentSessionRecord | null } } }, - ctx: Pick, + ctx: Pick< + AgentSessionTurnContext, + 'sessionId' | 'journal' | 'fence' | 'operationReceipt' | 'currentWork' + >, params: { envelope: { clientOperationId: string } body: AgentJournalMessageItem @@ -209,7 +191,7 @@ export async function maybeQueueStructuredAgentSessionSend( !shouldQueueStructuredAgentSessionSend({ journal: ctx.journal, record: context.deps.store.getRecord(ctx.sessionId), - fence: ctx.fence + work: contextStructuredAgentSessionCurrentWork(ctx) }) ) { return null @@ -257,20 +239,33 @@ export type QueuedMessageDrainDeps = { getRecord: (sessionId: string) => AgentSessionRecord | null serialize: (sessionId: string, task: () => Promise) => Promise conversationFence: (sessionId: string) => number + /** The host's projection of the chat's current work (`structuredAgentSessionCurrentWork`). */ + currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void + /** The host's retry (`StructuredAgentSessionRetry`), read lazily: it is built after the drain. */ + retry: () => Pick + /** Re-sends what clients read (`AgentSessionSubscribers.publish`): an abandon writes nothing. */ + publish: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger } /** * The serialized drain. Woken by every journal commit (turn, submission, prompt, - * command and Stop settlements are all commits), by draft mutations, and by the - * conversation opening; each step re-derives everything and consumes at most one - * draft — the consumed submission then owes work, which gates the next. + * command and Stop settlements are all commits), by draft mutations, by the + * conversation opening, and by a generation ending, which may write nothing + * (`StructuredAgentSessionClientDelivery.publishGenerationEnded`); each step + * re-derives everything and consumes at most one draft — the consumed submission + * then owes work, which gates the next. */ +const LOCKED = { contended: true } as const + export class StructuredAgentSessionQueuedMessageDrain { private readonly scheduled = new Set() private disposed = false + private readonly abandoned = new QueuedSendAbandonment((sessionId, journal) => + this.deps.publish(sessionId, journal) + ) constructor(private readonly deps: QueuedMessageDrainDeps) {} @@ -281,23 +276,41 @@ export class StructuredAgentSessionQueuedMessageDrain { this.disposed = true } + /** Whether this card's automatic send waits for the retry's next round, or was given up on. A + * person's Send now waits on neither. */ + waits = (sessionId: string, messageId: string): boolean => + this.deps.retry().sendWaits(sessionId) || this.abandoned.card(sessionId) === messageId + + /** The card a given-up send left, which the queue shows as not sent. */ + abandonedCard = (sessionId: string): string | undefined => this.abandoned.card(sessionId) + + /** A person moved the card on (Send now, Delete). */ + cardMoved(sessionId: string): void { + this.abandoned.clear(sessionId) + } + + /** The retry gave up on the chat's automatic send. */ + abandon(sessionId: string): void { + const journal = this.deps.sessions.get(sessionId)?.journal + const next = journal ? oldestActionableQueuedMessage(journal, true) : null + if (journal && next) { + this.abandoned.mark(sessionId, next.messageId, journal) + } + } + schedule(sessionId: string): void { const journal = this.disposed ? undefined : this.deps.sessions.get(sessionId)?.journal if (!journal) { return } // Cheap pre-check so token streams do not pay a serialized step per delta. - // Skipping while working is safe: whatever ends the work is itself a commit - // that schedules again, and the step re-reads every gate from the fold. + // Skipping while working is safe: whatever ends the work is a commit or a generation's end, + // and each schedules again; the step re-reads every gate. try { if ( !journal.queuedMessages.settlementOwed() && (oldestActionableQueuedMessage(journal, true) === null || - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), - journal.submissions(), - this.deps.conversationFence(sessionId) - )) + this.deps.currentWork(sessionId)?.working() !== false) ) { return } @@ -312,45 +325,72 @@ export class StructuredAgentSessionQueuedMessageDrain { void this.deps .serialize(sessionId, () => { this.scheduled.delete(sessionId) - return this.step(sessionId) + return this.step(sessionId, false) }) + // Another connection holds the database: the retry sends it in its next round, nothing shown. + .then((outcome) => outcome === 'contended' && this.deps.retry().signal(sessionId, LOCKED)) .catch((error: unknown) => { this.scheduled.delete(sessionId) - this.deps.logger.warn('draining queued messages failed', { - scope: 'queued-drain', - sessionId, - error - }) + this.warn(sessionId, error) }) } - private async step(sessionId: string): Promise { + /** The retry's own send of the chat's next card (`StructuredAgentSessionRetry`), in the chat's + * lane, which its visit already holds: whether another connection's lock refused it, which ends + * the retry's round. */ + sendForRetry(sessionId: string): Promise<'contended' | 'done'> { + return this.step(sessionId, true).catch((error: unknown) => { + this.warn(sessionId, error) + return 'done' as const + }) + } + + private warn(sessionId: string, error: unknown): void { + this.deps.logger.warn('draining queued messages failed', { + scope: 'queued-drain', + sessionId, + error + }) + } + + /** `forRetry`: the retry's own attempt, which a send it owes does not hold back. */ + private async step(sessionId: string, forRetry: boolean): Promise<'contended' | 'done'> { const session = this.deps.sessions.get(sessionId) if (this.disposed || !session) { - return + return 'done' } const journal = session.journal + // The automatic send is bookkeeping no person waits on: it writes only through this handle. + const writes = backgroundJournalWrites(journal) if (journal.queuedMessages.settlementOwed() || journal.queuedMessages.deliveredByEchoOwed()) { // A live per-row hook was skipped; heal now, before a draft sends, rather than at reopen. - await journal.queuedMessages.settleOwed().catch((error: unknown) => { + try { + await writes.queuedMessages.settleOwed() + } catch (error) { + if (isSqliteContentionFailure(error)) { + return 'contended' + } this.deps.logger.warn('settling owed queued-message bookkeeping failed', { scope: 'queued-settle-owed', sessionId, error }) - }) + } } const fence = this.deps.conversationFence(sessionId) // Whatever clears a hold publishes or commits, which re-derives this step. const record = this.deps.getRecord(sessionId) - const next = nextStructuredQueuedMessage({ journal, record, fence }) + const work = this.deps.currentWork(sessionId) + // The retry's own attempt sends what the drain alone waits on (`waits`). + const drainWaits = (messageId: string) => !forRetry && this.waits(sessionId, messageId) + const next = work ? nextStructuredQueuedMessage({ journal, record, work, drainWaits }) : null if (this.disposed || !next) { - return + return 'done' } // Always a fresh id: the submission names its draft by `queuedMessageId`, never by id equality. const submissionId = createStructuredAgentSessionOperationId(randomUUID) try { - await journal.appendSubmission( + await writes.appendSubmission( { clientMessageId: submissionId, // The queue's own automatic send, never kept as a card by a restart or a close. @@ -371,17 +411,18 @@ export class StructuredAgentSessionQueuedMessageDrain { } catch (error) { if (error instanceof QueuedMessageNotConsumableError) { // Lost a race with a Send-now, a Delete or a Stop; their transition stands. - return + return 'done' + } + if (isSqliteContentionFailure(error)) { + return 'contended' } - // Pre-consume failure: the draft stays waiting, held with the marker on - // the card (a stored fact, so it survives eviction and restart). The - // hold's own commit notification publishes it. An explicit Send retries; - // no automatic retry loop. - await journal.queuedMessages - .hold({ messageIds: [next.messageId], reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED }) - .catch(() => {}) + // A refusal: the draft stays waiting, shown as not sent and held (a stored fact once its hold + // lands, so it survives eviction and restart). An explicit Send retries. + this.abandoned.mark(sessionId, next.messageId, journal) throw error } + this.abandoned.clear(sessionId) this.deps.wakeDelivery(sessionId) + return 'done' } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts index a3fee6e2a339..f084a1193a8d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts @@ -4,6 +4,7 @@ // from the operation ledger. No mutation returns draft text: // the published list is the one authority a client renders. +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { agentSessionOperationKey } from '../../../shared/agent-session-operation-ledger' import type { @@ -131,7 +132,8 @@ export function sendQueuedStructuredAgentMessage( // The one queue gate; Send-now's override set is exactly `working` (plus // FIFO order and the stored hold, which the consume below clears). const record = context.deps.store.getRecord(ctx.sessionId) - const hold = structuredQueueHold({ journal: ctx.journal, record, fence: ctx.fence }) + const work = contextStructuredAgentSessionCurrentWork(ctx) + const hold = structuredQueueHold({ record, work }) if (hold === 'blocked') { return structuredAgentSessionSendBlock(record) ?? invalid('This conversation cannot send.') } @@ -154,7 +156,7 @@ export function sendQueuedStructuredAgentMessage( } // A command never steers: handed over mid-turn it would only be refused. Clients offer its // Send only while the agent is idle; this answers an older one that offers it mid-turn. - if (hold === 'working' && row.body.command) { + if (row.body.command && work.working()) { return invalid("A command can't be sent while the agent is working.") } const submissionId = operationId diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts index eeaafbbaff8d..e844aff6d08b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts @@ -38,11 +38,16 @@ async function queuedDraft(text: string): Promise { return queued.value.queued.messageId } -/** The chat reopens on a database a newer Orca stamped. The host's database object is the one - * the rig built, so its open-time verdict is set in place; every store write checks that. */ +/** The database a newer Orca stamped. The host's database object is the one the rig built, so + * its open-time verdict is set in place; every store write checks that. */ +function stampNewerOrca(): void { + Object.defineProperty(openTestJournalHostDatabase(rig.root), 'readOnly', { value: true }) +} + +/** The chat reopens on a database a newer Orca stamped. */ async function reopenOnNewerOrcaDatabase(close: () => Promise): Promise { await close() - Object.defineProperty(openTestJournalHostDatabase(rig.root), 'readOnly', { value: true }) + stampNewerOrca() } /** The history read a newer Orca's chat gets: refused, as every reader's is. */ @@ -80,7 +85,8 @@ describe("a newer Orca's journal", () => { it('refuses Resume of the queue a restart paused with the update words', async () => { await twoCardsBehindWork() - await reopenOnNewerOrcaDatabase(() => rig.restartHostProcess()) + // Stamped before the new process starts, as a newer Orca's run leaves it. + await rig.restartHostProcess(stampNewerOrca) await expectHistoryRefused() expect(await rig.resume()).toMatchObject(REFUSED_BY_NEWER_ORCA) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts index 75ff328b012e..6201108de1c8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts @@ -105,7 +105,7 @@ describe("a Stop's queue pause", () => { const inFlight = await handedOverUserSend('sent before the restart') // Derived from the journal, not remembered: a restart forgets nothing it needs. The process // dies with no close, as a quit writes no Stop event to end the pause either. - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() // Still derived, though after a restart no pause is shown. expect(await rig.queuePause()).toBeNull() expect(derivedPauses()).toEqual(['stopped', 'restarted']) @@ -447,7 +447,7 @@ describe('Resume', () => { it("of a Stop from before a restart, which no client offers there, also lifts the restart's pause", async () => { const draftId = await stoppedDraft() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.queuePause()).toBeNull() expect(derivedPauses()).toEqual(['stopped', 'restarted']) expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts index 9dd963bb43bf..7a794627ef1f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts @@ -7,6 +7,7 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store import type { DerivedQueuePause } from '../agent-session-journal/queued-message-pause' import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' /** A per-process id, minted once per host process like the runtime's own * `runtimeId` (`orca-runtime-runtime-id.ts`), stamped on the cards this process writes or hands @@ -31,14 +32,16 @@ export function structuredQueuePauses(journal: PauseJournal): DerivedQueuePause[ } /** The mark of a chat that stopped running with cards waiting — Orca quit or crashed, or the chat - * was closed — so they wait for its next turn (`queued-message-pause.ts`). Every open marks, and - * so does a person's close: the idle sweep never closes a chat with cards waiting, so its + * was closed — so they wait for its next turn (`queued-message-pause.ts`). Startup marks each + * chat with cards the earlier process left, and so does a person's close: the idle sweep never closes a chat with cards waiting, so its * eviction never reopens one. `since`: where the chat stopped, for a mark written after a send * that came later, which must still lift it. Bookkeeping, so a failure is reported and never - * thrown; the pause then starts where the mark would have gone, holding no less. */ + * thrown; the pause then starts where the mark would have gone, holding no less. One written + * through the retry's background handle that met another connection's lock throws, ending the + * retry's round. */ export async function markStructuredQueueReopen( sessionId: string, - journal: Pick, + journal: Pick & { background?: true }, fence: number, logger: StructuredAgentSessionLogger, since?: number @@ -46,6 +49,9 @@ export async function markStructuredQueueReopen( try { await journal.markQueueReopen(fence, since) } catch (error) { + if (journal.background && isSqliteContentionFailure(error)) { + throw error + } logger.warn('marking a reopened queue failed', { scope: 'queue-reopen-mark', sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index d48d9bf27c39..9cf9a0cf9317 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -15,10 +15,14 @@ import { resumableQueuePause } from '../agent-session-journal/queued-message-pause' import { structuredQueuePauses } from './structured-agent-session-queued-pause' -import { nextStructuredQueuedMessage } from './structured-agent-session-queued-messages' -import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + nextStructuredQueuedMessage, + type StructuredQueueGateInput +} from './structured-agent-session-queued-messages' +import { + heldStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkHost +} from './structured-agent-session-host-current-work' export type QueuePublication = { queuedMessages: AgentSessionQueuedMessage[] @@ -28,28 +32,57 @@ export type QueuePublication = { nextQueuedMessageId: string | null } -/** What the drain's gate reads beyond the journal; resolved per read. */ -export type QueueSendGate = () => { record: AgentSessionRecord | null; fence: number } +/** What the drain's gate reads beyond the journal; resolved per read. `abandoned`: the card whose + * automatic send was given up on, shown as not sent. */ +export type QueueSendGate = ( + journal: AgentSessionJournal +) => StructuredQueueGateInput & { abandoned?: string } + +/** What the drain itself holds back (`StructuredAgentSessionQueuedMessageDrain`). */ +export type QueuedDrainHolds = { + waits: (sessionId: string, messageId: string) => boolean + abandonedCard: (sessionId: string) => string | undefined +} +/** `drain`: the drain's own holds, which a publication reads so a client is never told a card + * sends next while the drain holds it back (the chat would read Working with nothing running). */ export function structuredQueueSendGate( - store: Pick, - sessionId: string + host: StructuredAgentSessionCurrentWorkHost, + sessionId: string, + drain?: QueuedDrainHolds ): QueueSendGate { - return () => ({ - record: store.getRecord(sessionId), - fence: structuredAgentSessionConversationFence(store, sessionId) - }) + return (journal) => { + const record = host.store.getRecord(sessionId) + const abandoned = drain?.abandonedCard(sessionId) + return { + record, + work: heldStructuredAgentSessionCurrentWork( + host.store, + sessionId, + journal, + host.sessions.get(sessionId) + ), + ...(drain ? { drainWaits: (messageId: string) => drain.waits(sessionId, messageId) } : {}), + ...(abandoned ? { abandoned } : {}) + } + } } /** Waiting and returned rows only. `paused` is a per-card hold (a failed conversion); a person's * Stop pauses the queue, published once beside it. */ -function computePublishedQueuedMessages(journal: AgentSessionJournal): AgentSessionQueuedMessage[] { +function computePublishedQueuedMessages( + journal: AgentSessionJournal, + abandoned: string | undefined +): AgentSessionQueuedMessage[] { const published: AgentSessionQueuedMessage[] = [] for (const row of journal.queuedMessages.list()) { if (row.state !== 'waiting' && row.state !== 'returned') { continue } - const held = row.state === 'waiting' && row.holdReason !== null + // A send given up on reads as main's failed send even when its stored hold did not land. + const holdReason = + row.messageId === abandoned ? QUEUED_MESSAGE_PAUSED_SEND_FAILED : row.holdReason + const held = row.state === 'waiting' && holdReason !== null published.push({ messageId: row.messageId, position: row.position, @@ -57,7 +90,7 @@ function computePublishedQueuedMessages(journal: AgentSessionJournal): AgentSess state: row.state, ...(held ? { paused: true as const } : {}), // The stored reason is a typed marker; an unknown one reads as a plain hold. - ...(held && isPublishedPausedReason(row.holdReason) ? { pausedReason: row.holdReason } : {}), + ...(held && isPublishedPausedReason(holdReason) ? { pausedReason: holdReason } : {}), ...(row.state === 'returned' ? { returnedReason: row.returnedReason } : {}), ...(row.state === 'returned' && row.returnedRejection ? { returnedRejection: row.returnedRejection } @@ -81,13 +114,16 @@ type ListMemo = { key: string; serialized: string; list: AgentSessionQueuedMessa const listMemos = new WeakMap() const publications = new WeakMap() -function readPublishedQueuedMessages(journal: AgentSessionJournal): AgentSessionQueuedMessage[] { - const key = String(journal.queuedMessages.revision()) +function readPublishedQueuedMessages( + journal: AgentSessionJournal, + abandoned: string | undefined +): AgentSessionQueuedMessage[] { + const key = `${journal.queuedMessages.revision()}:${abandoned ?? ''}` const memo = listMemos.get(journal) if (memo && memo.key === key) { return memo.list } - const list = computePublishedQueuedMessages(journal) + const list = computePublishedQueuedMessages(journal, abandoned) // Belt for the identity dedup: equal recomputed content keeps the previous reference. const serialized = JSON.stringify(list) if (memo && memo.serialized === serialized) { @@ -111,7 +147,8 @@ export function readQueuePublication( journal: AgentSessionJournal, gate: QueueSendGate ): QueuePublication { - const queuedMessages = readPublishedQueuedMessages(journal) + const input = gate(journal) + const queuedMessages = readPublishedQueuedMessages(journal, input.abandoned) // Read per emit: the pause also turns on submissions (a turn starting). Only a person's Stop is // shown, and only over a card Resume would send, so its header never offers to send nothing; // deleting a blocking returned card shows it again. After a /clear, a restart or a close nothing @@ -135,7 +172,7 @@ export function readQueuePublication( resumable && !queuePauseLiftOnItsWay(resumable, journal.submissions()) ? resumable : null // The test narrows the type. const queuePause = pause?.reason === 'stopped' ? { reason: pause.reason } : null - const nextQueuedMessageId = nextStructuredQueuedMessage({ journal, ...gate() })?.messageId ?? null + const nextQueuedMessageId = nextStructuredQueuedMessage({ journal, ...input })?.messageId ?? null const previous = publications.get(journal) if ( previous && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts index e6ecad009d4c..fc03a0d0716f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts @@ -177,7 +177,7 @@ describe("Stop's event", () => { await eventually(async () => expect((await rig.handoff(sentId))?.handedOverAt).toBeDefined()) await rig.stop() // The process dies with no close; a new host opens the same state directory. - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect((await rig.handoff(sentId))?.dispatchState).toBe('unknown') // After a restart no row shows, the Stop's included, and nothing sends. await expectHeld(null, waiting) @@ -198,15 +198,12 @@ describe("a restart's hold over a card queued after a Stop", () => { const mail = await mailTurn() const typed = await queuedDraft('typed during the mail turn') // The process dies with no close: a quit writes no Stop event, so only a turn ends the pauses. - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.settleAccepted(mail, 'mail') - // The new host opens the conversation for its first reader. The card came after the Stop, so - // only the restart holds it, and a restart's hold is never published. + // The mail turn, sent after the Stop, was accepted, which lifts the Stop's pause. The card came + // after the Stop anyway, so only the restart holds it, and a restart's hold is never published. await rig.queuePause() - expect(structuredQueuePauses(journal()).map((pause) => pause.reason)).toEqual([ - 'stopped', - 'restarted' - ]) + expect(structuredQueuePauses(journal()).map((pause) => pause.reason)).toEqual(['restarted']) await expectHeld(null, typed) }) @@ -219,7 +216,7 @@ describe("a restart's hold over a card queued after a Stop", () => { expect((await rig.handoff(correction))?.handedOverAt).toBeDefined() ) const typed = await queuedDraft('typed during that send') - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.settleAccepted(await rig.handoffId(correction), 'correction') await rig.queuePause() expect(structuredQueuePauses(journal()).map((pause) => pause.reason)).toEqual(['restarted']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts index fea528147494..75f533340284 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts @@ -14,6 +14,7 @@ import { } from '../agent-session-journal/queued-message-table' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' import { isMainAgentWorking } from './structured-agent-session-turns-cancel' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { structuredAgentSessionStopNamesTurnNotLive, structuredAgentSessionStoppedTurnId @@ -45,7 +46,11 @@ export async function runRecordedStop( const skipped = (error: unknown): void => report(ctx, 'event row', error) return stop(() => { try { - const turnId = structuredAgentSessionStoppedTurnId(ctx.journal, event.turnId) ?? undefined + const turnId = + structuredAgentSessionStoppedTurnId( + contextStructuredAgentSessionCurrentWork(ctx), + event.turnId + ) ?? undefined return ctx.journal .appendStopEvent({ ...event, ...(turnId ? { turnId } : {}) }, ctx.fence) .then(() => undefined, skipped) @@ -65,10 +70,10 @@ export async function runRecordedStop( * already over, as a late Stop from a phone can. */ export function stopReachesUnrecordedWork( - ctx: Pick, + ctx: Pick, namedTurnId: string | undefined ): 'unrecorded' | 'repeat' | 'late' { - const live = ctx.journal.activeTurnId() + const live = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() // No turn published yet while the agent works: the named one may still be opening. if ( structuredAgentSessionStopNamesTurnNotLive(namedTurnId, live) && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index fc8078ad649f..f536a7dc4399 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -8,6 +8,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { StructuredAgentSessionQueuedMessageDrain } from './structured-agent-session-queued-messages' import { deleteQueuedStructuredAgentMessage, @@ -36,7 +37,11 @@ export function wireStructuredAgentSessionQueuedMessages( serialize: (sessionId, task) => context().serialize(sessionId, task), conversationFence: (sessionId) => structuredAgentSessionConversationFence(context().deps.store, sessionId), + currentWork: (sessionId) => + hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), + retry: () => context().retry, + publish: (sessionId, journal) => context().publish(sessionId, journal), // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) }) @@ -72,18 +77,28 @@ export function wireStructuredAgentSessionQueuedMessages( ) => holdQueuedStructuredAgentMessageEdit(context(), caller, params), /** Every journal publish: turn, submission, prompt, command and Stop * settlements are all commits, and each re-derives the drain's gates. */ - onJournalActivity: (sessionId: string) => { - sessions.touch(sessionId) + /** `activity`: false for a generation's end, which wakes the drain but is no activity. */ + onJournalActivity: (sessionId: string, activity = true) => { + if (activity) { + sessions.touch(sessionId) + } drain.schedule(sessionId) }, queuedMessageSend: ( caller: StructuredAgentSessionCaller, params: Parameters[2] - ) => sendQueuedStructuredAgentMessage(context(), caller, params), + ) => { + // A person's Send now moves the card on: a send the drain was retrying starts fresh. + drain.cardMoved(params.envelope.sessionId) + return sendQueuedStructuredAgentMessage(context(), caller, params) + }, queuedMessageDelete: ( caller: StructuredAgentSessionCaller, params: Parameters[2] - ) => deleteQueuedStructuredAgentMessage(context(), caller, params), + ) => { + drain.cardMoved(params.envelope.sessionId) + return deleteQueuedStructuredAgentMessage(context(), caller, params) + }, queuedMessagesResume: ( caller: StructuredAgentSessionCaller, params: Parameters[2] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts index 87c03c071ea9..b6c8444a28bc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-readable-restorer.test.ts @@ -35,11 +35,7 @@ describe('StructuredAgentSessionReadableRestorer', () => { ) const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: { - store: { - getRecord: () => null, - listRecords: () => records, - replacedRuntime: () => undefined - }, + store: { getRecord: () => null, listRecords: () => records }, journalDatabase: openTestJournalHostDatabase(stateDirectory), logger: recordingStructuredAgentSessionLogger().logger }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts index 2ea74bd77499..b29e3dab59ad 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts @@ -12,10 +12,8 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-hand import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from './structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import { captureUnfinishedStructuredAgentSessionWork } from './structured-agent-session-unfinished-work' const SESSION = 'session-reasoning-sweep' @@ -72,7 +70,7 @@ describe('an open reasoning row a dead generation left', () => { it('ends, with no claimed time, when the generation is settled at its exit', async () => { await seedOpenReasoning('running') await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, @@ -81,7 +79,7 @@ describe('an open reasoning row a dead generation left', () => { verdict: { state: 'interrupted', completedAt: 1_500 }, showUnexpectedExitOutcome: false }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) expect(reasoningBody()).toEqual({ kind: 'message', role: 'reasoning', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts new file mode 100644 index 000000000000..da8d0e5986d0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts @@ -0,0 +1,16 @@ +// The one backoff the host's retry of background bookkeeping uses: from 1 s to 2 s, given up after +// about 3 minutes of rounds failing in a row, so nothing is owed forever. Short: a background write +// fails at once, so a round is cheap, waiting long only delays work a lifted lock already allows, +// and nothing announces the lift. + +export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 2_000 } as const +/** The time since an episode's first refused round after which its retry gives up. */ +export const RECONCILIATION_GIVE_UP_MS = 180_000 + +/** The wait before the next try, after `failures` consecutive failed ones. */ +export function reconciliationBackoffDelay(failures: number): number { + return Math.min( + RECONCILIATION_BACKOFF_MS.max, + RECONCILIATION_BACKOFF_MS.first * 2 ** (failures - 1) + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts new file mode 100644 index 000000000000..ef3df020d479 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts @@ -0,0 +1,77 @@ +// The host retry's own read of a chat nobody has open: replayed outside the chat's lane, and never +// indexed as the chat's conversation, so what its pass writes reaches no reader and no status +// surface. A reader that opens the chat replays it fresh; the retry then uses theirs. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + readJournalRowsAfter, + readJournalSessionEpoch +} from '../agent-session-journal/journal-row-table' +import { isJournalOpenFailurePermanent } from '../agent-session-journal/journal-open-failure' +import type { StructuredAgentSessionConversationOpenDeps } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' + +export type StructuredAgentSessionReconciliationLoad = + | { kind: 'loaded'; session: StructuredAgentSessionHostSession } + /** Opened by a reader meanwhile, or quitting: nothing read. */ + | { kind: 'skipped' } + /** Nothing owed: no journal (nothing was ever written), a named epoch holding no row (founded + * afresh in memory, so nothing durable is there to settle), or one damaged or written by a + * newer Orca, which no retry reads (reported once here). */ + | { kind: 'nothing' } + +/** Replays a closed chat's journal, for a caller in a background slot. Throws only a failure that + * can clear. `skip`: read once the turn comes, so a reader's open meanwhile wins. */ +export async function loadStructuredAgentSessionForReconciliation( + deps: StructuredAgentSessionConversationOpenDeps, + sessionId: string, + skip: () => boolean +): Promise { + // A replay is synchronous SQLite: a macrotask before each keeps a send or a read from waiting + // behind a whole scan's worth of them. + await yieldToEventLoop() + if (skip()) { + return { kind: 'skipped' } + } + try { + const opened = await restoreStructuredAgentSessionRead(deps, sessionId) + if (!opened) { + return { kind: 'nothing' } + } + const { journal } = opened.session + // The read founded a fresh epoch because the named one holds no row: nothing durable to settle, + // and a handle on an epoch the store never names would read as stale on every attempt. + if (readJournalSessionEpoch(deps.journalDatabase.db, sessionId) !== journal.cursor().epoch) { + await journal.close() + return { kind: 'nothing' } + } + return { kind: 'loaded', session: opened.session } + } catch (error) { + if (!isJournalOpenFailurePermanent(error)) { + throw error + } + deps.logger.warn("a chat's history cannot be loaded, so nothing is settled", { + scope: 'reconciliation', + sessionId, + error + }) + return { kind: 'nothing' } + } +} + +/** Whether a handle still holds the journal's newest row: another handle (a reader that opened, + * wrote and closed the chat meanwhile) leaves it stale, and a stale handle must not write. */ +export function structuredAgentSessionJournalIsCurrent( + deps: StructuredAgentSessionConversationOpenDeps, + sessionId: string, + journal: Pick +): boolean { + const { db } = deps.journalDatabase + const { epoch, sequence } = journal.cursor() + return ( + readJournalSessionEpoch(db, sessionId) === epoch && + readJournalRowsAfter(db, sessionId, epoch, sequence, 1).length === 0 + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts new file mode 100644 index 000000000000..85da9b49c136 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts @@ -0,0 +1,179 @@ +// What the retry keeps for a chat whose lease recovers: the proofs it still holds, never an exit's +// account (it judges only its own generation, which a later one may have replaced), and nothing at +// all once the chat's record is gone. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' +import { + openScanHost, + SCAN_NOW, + scanIdle as idle, + scanRecord, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-bbbbbbb1' +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-parked-debts-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +/** Generation 13 left a running turn; generation 14 outlived the earlier process, holding a send + * it was handed and never answered, so its lease is latched in recovery. */ +async function recoveringAt14() { + const base = scanRecord(CHAT, false) + await seedTestAgentSessionRecordStore(root, { + records: [ + { + ...base, + providerHandleChain: base.providerHandleChain.map((link) => ({ + ...link, + mintedAtFence: 14 + })), + lease: { ...base.lease, runtimeFence: 14 } + } + ] + }) + await seedScanJournal(root, CHAT, { handedOverAt: 14 }) + store = await openTestAgentSessionRecordStore(root) + let alive = true + const probeOwner = async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + host = openScanHost(root, store, { + probeOwner, + stopOwnerProcess: () => { + alive = false + } + }) + await host.reconcileRestartLeases() + await host.startupSettled() + await idle(host, [CHAT]) + expect(store.getRecord(CHAT)?.lease).toMatchObject({ + handoffStage: 'recovering', + runtimeFence: 14 + }) + return host +} + +const proof13 = { + kind: 'exit-observed' as const, + detail: 'generation 13 exited', + observedAt: SCAN_NOW - 1_000, + ownerFence: 13 +} + +/** Generation 13's exit account: it died while still starting. */ +const exit13: StructuredAgentSessionExitSettlement = { + ownerFence: 13, + settlementId: 'exit-13', + verdict: { state: 'interrupted', completedAt: SCAN_NOW - 1_000 }, + pendingSubmissionReason: 'provider_exited', + exitFailure: { kind: 'providerStartFailed' }, + exitedDuringStartup: { generation: 'generation-13' } +} + +function attach(current: StructuredAgentSessionHost) { + const params = attachParamsForRecord(store.getRecord(CHAT)!, { + clientOperationId: 'attach-1', + expectedRuntimeFence: 14 + }) + return current.attach({ callerKey: 'client-1' }, params).catch(() => undefined) +} + +describe('a chat whose lease recovers', () => { + it("parks generation 13's proof, never its exit account, which would judge generation 14's sends", async () => { + const current = await recoveringAt14() + const { reconciliation } = current.collaboratorsForTests() + + // Generation 13's exit could not be settled; the lease recovers, so the retry retires. + reconciliation.signal(CHAT, { evidence: proof13, exit: exit13 }) + await idle(current, [CHAT]) + expect(reconciliation['owed'].get(CHAT)).toMatchObject({ + due: false, + debts: { evidence: [proof13] } + }) + expect(reconciliation['owed'].get(CHAT)?.debts.exit).toBeUndefined() + + // Generation 14 ends with no exit account of its own: an attach decides its recovery. + await attach(current) + await idle(current, [CHAT]) + + // The proof judged what generation 13 left, and was taken. + expect(await turnState(current, CHAT)).toBe('interrupted') + expect(reconciliation['owed'].has(CHAT)).toBe(false) + // Generation 14's send ran on a child that proved its start: in doubt, never "did not start". + const { submissions } = await current.journalSnapshot(CHAT) + expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-handed`)).toMatchObject({ + dispatchState: 'unknown' + }) + }) +}) + +describe('a chat waiting out a backoff with an exit account', () => { + it("drops it once a later generation acquired, so generation 14's send stays in doubt", async () => { + const current = await recoveringAt14() + const { reconciliation } = current.collaboratorsForTests() + // The pass fails once (storage busy): the retry backs off still holding generation 13's. + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune').mockRejectedValueOnce( + Object.assign(new Error('database is locked'), { errcode: 5 }) + ) + reconciliation.signal(CHAT, { evidence: proof13, exit: exit13 }) + await reconciliation.attempted(CHAT) + + // Generation 14's recovery is decided meanwhile, with no exit account of its own. + await attach(current) + await idle(current, [CHAT]) + + const { submissions } = await current.journalSnapshot(CHAT) + expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-handed`)).toMatchObject({ + dispatchState: 'unknown' + }) + }) +}) + +describe('a chat whose record is gone', () => { + it('leaves nothing behind in the reconciliation: no parked proof, no first-open cursor', async () => { + const current = await recoveringAt14() + const { reconciliation } = current.collaboratorsForTests() + reconciliation.signal(CHAT, { evidence: proof13 }) + await idle(current, [CHAT]) + // A replaced runtime's turn is settled under the latch; the proof still waits for its decision. + expect(await turnState(current, CHAT)).toBe('interrupted') + expect(reconciliation['owed'].get(CHAT)?.debts.evidence).toEqual([proof13]) + expect(reconciliation['firstOpened'].has(CHAT)).toBe(true) + + const getRecord = store.getRecord.bind(store) + vi.spyOn(store, 'getRecord').mockImplementation((id) => (id === CHAT ? null : getRecord(id))) + reconciliation.signal(CHAT) + await idle(current, [CHAT]) + + expect(reconciliation['owed'].has(CHAT)).toBe(false) + expect(reconciliation['firstOpened'].has(CHAT)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts new file mode 100644 index 000000000000..0388700c1d3a --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -0,0 +1,236 @@ +// One visit of the host's retry to a chat, for a caller inside the chat's action lane: the short +// recheck and the writes, all through the background handles (`structured-agent-session- +// background-writes.ts`), so another connection's lock ends the round instead of waiting. Everything owed is re-derived here from the lease, this host's +// sight of an exit and the journal, so a pass that finds nothing owed writes nothing; only what +// cannot be derived rides in from the signals (an exit's own account, a proof the lease no longer +// holds). A lease latched in recovery still gets (b), which touches no process and no lease, as +// every open did on main, and (c) for a generation a runtime this one replaced held, which is over +// whatever the recovery decides; (a), and (c) otherwise, wait for its decision, whose release +// signals again. +// +// The steps stay apart, so one never retires another's debt: +// (a) lease-release repair: an exit this host observed whose release write failed; +// (b) what an earlier host process left: the cards' repair and prune, and the unsent sends it +// accepted kept as cards; then, best effort, the queue's reopen mark and a rewind left in +// doubt (neither is owed: the pause is derived from where the handle opened, and a rewind +// only its provider can prove waits for an acquisition); +// (c) item settlement (`settleStructuredAgentSessionLeftovers`), one planned batch. +// An empty item plan says nothing of (a) or (b). + +import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import { + backgroundJournalWrites, + backgroundStoreWrites, + type BackgroundJournalWrites, + type BackgroundStoreWrites +} from './structured-agent-session-background-writes' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import { queuedMessageReopenMarkStart } from '../agent-session-journal/queued-message-reopen-floor' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + releaseLeaseOfEndedStructuredAgentSessionChild, + structuredAgentSessionEndedChildHoldsLease +} from './structured-agent-session-child-close' +import { + settleStructuredAgentSessionLeftovers, + type StructuredAgentSessionExitSettlement +} from './structured-agent-session-leftover-settlement' +import { structuredAgentSessionReplacedRuntimeEnded } from './structured-agent-session-current-work' +import { + markStructuredQueueReopen, + structuredAgentSessionHostInstance +} from './structured-agent-session-queued-pause' +import { + recoverStructuredRewind, + structuredRewindNeedsProvider +} from './structured-rewind-recovery' + +export type StructuredAgentSessionReconciliationPassContext = Pick< + StructuredAgentSessionLifetimeContext, + 'deps' | 'sessions' | 'now' +> + +/** What the signals left that the journal and the lease cannot tell. */ +export type StructuredAgentSessionReconciliationDebts = { + exit?: StructuredAgentSessionExitSettlement + /** Each ended generation's proof, as its end carried it: a reservation clears the lease's, and a + * later release replaces it with its own. */ + evidence?: AgentSessionDeathEvidence[] +} + +export type StructuredAgentSessionReconciliationPass = { + /** A step could not finish: the retry visits again after its backoff. */ + failed: unknown[] + /** Rows were written, so the next pass verifies nothing is left. */ + wrote: boolean + /** The lease is latched in recovery: what ended waits for its decision's signal. */ + recovering: boolean +} + +/** `session`: the chat's open conversation, or the retry's own read of a closed one. + * `processOpened`: where this host first opened the chat's journal; a send at or before it was an + * earlier process's. A later handle's own open cannot tell: this process may have accepted sends + * through an earlier handle, which its close or its delivery loop still owns. */ +export async function runStructuredAgentSessionReconciliationPass( + context: StructuredAgentSessionReconciliationPassContext, + sessionId: string, + session: Pick, + debts: StructuredAgentSessionReconciliationDebts, + processOpened: AgentJournalCursor +): Promise { + const lease = context.deps.store.getRecord(sessionId)?.lease + const recovering = lease?.handoffStage === 'recovering' + const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false, recovering } + // Every write a pass makes is bookkeeping no person waits on: only these handles write. + const writes = { + store: backgroundStoreWrites(context.deps.store), + journal: backgroundJournalWrites(session.journal) + } + // An exit's account judges only its own generation: once a later one acquired, it is spent. + if (debts.exit && (!lease || lease.runtimeFence > debts.exit.ownerFence + 1)) { + delete debts.exit + } + if ( + !recovering && + structuredAgentSessionEndedChildHoldsLease(context, sessionId) && + (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId, writes.store)) + ) { + pass.failed.push(new Error('agent_session_exit_release_owed')) + } + await settleEarlierProcess(context, sessionId, session.journal, writes, processOpened, pass) + if (pass.failed.some(isSqliteContentionFailure)) { + return pass + } + const settle = (proof?: AgentSessionDeathEvidence & { ownerFence: number }) => + settleStructuredAgentSessionLeftovers({ + store: context.deps.store, + sessionId, + journal: session.journal, + writes: writes.journal, + ...(debts.exit && !recovering ? { exit: debts.exit } : {}), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), + ...(proof ? { proof } : {}) + }) + if (recovering) { + // What else ended is not known until its recovery is decided; the debts wait for that signal. + const replaced = context.deps.store.replacedRuntime(sessionId) + if (lease && structuredAgentSessionReplacedRuntimeEnded(lease, replaced)) { + const settled = await settle() + pass.failed.push(...(settled.ok ? [] : [settled.error])) + pass.wrote ||= settled.ok && settled.planned > 0 + } + return pass + } + // Each proof the lease no longer holds judges what its own generation left, oldest first. + const leaseEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null + for (const proof of heldProofs(debts, leaseEvidence)) { + const settled = await settle(proof) + if (!settled.ok) { + pass.failed.push(settled.error) + return pass + } + pass.wrote ||= settled.planned > 0 + } + const settled = await settle() + if (!settled.ok) { + pass.failed.push(settled.error) + return pass + } + // Kept until everything they judge is settled: a later reservation clears the lease's copy. + delete debts.evidence + // Written, or overtaken by a later generation: either way the exit's account is spent. + delete debts.exit + pass.wrote ||= settled.planned > 0 + return pass +} + +function heldProofs( + debts: StructuredAgentSessionReconciliationDebts, + leaseEvidence: AgentSessionDeathEvidence | null +): (AgentSessionDeathEvidence & { ownerFence: number })[] { + return (debts.evidence ?? []) + .filter( + (evidence): evidence is AgentSessionDeathEvidence & { ownerFence: number } => + evidence.ownerFence !== undefined && + !( + leaseEvidence?.ownerFence === evidence.ownerFence && + leaseEvidence.observedAt === evidence.observedAt + ) + ) + .sort((left, right) => left.ownerFence - right.ownerFence) +} + +/** Each step finds its own debt in memory and writes only what it finds, so it is safe on every + * pass: the conversion, sends still queued from an earlier process; the mark, a converted send or + * a reopen floor this handle has not marked; the rewind, a record still in doubt. */ +async function settleEarlierProcess( + context: StructuredAgentSessionReconciliationPassContext, + sessionId: string, + journal: StructuredAgentSessionHostSession['journal'], + writes: { store: BackgroundStoreWrites; journal: BackgroundJournalWrites }, + processOpened: AgentJournalCursor, + pass: StructuredAgentSessionReconciliationPass +): Promise { + const record = context.deps.store.getRecord(sessionId) + if (!record) { + return + } + const fence = record.lease.runtimeFence + const sameEpoch = processOpened.epoch === journal.cursor().epoch + const earlier = (sequence: number | undefined): boolean => + sameEpoch && sequence !== undefined && sequence <= processOpened.sequence + const attempt = async (step: () => Promise): Promise => + step().then( + () => true, + (error: unknown) => { + pass.failed.push(error) + return false + } + ) + // Another connection's lock ends the pass: the round ends, and every step is tried again. + if (!(await attempt(() => writes.journal.queuedMessages.repairAndPrune()))) { + return + } + // Before a Stop can withdraw one: a Stop never withdraws a card. + let converted = false + const held = await attempt(async () => { + converted = + (await holdUnsentSends(journal, { + fence, + hostInstance: structuredAgentSessionHostInstance(), + hold: { cause: 'hostRestarted', which: (entry) => earlier(entry.acceptedSequence) }, + writes: writes.journal + })) !== null + }) + if (!held && pass.failed.some(isSqliteContentionFailure)) { + return + } + // After every card the earlier process left, so a card this one queued is never held by it. + // Reported, never owed: the pause starts there anyway, and a failed mark leaves the floor set, + // so the next pass marks again. + const floor = journal.reopenFloor() + const since = queuedMessageReopenMarkStart(floor, processOpened, journal.cursor().epoch) + if ((converted || floor !== null) && since !== null) { + await markStructuredQueueReopen(sessionId, writes.journal, fence, context.deps.logger, since) + } + // A rewind left prepared refuses every send until settled. With no provider here, one only its + // provider can prove stays, quietly, for the next acquisition, which recovers it. + if (!structuredRewindNeedsProvider(record.rewind)) { + const deps = { ...context.deps, background: writes } + await recoverStructuredRewind(deps, sessionId, journal, fence).catch((error: unknown) => { + // Another connection's lock ends the round, as every other write here does. + if (isSqliteContentionFailure(error)) { + pass.failed.push(error) + return + } + context.deps.logger.warn('recovering a rewind an earlier process left did not finish', { + scope: 'reconciliation-rewind', + sessionId, + error + }) + }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts new file mode 100644 index 000000000000..f78f530a2e27 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts @@ -0,0 +1,301 @@ +// The retry of a chat's background bookkeeping: what settles it, what keeps it owed, and what +// readers see before it has written anything. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { + liveTestJournalRows, + openTestJournalHostDatabase, + updateTestJournalRowJson +} from '../agent-session-journal/journal-host-database-test-support' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD +} from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' +import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' +import { + currentJournal, + exitChild, + exitWhileSettlementFails, + leaveUnfinishedWork, + reconciled, + recoveredRows, + REJECT_RECOVERED_ROWS, + turnState, + watchSettlementCommits +} from './structured-agent-session-leftover-settlement.test-fixture' +import { retryIdle, retryOwes } from './structured-agent-session-retry.test-fixture' +import { backgroundSettlementWrites } from './structured-agent-session-background-writes' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +function retryOf(current: QueuedMessageTestRig) { + return current.host.collaboratorsForTests().reconciliation +} + +describe('a retry dies', () => { + it('on supersession: a later settlement covered what it owed, so it writes nothing and retires', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + // A crash, and a new process whose storage refuses settlement rows from its first write; its + // recovery releases the owner it cannot prove gone. + await current.crashRestartHostProcess(() => + openTestJournalHostDatabase(current.root).db.exec(REJECT_RECOVERED_ROWS) + ) + const database = openTestJournalHostDatabase(current.root) + expect(retryOwes(retryOf(current), SESSION)).toBe(true) + expect(turnState(current)).toBe('running') + database.db.exec('DROP TRIGGER reject_recovered') + + // Another settlement lands first, as a later generation's end would run it. + expect( + await settleStructuredAgentSessionLeftovers({ + store: current.store, + sessionId: SESSION, + journal: currentJournal(current), + writes: backgroundSettlementWrites(currentJournal(current)) + }) + ).toMatchObject({ ok: true }) + expect(turnState(current)).toBe('unverifiable') + const commits = watchSettlementCommits() + const settled = recoveredRows(database.db) + + await reconciled(current) + + expect(commits()).toEqual([]) + expect(recoveredRows(database.db)).toEqual(settled) + }) + + it('on the chat’s deletion: no record, nothing owed', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { database } = await exitWhileSettlementFails(current) + const getRecord = current.store.getRecord + vi.spyOn(current.store, 'getRecord').mockImplementation((sessionId) => + sessionId === SESSION ? null : getRecord(sessionId) + ) + database.db.exec('DROP TRIGGER reject_recovered') + + await reconciled(current) + + expect(recoveredRows(database.db)).toEqual([]) + }) + + it('at shutdown: its timer stops, and nothing is written after', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { database } = await exitWhileSettlementFails(current) + database.db.exec('DROP TRIGGER reject_recovered') + + current.host.stopDelivery() + + expect(retryOwes(retryOf(current), SESSION)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_500)) + expect(recoveredRows(database.db)).toEqual([]) + expect(turnState(current)).toBe('running') + }) +}) + +describe('an observed exit whose release write failed', () => { + it('is Working for no reader at once, restated with no row, and the retry owes the release until it lands', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const fence = current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const events: AgentSessionSubscribeEvent[] = [] + const unsubscribe = await current.host.subscribe({ + id: 'reader', + sessionId: SESSION, + emit: (event) => events.push(event) + }) + const cursor = currentJournal(current).cursor() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const database = openTestJournalHostDatabase(current.root) + database.db.exec(REJECT_RECOVERED_ROWS) + const release = vi + .spyOn(current.store, 'transitionHandoff') + .mockRejectedValue(new Error('temporary storage failure')) + + await exitChild(current) + + // The lease still names the gone child, and the journal has not moved... + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + runtimeFence: fence, + claimStatus: 'live' + }) + expect(currentJournal(current).cursor()).toEqual(cursor) + expect(turnState(current)).toBe('running') + // ...yet the host's view already says nothing runs, and the reader was told so. + await vi.waitFor(() => + expect(events.at(-1)).toMatchObject({ + type: 'batch', + batch: { items: [], cursor }, + working: false, + latestTurn: { turn: { state: 'interrupted' } } + }) + ) + + // The items settle once storage takes rows again; the release is still owed, and an empty + // item plan never retires it. + database.db.exec('DROP TRIGGER reject_recovered') + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 10_000 }) + await retryIdle(retryOf(current), SESSION) + expect(retryOwes(retryOf(current), SESSION)).toBe(true) + expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + + release.mockRestore() + await reconciled(current) + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + runtimeFence: fence + 1, + claimStatus: 'released', + deathEvidence: { kind: 'exit-observed', ownerFence: fence } + }) + unsubscribe() + }) +}) + +describe('a journal no retry can load', () => { + it.each([ + ['damaged', '}{'], + ['saved by a newer Orca', null] + ] as const)( + '%s: the startup retry settles it instead of replaying it again', + async (_why, json) => { + // The owner is proven gone at the restart, so no recovery waits on a person's attach. + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) + const current = rig + await current.settleAccepted(await current.workingSend(), 'work') + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let opens = 0 + await current.crashReloadHostProcess(() => { + const { db } = openTestJournalHostDatabase(current.root) + const epochRow = liveTestJournalRows(db, SESSION).find((row) => row.seq === 1)! + const newer = { + ...JSON.parse(epochRow.rowJson), + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION + 1 + } + updateTestJournalRowJson(db, SESSION, 1, json ?? JSON.stringify(newer)) + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + opens += 1 + return open.call(this) + }) + }) + + expect(retryOwes(retryOf(current), SESSION)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(opens).toBe(1) + } + ) +}) + +describe('the reopen mark after a restart', () => { + // Bookkeeping, reported and never owed: the pause starts where the handle opened either way. + it.each([false, true])( + 'never holds the queue past the turn that lifts it (its write fails: %s)', + async (failMark) => { + rig = await createQueuedMessageTestRig() + const current = rig + await current.workingSend() + const queued = await current.send('queued before the crash', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + const draftId = queued.value.queued.messageId + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + await current.crashRestartHostProcess(() => { + if (failMark) { + vi.spyOn(AgentSessionJournal.prototype, 'appendQueueReopen').mockRejectedValue( + new Error('persistent storage failure') + ) + } + }) + const journal = currentJournal(current) + // Held from the open, before any mark could land. + expect(structuredQueuePauses(journal).map((pause) => pause.reason)).toEqual(['restarted']) + + const next = current.send('sent after the restart') + await next.result + await current.settleAccepted(next.id, 'next') + + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + await reconciled(current) + } + ) +}) + +describe("an ended generation's closing tail", () => { + it('lands at its own fence after the exit settled, and the retry settles what it left running', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + // A running turn is left, so the exit's settlement writes rows. + await leaveUnfinishedWork(current) + const deadFence = current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await exitChild(current) + await reconciled(current) + const tail = (ordinal: number): AgentJournalItemIdentity => ({ + provider: 'codex', + threadId: THREAD, + turnId: 'late', + ordinal + }) + const late = { fence: deadFence, ownerFence: deadFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + + // What the closed sink had already taken, delivered after the exit: it lands. + await currentJournal(current).appendItem( + tail(1), + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'final answer' }] }, + late + ) + await currentJournal(current).appendItem( + tail(2), + { kind: 'turn', turnId: 'late', state: 'running' }, + late + ) + + expect(currentJournal(current).itemBody(agentJournalItemKey(tail(1)))).not.toBeNull() + // Below the moved fence, so its commit woke the retry, which settles it by the exit's proof. + await vi.waitFor(() => + expect( + readAgentJournalTurn( + currentJournal(current).itemBody(agentJournalItemKey(tail(2))) ?? undefined + )?.state + ).toBe('interrupted') + ) + await reconciled(current) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts new file mode 100644 index 000000000000..addf76959305 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -0,0 +1,339 @@ +// The host's one retry of background bookkeeping on its database connection: what ended +// generations and an earlier host process left, and queue sends that met another connection's lock. +// One owed set (`signal`; a reader's open puts an owed chat first). A round runs (a) the store-wide +// step while owed (`reconcileOwed`), then per owed chat, one at a time in its lane, (b) release +// repair, (c) settlement (`runStructuredAgentSessionReconciliationPass`), (d) the queue's next send. +// Every write is background: a lock fails it at once and stops the round. +// One wake (`wake`): marking a chat owed or any commit runs a round now unless one runs or the timer +// waits; a commit after a lock refused the last round ends that wait. The timer is the backoff +// after a refused round (1 s to 2 s), or its first step after a busy lane. One budget: about 3 +// minutes since the episode's first refused round give up (`RECONCILIATION_GIVE_UP_MS`), keeping +// the set, dropping its loaded reads and showing owed sends as not sent; only a round that made +// progress ends the episode, and only the next wake after a give-up resets it. + +import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' +import { setImmediate as yieldToEvents } from 'node:timers/promises' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' +import { + RECONCILIATION_GIVE_UP_MS, + reconciliationBackoffDelay +} from './structured-agent-session-reconciliation-backoff' +import { StructuredAgentSessionReconciliationSlots } from './structured-agent-session-reconciliation-slots' +import { providerChildExitSettling } from './structured-agent-session-provider-child' +import { + dropStructuredAgentSessionLoaded, + noteStructuredAgentSessionOpened, + visitStructuredAgentSessionOwedChat, + type StructuredAgentSessionOwedChat as OwedChat, + type StructuredAgentSessionRetryContext, + type StructuredAgentSessionVisit as Visit, + type StructuredAgentSessionVisitHost +} from './structured-agent-session-reconciliation-visit' + +export type { StructuredAgentSessionRetryContext } from './structured-agent-session-reconciliation-visit' + +export type StructuredAgentSessionReconciliationSignal = { + /** Proof the lease no longer holds (`AgentSessionGenerationEnd.evidence`). */ + evidence?: AgentSessionGenerationEnd['evidence'] + /** An observed exit whose own settlement did not land. */ + exit?: StructuredAgentSessionExitSettlement + /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ + restate?: boolean + /** The exit's own writes, or the queue's send, met another connection's lock. */ + contended?: boolean +} + +export class StructuredAgentSessionRetry { + private readonly owed = new Map() + /** Where this process first opened each chat's journal: every send at or before it was accepted + * by an earlier process. Kept for the whole process; dropped only with the chat's record. */ + private readonly firstOpened = new Map() + private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) + private readonly unsubscribe: () => void + /** The one timer ('soon': a round queued now). */ + private timer: ReturnType | 'soon' | null = null + private running = false + /** Woken while a round ran: another follows at once. */ + private again = false + /** Refused rounds in a row, and when the first was (`performance.now()`); at the budget the + * episode ends. `locked`: the last one met another connection's lock, which a commit here proves + * gone. */ + private failures = 0 + private failingSince: number | null = null + private locked = false + private disposed = false + /** The round's chain of chat turns (`inTurn`), and whether contention stopped the round. */ + private turn: Promise = Promise.resolve() + private stopped = false + private readonly visitHost: StructuredAgentSessionVisitHost + + constructor(private readonly context: StructuredAgentSessionRetryContext) { + this.visitHost = { + context, + disposed: () => this.disposed, + firstOpened: this.firstOpened, + settle: (sessionId, chat) => this.settle(sessionId, chat), + warn: (sessionId, error) => this.warn(sessionId, error), + inTurn: (run) => this.inTurn(run) + } + this.unsubscribe = context.deps.store.onGenerationEnded((ended) => + this.signal(ended.sessionId, { evidence: ended.evidence }) + ) + } + + signal = (sessionId: string, signal: StructuredAgentSessionReconciliationSignal = {}): void => { + const session = this.context.sessions.get(sessionId) + if (session) { + session.operationalRevision = (session.operationalRevision ?? 0) + 1 + } + const chat = this.disposed ? null : this.owe(sessionId) + if (chat && signal.evidence) { + chat.debts.evidence = [...(chat.debts.evidence ?? []), signal.evidence] + } + if (chat && signal.exit) { + chat.debts.exit = signal.exit + } + if (chat && signal.contended) { + // Its queued send met, or would meet, the same lock: the round sends it. + chat.send = true + } + this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) + this.wake() + } + + /** A committed row: one below the lease's fence is an ended generation's late write; any one + * wakes the retry. */ + observeCommit = (sessionId: string, lowestFence: number | null): void => { + const fence = this.context.deps.store.getRecord(sessionId)?.lease.runtimeFence + if (lowestFence !== null && fence !== undefined && lowestFence < fence) { + this.signal(sessionId) + } else if ([...this.owed.values()].some((chat) => chat.due)) { + this.wake(this.locked) + } + } + + /** A reader opened the chat: its handle marks the process boundary if it is the first, and an + * owed chat goes ahead of the scan. */ + opened = (sessionId: string, journal: Pick): void => { + noteStructuredAgentSessionOpened(this.firstOpened, sessionId, journal) + const chat = this.owed.get(sessionId) + if (chat) { + this.slots.prioritize(chat) + this.wake() + } + } + + /** Whether the chat's queued send waits for a round (or for a fresh episode, given up): it met a + * lock, or the exit it follows is still settling. */ + sendWaits = (sessionId: string): boolean => + this.owed.get(sessionId)?.send === true || + providerChildExitSettling(this.context.sessions.get(sessionId)) + + /** Resolves once the chat's next visit finished, or at once when it owes nothing. */ + attempted = (sessionId: string): Promise => { + const chat = this.owed.get(sessionId) + return chat?.due ? new Promise((resolve) => chat.attempted.push(resolve)) : Promise.resolve() + } + + /** Shutdown: the timer stops and every slot wait ends; a visit already writing finishes, + * tracked, and one that has not started work starts none. */ + dispose = (): void => { + this.disposed = true + this.unsubscribe() + this.stopTimer() + for (const [sessionId, chat] of this.owed) { + chat.slotWait?.abort() + this.settle(sessionId, chat) + } + } + + private owe(sessionId: string): OwedChat { + let chat = this.owed.get(sessionId) + if (!chat) { + chat = { debts: {}, due: true, dirty: false, attempted: [], urgent: false } + this.owed.set(sessionId, chat) + } + chat.due = true + chat.dirty = true + return chat + } + + /** The one wake: a round now, unless one runs (it runs again) or the timer is pending, which + * `early` (a commit after a lock refused the last round) ends. After a give-up it starts a fresh + * episode, the budget's only reset. */ + private wake(early = false): void { + if (this.disposed) { + return + } + if (this.running) { + this.again = true + return + } + // No backoff pending once the budget is spent: the episode gave up, and this wake starts anew. + if (this.timer === null && this.failingFor() >= RECONCILIATION_GIVE_UP_MS) { + this.failures = 0 + this.failingSince = null + } + if (early) { + this.stopTimer() + } + if (this.timer !== null) { + return + } + this.timer = 'soon' + queueMicrotask(() => void this.round()) + } + + private async round(): Promise { + this.timer = null + if (this.disposed || this.running) { + return + } + this.running = true + this.again = false + this.stopped = false + let failed = false + let locked = false + let busy = false + let progress = false + const end = (visit: Visit) => { + progress ||= visit === 'settled' || visit === 'again' + failed ||= visit === 'failed' || visit === 'contended' + locked ||= visit === 'contended' + busy ||= visit === 'busy' + this.stopped ||= visit === 'contended' + this.again ||= visit === 'again' + } + if (this.context.reconcileOwed()) { + // Store-wide: once a round, for every chat, never once per chat. No chat is visited while it + // owes anything. + const reconciled = await this.context.reconcile() + end(reconciled) + this.stopped = reconciled !== 'settled' + } + const due = [...this.owed].filter(([, chat]) => chat.due) + // Nothing owed is progress too; a round only busy or parked is not. + progress ||= due.length === 0 + await Promise.all( + due.map(async ([sessionId, chat]) => { + const visit = this.stopped ? null : await this.visitInSlot(sessionId, chat) + if (visit) { + end(visit) + } + }) + ) + this.running = false + this.roundEnded(failed, locked, busy, progress) + // After the round's outcome is set, so a waiter reads its backoff, never a round still running. + for (const [, chat] of due) { + chat.attempted.splice(0).forEach((resolve) => resolve()) + } + } + + private roundEnded(failed: boolean, locked: boolean, busy: boolean, progress: boolean): void { + if (this.disposed) { + return + } + this.locked = locked + if (!failed) { + if (progress) { + this.failures = 0 + this.failingSince = null + } + if (this.again) { + this.wake() + } else if (busy) { + // A person's operation held a lane: its chat waits the timer's first step, not a budget. + this.arm(reconciliationBackoffDelay(1)) + } + return + } + this.failures += 1 + this.failingSince ??= performance.now() + if (this.failingFor() < RECONCILIATION_GIVE_UP_MS) { + this.arm(reconciliationBackoffDelay(this.failures)) + return + } + // The episode ends; the set stays for the next wake. + this.context.deps.logger.warn("gave up settling a gone agent's leftover work for now", { + scope: 'reconciliation', + error: new Error(`${this.failures} rounds failed`) + }) + for (const [sessionId, chat] of this.owed) { + // Owed still, but no hidden chat's replay is held through the wait for the next wake. + dropStructuredAgentSessionLoaded(chat) + if (chat.send) { + this.context.abandonSend(sessionId) + } + } + } + + private failingFor(): number { + return this.failingSince === null ? 0 : performance.now() - this.failingSince + } + + private arm(delay: number): void { + this.stopTimer() + const timer = setTimeout(() => void this.round(), delay) + // A backoff alone never keeps the process alive. + timer.unref?.() + this.timer = timer + } + + /** An open chat, or one already loaded, goes straight to its lane; a closed one is replayed in a + * background slot, never once the round stopped. */ + private async visitInSlot(sessionId: string, chat: OwedChat): Promise { + if (this.context.sessions.has(sessionId) || chat.loaded) { + return this.context.track(this.visit(sessionId, chat)) + } + // The replay yields a macrotask first (`loadStructuredAgentSessionForReconciliation`). + const result = await this.slots.run(chat, async () => + this.disposed || this.stopped ? null : this.context.track(this.visit(sessionId, chat)) + ) + return result?.value ?? null + } + + private visit(sessionId: string, chat: OwedChat): Promise { + return visitStructuredAgentSessionOwedChat(this.visitHost, sessionId, chat) + } + + /** Chats load in the slots side by side, but write one at a time, in the order they are ready (a + * reader's open chat first), yielding between chats: each visit's writes stay bounded, and a + * person's operation queued meanwhile runs before the next chat's turn takes its lane. */ + private inTurn(run: () => Promise): Promise { + const next = this.turn.then(() => (this.stopped || this.disposed ? null : run())) + this.turn = next.then( + () => yieldToEvents(), + () => yieldToEvents() + ) + return next + } + + private settle(sessionId: string, chat: OwedChat): 'settled' { + if (this.owed.get(sessionId) === chat) { + this.owed.delete(sessionId) + } + dropStructuredAgentSessionLoaded(chat) + chat.attempted.splice(0).forEach((resolve) => resolve()) + return 'settled' + } + + private stopTimer(): void { + if (this.timer && this.timer !== 'soon') { + clearTimeout(this.timer) + } + if (this.timer !== 'soon') { + this.timer = null + } + } + + private warn(sessionId: string, error: unknown): void { + this.context.deps.logger.warn("settling a gone agent's leftover work did not finish", { + scope: 'reconciliation', + sessionId, + error + }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts new file mode 100644 index 000000000000..2ad0787bfc56 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts @@ -0,0 +1,55 @@ +// The host retry's few background slots, for a chat's replay and its startup recovery step: outside every chat's lane, so a send or a read never waits behind the scan. A chat a reader +// opened goes ahead of the rest; a quit ends every wait, and a task it has not started never runs. + +import { PrioritySemaphore } from '../../../shared/priority-semaphore' + +const BACKGROUND_SLOTS = 4 +const URGENT = 0 +const SCAN = 1 + +/** One chat's place in line. */ +export type StructuredAgentSessionReconciliationSlotWaiter = { + /** Goes ahead of the startup scan: a reader opened the chat. */ + urgent: boolean + /** The wait in progress, aborted to requeue it ahead or to quit. */ + slotWait?: AbortController +} + +export class StructuredAgentSessionReconciliationSlots { + private readonly slots = new PrioritySemaphore(BACKGROUND_SLOTS) + + constructor(private readonly disposed: () => boolean) {} + + /** Requeues the chat's wait ahead of the scan. */ + prioritize(waiter: StructuredAgentSessionReconciliationSlotWaiter): void { + if (!waiter.urgent) { + waiter.urgent = true + waiter.slotWait?.abort() + } + } + + /** Runs the task in a slot; null when quit came first, and the task never ran. */ + async run( + waiter: StructuredAgentSessionReconciliationSlotWaiter, + task: () => Promise + ): Promise<{ value: T } | null> { + let release: (() => void) | null = null + while (!release) { + if (this.disposed()) { + return null + } + const wait = new AbortController() + waiter.slotWait = wait + // Aborted to requeue ahead or to quit: the loop decides which. + release = await this.slots + .acquire(waiter.urgent ? URGENT : SCAN, wait.signal) + .catch(() => null) + waiter.slotWait = undefined + } + try { + return this.disposed() ? null : { value: await task() } + } finally { + release() + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts new file mode 100644 index 000000000000..43942dd4f17c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts @@ -0,0 +1,213 @@ +// One visit of the retry (`StructuredAgentSessionRetry`) to one owed chat: release repair, +// settlement and what an earlier process left (`runStructuredAgentSessionReconciliationPass`) in +// the chat's lane, then the queue's next send. Its outcome tells the round how to go on. + +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + runStructuredAgentSessionReconciliationPass, + type StructuredAgentSessionReconciliationDebts, + type StructuredAgentSessionReconciliationPassContext +} from './structured-agent-session-reconciliation-pass' +import { + loadStructuredAgentSessionForReconciliation, + structuredAgentSessionJournalIsCurrent +} from './structured-agent-session-reconciliation-load' +import type { StructuredAgentSessionReconciliationSlotWaiter } from './structured-agent-session-reconciliation-slots' + +export type StructuredAgentSessionRetryContext = StructuredAgentSessionReconciliationPassContext & { + deps: StructuredAgentSessionReconciliationPassContext['deps'] + /** The chat's action lane, and whether anything holds it now. */ + serialize: (sessionId: string, task: () => Promise) => Promise + laneBusy: (sessionId: string) => boolean + /** Lets a quit wait for a visit already writing. */ + track: (operation: Promise) => Promise + /** Publishes what is current now and wakes the queued-card drain. */ + publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void + /** Whether the store-wide step owes anything: a lease unreconciled, or the restart restore's + * latched recovery its own write could not decide. */ + reconcileOwed: () => boolean + /** That step, as background bookkeeping: whether it all settled, or why not. */ + reconcile: () => Promise<'settled' | 'contended' | 'failed'> + /** The queue's automatic send of the chat's next card, inside the lane the visit holds, and an + * episode giving up on it (`StructuredAgentSessionQueuedMessageDrain.sendForRetry`, `.abandon`). */ + sendQueued: (sessionId: string) => Promise<'contended' | 'done'> + abandonSend: (sessionId: string) => void +} + +/** `again`: it wrote, or was signalled meanwhile, so the next round verifies nothing is left. + * `busy`: a person's operation held its lane, so it waits, owed, for a later round. */ +export type StructuredAgentSessionVisit = + | 'settled' + | 'again' + | 'parked' + | 'failed' + | 'contended' + | 'busy' + +/** A chat with something owed, until a visit finds nothing left. */ +export type StructuredAgentSessionOwedChat = StructuredAgentSessionReconciliationSlotWaiter & { + debts: StructuredAgentSessionReconciliationDebts + /** A visit is owed now; false while only a recovering lease's proofs wait for its decision. */ + due: boolean + /** The queue's automatic send met contention: it waits for the next round to wake it. */ + send?: true + /** Signalled while its visit ran: the next round visits it again. */ + dirty: boolean + attempted: (() => void)[] + /** The visit's own read of the chat while nobody has it open, kept across rounds; closed when + * the chat settles, a reader opens it, or another handle wrote past it. */ + loaded?: StructuredAgentSessionHostSession +} + +/** What a visit reads of the retry running it. */ +export type StructuredAgentSessionVisitHost = { + context: StructuredAgentSessionRetryContext + disposed: () => boolean + /** Where this process first opened each chat's journal (`StructuredAgentSessionRetry`). */ + firstOpened: Map + /** The chat leaves the owed set. */ + settle: (sessionId: string, chat: StructuredAgentSessionOwedChat) => 'settled' + warn: (sessionId: string, error: unknown) => void + /** The round's one-at-a-time turn for a chat's writes; null once the round stopped. */ + inTurn: (run: () => Promise) => Promise +} + +type Chat = StructuredAgentSessionOwedChat +type Visit = StructuredAgentSessionVisit + +export async function visitStructuredAgentSessionOwedChat( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Promise { + const { context } = host + try { + const { store } = context.deps + if (!store.getRecord(sessionId) || store.readOnly) { + // The chat is gone, or nothing here may write: nothing of it is owed again. + if (!store.getRecord(sessionId)) { + host.firstOpened.delete(sessionId) + } + return host.settle(sessionId, chat) + } + if (!context.sessions.has(sessionId) && !chat.loaded) { + const loaded = await loadStructuredAgentSessionForReconciliation( + context.deps, + sessionId, + () => host.disposed() || context.sessions.has(sessionId) + ) + if (loaded.kind === 'loaded') { + chat.loaded = loaded.session + noteStructuredAgentSessionOpened(host.firstOpened, sessionId, loaded.session.journal) + } else if (loaded.kind === 'nothing') { + return host.settle(sessionId, chat) + } + } + // Its read stays loaded should the round stop before its turn: the next one writes, not replays. + const visit = await host.inTurn(() => write(host, sessionId, chat)) + return visit === 'settled' ? host.settle(sessionId, chat) : visit + } catch (error) { + host.warn(sessionId, error) + return isSqliteContentionFailure(error) ? 'contended' : 'failed' + } +} + +/** The chat's writes, in its lane: the pass, then (d) once nothing else of it is due, the send it + * owes, with the card named again. Never waits for a lane inside the round's turn: a person's + * operation holding it leaves the chat owed for a later round. */ +async function write( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Promise { + const { context } = host + if (context.laneBusy(sessionId)) { + return 'busy' + } + return context.serialize(sessionId, async () => { + const visit = await pass(host, sessionId, chat) + if ((visit !== 'settled' && visit !== 'parked') || !chat.send) { + return visit + } + if ((await context.sendQueued(sessionId)) === 'contended') { + return 'contended' + } + delete chat.send + context.publishGenerationEnded(sessionId) + return visit + }) +} + +/** Where this process first opened the chat, if this is the first. */ +export function noteStructuredAgentSessionOpened( + firstOpened: Map, + sessionId: string, + journal: { openedAt: () => AgentJournalCursor } +): void { + if (!firstOpened.has(sessionId)) { + firstOpened.set(sessionId, journal.openedAt()) + } +} + +/** Closes the visit's own read of the chat. */ +export function dropStructuredAgentSessionLoaded(chat: Chat): void { + const journal = chat.loaded?.journal + chat.loaded = undefined + void journal?.close() +} + +/** The pass, inside the chat's lane, on the conversation a reader holds or the visit's own. */ +async function pass( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Promise { + const session = sessionFor(host, sessionId, chat) + if (host.disposed() || !session) { + // Closed, or written by another handle, since this visit began: the next one reads it again. + return host.disposed() ? 'settled' : 'failed' + } + chat.dirty = false + const result = await runStructuredAgentSessionReconciliationPass( + host.context, + sessionId, + session, + chat.debts, + host.firstOpened.get(sessionId) ?? session.journal.openedAt() + ) + if (result.failed.length > 0) { + host.warn(sessionId, result.failed[0]) + return result.failed.some(isSqliteContentionFailure) ? 'contended' : 'failed' + } + if (result.wrote || chat.dirty) { + return 'again' + } + if (result.recovering && chat.debts.evidence) { + // Its proofs wait for the recovery's decision, whose release signals again. An exit's account + // judges only its own generation, which a later one may have replaced by then. + delete chat.debts.exit + chat.due = false + return 'parked' + } + return 'settled' +} + +function sessionFor( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Pick | undefined { + const open = host.context.sessions.get(sessionId) + const { loaded } = chat + if ( + open || + (loaded && + !structuredAgentSessionJournalIsCurrent(host.context.deps, sessionId, loaded.journal)) + ) { + dropStructuredAgentSessionLoaded(chat) + return open + } + return loaded +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts index 6dfcb4f898e9..ba5d609b94e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts @@ -22,10 +22,8 @@ import { selectStructuredAgentSettledTurns } from '../../../shared/structured-ag import { AgentHookServer, _internals } from '../../agent-hooks/server' import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from './structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import type { StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' import { indexedStatusFeedSession } from './structured-agent-session-status-feed-test-session' @@ -128,8 +126,8 @@ async function sessionWithRunningTurn() { function settleDeadGeneration( journal: AgentSessionJournal, verdict: StructuredAgentSessionTurnVerdict -): ReturnType { - return settleStructuredAgentSessionDeadGeneration({ +): ReturnType { + return settleObservedExitForTest({ journal, sessionId: SESSION, fence: 2, @@ -159,7 +157,7 @@ describe('a turn recovery settled after its host went away', () => { async (_label, verdict, outcome, mark) => { const session = await sessionWithRunningTurn() session.recoverAt(RECOVERED) - expect(await settleDeadGeneration(session.journal, verdict)).toEqual({ ok: true }) + expect(await settleDeadGeneration(session.journal, verdict)).toMatchObject({ ok: true }) session.publish() expect(session.summaries.at(-1)).toMatchObject({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts index 257072f7c23b..7c2eb4af060d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts @@ -17,13 +17,19 @@ import { } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../../provider-process/provider-process-supervisor' -import { releaseUnprovenAgentSessionOwner } from '../../runtime/agent-session-lease-transitions' +import { + evictAgentSessionOwner, + releaseUnprovenAgentSessionOwner +} from '../../runtime/agent-session-lease-transitions' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionLeaseWrites } from './structured-agent-session-lease-release' export type StructuredSessionRecoveryStopSignal = 'SIGTERM' | 'SIGKILL' export type StructuredSessionRecoveryResolutionDeps = { store: AgentSessionRecordStore + /** The store itself, or a startup restore's background handle (`backgroundLeaseWrites`). */ + writes?: StructuredAgentSessionLeaseWrites probeRecord: (record: AgentSessionRecord) => Promise now: () => number stopOwnerProcess?: (pid: number, signal: StructuredSessionRecoveryStopSignal) => void @@ -69,21 +75,14 @@ export async function resolveStructuredSessionRecovery( probe = await stopOwnerAndReprobe(deps, record, owner.pid) } } + const expectedFence = record.lease.runtimeFence + const now = deps.now() try { - await (owner && !isProvenDeadProbe(probe) - ? deps.store.transitionHandoff(sessionId, (latest) => - releaseUnprovenAgentSessionOwner({ - record: latest, - expectedFence: record.lease.runtimeFence, - now: deps.now() - }) - ) - : deps.store.evictProvenDeadOwner({ - sessionId, - expectedFence: record.lease.runtimeFence, - probe, - now: deps.now() - })) + await (deps.writes ?? deps.store).transitionHandoff(sessionId, (latest) => + owner && !isProvenDeadProbe(probe) + ? releaseUnprovenAgentSessionOwner({ record: latest, expectedFence, now }) + : evictAgentSessionOwner({ record: latest, expectedFence, probe, now }) + ) return 'resolved' } catch (error) { const code = error instanceof Error ? error.message : String(error) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts index 822fb635b966..56b8a0c0d6bc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts @@ -5,7 +5,6 @@ import { type AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { completedStructuredAgentTurnSeconds, @@ -18,10 +17,6 @@ import { closeTestJournalHostDatabases, openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { - openStructuredAgentSessionConversationJournal, - type StructuredAgentSessionConversationOpenDeps -} from './structured-agent-session-conversation-open' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { HOST_TEST_NOW, hostTestAttachParams } from './structured-agent-session-host-test-data' import { @@ -32,6 +27,7 @@ import { REST_TEST_THREAD as THREAD, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 @@ -118,6 +114,19 @@ async function startServer(options: { newProcess: boolean }): Promise { }) } +/** Startup as the runtime runs it: the lease reconcile, the retry's settlement, the visible tabs. */ +async function startUp(): Promise { + await rig.host.reconcileRestartLeases() + await rig.host.startupSettled() + await settled() + await rig.host.restoreReadableSessions() + await settled() +} + +function settled(): Promise { + return retryIdle(rig.host.collaboratorsForTests().reconciliation, SESSION) +} + function turnsOf(items: readonly AgentJournalRenderItem[]) { return items.flatMap((item) => readAgentJournalTurn(item.body) ?? []) } @@ -150,7 +159,8 @@ describe('a server replaced mid-turn on the same execution host', () => { it('interrupts the cut turn at its last saved output with no provider identity left', async () => { await streamThenServerKilled() await startServer({ newProcess: true }) - await rig.host.restoreReadableSessions() + // Settled at startup, not on open. + await startUp() await expectCutTurnSettledOnce() await rig.host.reconcileRestartLeases() @@ -211,31 +221,107 @@ describe('a server replaced mid-turn on the same execution host', () => { await streamThenServerKilled() // The same process reopening its store replaced no runtime: nothing proves the turn over. await startServer({ newProcess: false }) - await rig.host.restoreReadableSessions() - await rig.host.reconcileRestartLeases() - await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) + await startUp() const left = turnsOf((await rig.host.journalSnapshot(SESSION)).items) expect(left).toEqual([expect.objectContaining({ state: 'unverifiable' })]) + // Healed at startup, not on open. await startServer({ newProcess: true }) - await rig.host.restoreReadableSessions() + await startUp() await expectCutTurnSettledOnce() - const settled = await rig.host.journalSnapshot(SESSION) + const healed = await rig.host.journalSnapshot(SESSION) await startServer({ newProcess: true }) - await rig.host.restoreReadableSessions() - expect((await rig.host.journalSnapshot(SESSION)).items).toEqual(settled.items) + await startUp() + expect((await rig.host.journalSnapshot(SESSION)).items).toEqual(healed.items) }) +}) + +// No write on open: before the startup lease check, the projection derives the end from the +// runtime this one replaced; the startup pass then settles it, even under a recovery latch. +describe('a chat whose turn a replaced server held, before anything settles it', () => { + it('reads its cut turn interrupted when opened before the startup lease check', async () => { + await streamThenServerKilled() + await startServer({ newProcess: true }) + + const { page } = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + + expect(page.latestTurn?.turn).toMatchObject({ turnId: TURN.turnId, state: 'interrupted' }) + expect(page.working).toBe(false) + // Derived, not written: the journal holds the turn as the crash left it. + const { items } = await rig.host.journalSnapshot(SESSION) + expect(turnsOf(items)).toEqual([expect.objectContaining({ state: 'running' })]) + }) + + /** Its agent outlived the server; startup runs with no tab showing the chat. */ + async function survivingOwnerStartup(newProcess: boolean) { + await streamThenServerKilled() + await startServer({ newProcess }) + let alive = true + rig.host.deps.probeOwner = async () => + alive + ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } + : { outcome: 'pid-absent' } + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + rig.host.deps.stopOwnerProcess = stopOwnerProcess + await rig.host.reconcileRestartLeases() + await rig.host.startupSettled() + await settled() + const lease = rig.store.getRecord(SESSION)?.lease + expect(lease?.handoffStage).toBe('recovering') + return { stopOwnerProcess, lease } + } + + it('reads settled from history after startup, with no send or attach, and its next send starts a new agent', async () => { + const { stopOwnerProcess, lease } = await survivingOwnerStartup(true) - it('only lets an acquisition open a journal without the store', () => { - const open = ( - deps: Omit, - record: AgentSessionRecord - ) => { - void openStructuredAgentSessionConversationJournal(deps, record, { acquisition: true }) - // @ts-expect-error a reader open settles, so without the store its turns would stay unverifiable - void openStructuredAgentSessionConversationJournal(deps, record) + await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + await expectCutTurnSettledOnce() + // The settle stopped no process and left the recovery's lease as it was. + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(rig.store.getRecord(SESSION)?.lease).toEqual(lease) + + // The attach resolves the recovery first, which moves the fence it expected. + const stale = await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(lease?.runtimeFence ?? null) + ) + const attached = stale.ok + ? stale + : await rig.host.attach( + REST_TEST_CALLER, + hostTestAttachParams(stale.refusal.currentFence ?? null) + ) + if (!attached.ok) { + throw new Error(`attach refused: ${attached.refusal.code}`) } - expect(open).toBeTypeOf('function') + const sent = await rig.host.send( + REST_TEST_CALLER, + restTestSend('after restart', attached.fence) + ) + expect(sent.ok).toBe(true) + await vi.waitFor(() => expect(rig.adapter.dispatch).toHaveBeenCalledTimes(2)) + expect(rig.adapter.acquire).toHaveBeenCalledTimes(2) + }) + + it('reads unverifiable and not working while its owner recovers with no child here', async () => { + await survivingOwnerStartup(false) + + const { page } = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + + expect(page.working).toBe(false) + expect(page.latestTurn?.turn).toMatchObject({ turnId: TURN.turnId, state: 'unverifiable' }) + expect(rig.host.currentWork(SESSION)?.working()).toBe(false) + }) + + it('settles nothing and stops nothing with no runtime replaced, while its recovery waits', async () => { + const { stopOwnerProcess, lease } = await survivingOwnerStartup(false) + + const { items } = await rig.host.journalSnapshot(SESSION) + expect(turnsOf(items)).toEqual([expect.objectContaining({ state: 'running' })]) + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(rig.store.getRecord(SESSION)?.lease).toEqual(lease) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts index 0e91f3d1cf89..d360588e2fd6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts @@ -18,6 +18,7 @@ import { HOST_TEST_SESSION as SESSION, hostTestMessage } from './structured-agent-session-host-test-data' +import { retryIdle } from './structured-agent-session-retry.test-fixture' let root: string let store: AgentSessionRecordStore @@ -67,6 +68,9 @@ describe('send without ledger policy', () => { it('accepts an id older than 24 hours while its chat is closed', async () => { await attach() await host.close(SESSION, 'evict') + // The stop's release wakes the host's retry, which opens it, finds nothing owed, and closes it. + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) await expect(host.send(CALLER, expiredParams('long gone'))).resolves.toMatchObject({ ok: true, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts index 639cdc9b3ae7..0cc5a8cf3d0e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -36,6 +36,7 @@ import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idl import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { retryIdle } from './structured-agent-session-retry.test-fixture' export const REST_TEST_CALLER = { callerKey: 'client-1' } export const IDLE_MS = STRUCTURED_AGENT_SESSION_IDLE_MS @@ -225,8 +226,12 @@ export async function foundRestTestChat(rig: RestTestRig): Promise { } /** Runs one sweep pass now, for a test that set `idleSweep.intervalMs` out of reach. */ -export function sweepOnce(host: StructuredAgentSessionHost): Promise { - return host.collaboratorsForTests().lifetime.idleSweep.tick() +export async function sweepOnce(host: StructuredAgentSessionHost): Promise { + const { lifetime, reconciliation, serialize } = host.collaboratorsForTests() + await lifetime.idleSweep.tick() + // A close the sweep left to the host's retry lands once that retires. + await retryIdle(reconciliation, SESSION) + await serialize(SESSION, async () => {}) } /** Waits long enough for several sweep ticks to have run. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts index 09e48d3a3ef2..3027ffae3792 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts @@ -75,11 +75,16 @@ describe('restart offers after context clear', () => { ...value(), providerHandleChain: [proved] } + // Teardown runs while the child at fence 2 still holds the lease; the restart finds it released. + const heldRecord = { + ...currentRecord, + lease: { ...currentRecord.lease, claimStatus: 'live' as const, runtimeFence: 2 } + } const turn = { ...turnItem('new-turn', 'running'), sequence: 6 } const offer = structuredAgentSessionWorkingAtStop({ sessionId: SESSION, session: { journal: journal([clear, turn]), child: { fence: 2 } }, - getRecord: () => currentRecord, + getRecord: () => heldRecord, childWork: () => undefined, trigger: 'quit', teardownId: TEARDOWN_CURRENT, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts index db9107193361..f3b153cf79ba 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts @@ -181,6 +181,9 @@ export async function interruptedRestart( now: () => clock.now }) replaceHostTestState({ store, host }) + // As the runtime starts it, before any client reaches it. + await host.reconcileRestartLeases() + await host.startupSettled() previous.acquire.mockClear() previous.releaseAcquisition.mockClear() previous.dispatch.mockClear() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts index 932bd700fbf7..aa63dc5f889e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts @@ -31,12 +31,13 @@ afterEach(() => vi.restoreAllMocks()) function newerOrcaRow(root: string, sessionId: string = SESSION): void { const { db } = openTestJournalHostDatabase(root) const last = liveTestJournalRows(db, sessionId).at(-1) - const parsed: unknown = last ? JSON.parse(last.rowJson) : null - if (!last || typeof parsed !== 'object' || parsed === null || !('epoch' in parsed)) { + const parsed: unknown = last ? JSON.parse(last.rowJson) : { epoch: 'newer-orca-epoch' } + if (typeof parsed !== 'object' || parsed === null || !('epoch' in parsed)) { throw new Error('the chat has no journal row to follow') } + // A chat nothing was written to has no journal yet: the row founds one, after its epoch row. const { epoch } = parsed - const seq = last.seq + 1 + const seq = (last?.seq ?? 1) + 1 insertTestJournalRowJson( db, sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts index 694443515dd9..6c766d4447fa 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts @@ -25,6 +25,7 @@ import { REST_TEST_THREAD as THREAD, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 const SAVED_AT = HOST_TEST_NOW + 60_000 @@ -351,10 +352,16 @@ describe('provider output survives host restart', () => { const stopOwnerProcess = vi.fn() rig.host.deps.stopOwnerProcess = stopOwnerProcess await rig.host.reconcileRestartLeases() - const snapshot = await rig.host.journalSnapshot(SESSION) - const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) + await rig.host.startupSettled() + await retryIdle(rig.host.collaboratorsForTests().reconciliation, SESSION) + const read = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + // No runtime was replaced and no child here holds it: the turn reads unverifiable and not + // working, derived, while its recovery is undecided; nothing is written. + expect(rig.store.getRecord(SESSION)?.lease.handoffStage).toBe('recovering') + const turn = read.page.latestTurn?.turn expect(turn?.state).toBe('unverifiable') expect(turn?.completedAt).toBeUndefined() + expect(read.page.working).toBe(false) expect(stopOwnerProcess).not.toHaveBeenCalled() expect(rig.store.getRecord(SESSION)?.lease.deathEvidence).toBeFalsy() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts new file mode 100644 index 000000000000..8f612a061600 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts @@ -0,0 +1,177 @@ +// Startup's lease reconcile, failed because another connection held the database, is retried for +// every chat as ONE store-wide step of the retry's round: one call per round however many chats +// wait on it, so N chats never stack N waits on a busy store, and it never stops a process. + +import { DatabaseSync } from 'node:sqlite' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + scanRecord, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined +let locker: DatabaseSync | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-restart-reconcile-retry-')) +}) + +afterEach(async () => { + try { + locker?.exec('ROLLBACK') + } catch { + // Never begun. + } + locker?.close() + locker = undefined + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +async function seedChats(count: number, released: boolean): Promise { + const chats = Array.from({ length: count }, (_, i) => `chat-r${String(i).padStart(6, '0')}`) + await seedTestAgentSessionRecordStore(root, { + records: chats.map((id) => scanRecord(id, released)) + }) + for (const id of chats) { + await seedScanJournal(root, id) + } + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + return chats +} + +describe('a failed startup reconcile, retried by every chat that waits on it', () => { + it('runs once per backoff step for 40 chats, and stops no process', async () => { + await seedChats(40, false) + const reconcile = vi + .spyOn(store, 'reconcileOnRestart') + .mockRejectedValue(Object.assign(new Error('database is locked'), { errcode: 5 })) + const stops: unknown[] = [] + host = openScanHost(root, store, { + probeOwner: async () => ({ outcome: 'pid-absent' }), + stopOwnerProcess: (...args) => stops.push(args) + }) + + await host.reconcileRestartLeases() + await new Promise((resolve) => setTimeout(resolve, 3_500)) + + // Startup, then the shared retry at 0 s, 1 s and 3 s: never one per chat. + expect(reconcile.mock.calls.length).toBeGreaterThanOrEqual(3) + expect(reconcile.mock.calls.length).toBeLessThanOrEqual(4) + expect(stops).toEqual([]) + }) + + it('meets a real write lock once per round for 40 chats, never once per chat', async () => { + await seedChats(40, true) + expect(store.listRecords().every((record) => record.lease.unreconciled)).toBe(true) + openTestJournalHostDatabase(root).db.pragma('busy_timeout = 300') + const reconcile = vi.spyOn(store, 'reconcileOnRestart') + locker = new DatabaseSync(join(root, 'agent-session-journal.db')) + locker.exec('BEGIN IMMEDIATE') + host = openScanHost(root, store) + + await host.reconcileRestartLeases() + await new Promise((resolve) => setTimeout(resolve, 4_000)) + + // Startup's own, then one per round at 0 s, 1 s and 3 s: background, so each fails at once. + expect(reconcile.mock.calls.length).toBeLessThanOrEqual(5) + expect(store.listRecords().every((record) => record.lease.unreconciled)).toBe(true) + + // The lock lifts: the next step reconciles every lease. + locker.exec('ROLLBACK') + await vi.waitFor( + () => expect(store.listRecords().some((record) => record.lease.unreconciled)).toBe(false), + { timeout: 10_000 } + ) + }) +}) + +describe('a round of the retry', () => { + it("visits one chat at a time: a person's operation on a later chat runs before its visit", async () => { + const chats = await seedChats(3, true) + const order: string[] = [] + let release: () => void = () => undefined + const gate = new Promise((resolve) => (release = resolve)) + const pass = reconciliationPass.runStructuredAgentSessionReconciliationPass + vi.spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass').mockImplementation( + async (...args) => { + order.push(`visit:${args[1]}`) + if (order.length === 1) { + await gate + } + return pass(...args) + } + ) + host = openScanHost(root, store) + await host.reconcileRestartLeases() + await vi.waitFor(() => expect(order.length).toBeGreaterThan(0), { timeout: 10_000 }) + + // A chat the round has not reached: a person's operation on its lane, as a send runs. + const later = chats.find((id) => !order.includes(`visit:${id}`)) + if (!later) { + release() + throw new Error('every chat was visited at once') + } + const person = host + .collaboratorsForTests() + .serialize(later, async () => void order.push(`person:${later}`)) + await new Promise((resolve) => setTimeout(resolve, 50)) + release() + const waited = new Promise((_, reject) => + setTimeout(() => reject(new Error("the person's operation waited behind the round")), 5_000) + ) + await Promise.race([person, waited]) + await vi.waitFor(() => expect(order).toContain(`visit:${later}`), { timeout: 10_000 }) + + expect(order.indexOf(`person:${later}`)).toBeLessThan(order.indexOf(`visit:${later}`)) + expect(new Set(order.filter((entry) => entry.startsWith('visit:'))).size).toBe(3) + }) + + it("never holds the round for a chat's busy lane: a person's long operation delays no other chat", async () => { + const chats = await seedChats(3, true) + const started = performance.now() + const visits: Record = {} + const pass = reconciliationPass.runStructuredAgentSessionReconciliationPass + vi.spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass').mockImplementation( + async (...args) => { + visits[args[1]] ??= performance.now() - started + return pass(...args) + } + ) + host = openScanHost(root, store) + // A person's operation holds the first chat's lane for 3 s (a Stop waiting out its deadline). + const [busy, ...others] = chats + const person = host + .collaboratorsForTests() + .serialize(busy!, () => new Promise((resolve) => setTimeout(resolve, 3_000))) + await host.reconcileRestartLeases() + await vi.waitFor(() => expect(Object.keys(visits)).toHaveLength(2), { timeout: 10_000 }) + + expect(Math.max(...others.map((id) => visits[id] ?? Infinity))).toBeLessThan(1_500) + expect(visits[busy!]).toBeUndefined() + // The busy chat is visited once its lane frees. + await person + await vi.waitFor(() => expect(visits[busy!]).toBeGreaterThan(2_900), { timeout: 10_000 }) + }, 20_000) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 29f0c9f1047d..6fbbe5b14b1e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -9,6 +9,7 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { classifyStoreFailure } from './structured-agent-session-attach' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import type { JournalWriteOptions } from '../agent-session-journal/journal-database' const MAX_RECONCILIATION_PASSES = 8 @@ -21,12 +22,25 @@ export function createRestartReconciler(deps: { records: readonly AgentSessionRecord[] ) => Promise> now: () => number -}): (sessionId: string) => Promise { +}): ( + sessionId: string | null, + options?: JournalWriteOptions +) => Promise { let pending: Promise | null = null - return async (sessionId) => { + let background: Promise | null = null + return async (sessionId, options) => { if (!deps.store.listRecords().some((record) => record.lease.unreconciled)) { return null } + // Background bookkeeping joins a person's run, or another background one, never the reverse: + // no person's run inherits its refusal to wait for another connection's lock. + if (options?.background && !pending) { + background ??= reconcileCurrentLeases(deps, options).finally(() => { + background = null + }) + await background + return null + } if (!pending) { const run = reconcileCurrentLeases(deps) pending = run.finally(() => { @@ -37,11 +51,8 @@ export function createRestartReconciler(deps: { await pending return null } catch (error) { - return classifyStoreFailure( - error, - deps.store.getRecord(sessionId)?.lease.runtimeFence ?? null, - deps.store.getRecord(sessionId) - ) + const record = sessionId === null ? null : deps.store.getRecord(sessionId) + return classifyStoreFailure(error, record?.lease.runtimeFence ?? null, record) } } } @@ -54,8 +65,7 @@ export type ReaderBookkeepingFailures = { clear: () => void } -/** Startup and the read carry on past a failure: the next attach or send reconciles and resolves - * recovery again before it acts. */ +/** Startup and the read carry on past a failure: the host's retry tries it again with a backoff, and the next attach or send reconciles and resolves recovery again before it acts. */ export function reportEachFailureOnce( logger: StructuredAgentSessionLogger ): ReaderBookkeepingFailures { @@ -113,20 +123,26 @@ function failureKey(failure: unknown): string { return String(failure) } -async function reconcileCurrentLeases(deps: { - store: AgentSessionRecordStore - probe: (record: AgentSessionRecord) => Promise - probeMany?: ( - records: readonly AgentSessionRecord[] - ) => Promise> - now: () => number -}): Promise { +async function reconcileCurrentLeases( + deps: { + store: AgentSessionRecordStore + probe: (record: AgentSessionRecord) => Promise + probeMany?: ( + records: readonly AgentSessionRecord[] + ) => Promise> + now: () => number + }, + options?: JournalWriteOptions +): Promise { for (let pass = 0; pass < MAX_RECONCILIATION_PASSES; pass += 1) { - await deps.store.reconcileOnRestart({ - probe: deps.probe, - ...(deps.probeMany ? { probeMany: deps.probeMany } : {}), - now: deps.now() - }) + await deps.store.reconcileOnRestart( + { + probe: deps.probe, + ...(deps.probeMany ? { probeMany: deps.probeMany } : {}), + now: deps.now() + }, + options + ) if (!deps.store.listRecords().some((record) => record.lease.unreconciled)) { return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts index 1b74a23911f3..7df838d04e82 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.test.ts @@ -28,7 +28,7 @@ afterAll(() => { }) const NO_OPEN_DEPS = { - store: { getRecord: () => null, listRecords: () => [], replacedRuntime: () => undefined }, + store: { getRecord: () => null, listRecords: () => [] }, journalDatabase: openTestJournalHostDatabase(stateDirectory), logger: recordingStructuredAgentSessionLogger().logger } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 6c6efc49d728..3c4ca6fbffbe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -80,7 +80,8 @@ export async function restoreStructuredAgentSessionsOnRestart( } // One check for the pass. Each chat checks again while it holds, since another writer can mark // leases unreconciled mid-pass; after the first failure, retrying per chat only waits on the - // same store again, and the next attach or send settles those chats instead. + // same store again; the host's retry tries it once a round with a backoff instead, and the + // next attach or send reconciles before it acts. let settled = await input.reconcile(first.sessionId) const settleLeases = async (sessionId: string): Promise => { // A session latched in recovery exits here at startup, without waiting for a client. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts index e5fa9576f6ae..d2fcee941a45 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts @@ -155,11 +155,14 @@ export function journal( items: AgentJournalRenderItem[], submissions: AgentJournalSubmission[] = [] ) { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test calls only snapshot(), submissions() and appendItem(); a real AgentSessionJournal needs an on-disk SQLite store. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test calls only snapshot(), submissions(), appendItem() and the provenance reads; a real AgentSessionJournal needs an on-disk SQLite store. return { snapshot: () => ({ items, submissions, cursor: { epoch: EPOCH, sequence: items.length } }), submissions: () => submissions, - appendItem: async () => undefined + appendItem: async () => undefined, + // Created by the generation the record names: current while its lease is held. + itemFence: () => undefined, + wroteBeforeOpen: () => false } as never } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts index 5deb2e482db9..58b84c9a20a1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts @@ -27,6 +27,7 @@ import { REST_TEST_THREAD as THREAD, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const LAST_CHUNK_AT = HOST_TEST_NOW + 5_500 const RESTARTED_AT = HOST_TEST_NOW + 3_600_000 @@ -108,6 +109,11 @@ async function restartAfterCrash(): Promise { }) } +/** The host's retry has settled what the restart left. */ +function settled(): Promise { + return retryIdle(rig.host.collaboratorsForTests().reconciliation, SESSION) +} + async function expectSettledStream(): Promise { const snapshot = await rig.host.journalSnapshot(SESSION) expect( @@ -173,13 +179,14 @@ describe('a provider stream cut short before its next lease renewal', () => { it('retains the output bound when a client opens the journal before owner reconciliation', async () => { await streamBeforeCrash() await restartAfterCrash() - const beforeProof = await rig.host.journalSnapshot(SESSION) - expect( - beforeProof.items.some((item) => readAgentJournalTurn(item.body)?.state === 'unverifiable') - ).toBe(true) + // The open writes nothing; with its owner not yet reconciled the turn reads unverifiable. + const beforeProof = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(beforeProof.page.latestTurn?.turn.state).toBe('unverifiable') + expect(beforeProof.page.working).toBe(false) await rig.host.reconcileRestartLeases() await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) + await settled() await expectSettledStream() }) @@ -204,6 +211,7 @@ describe('a provider stream cut short before its next lease renewal', () => { reason: 'probe unavailable' }) await rig.host.restoreReadableSessions() + await settled() const snapshot = await rig.host.journalSnapshot(SESSION) const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts new file mode 100644 index 000000000000..00b2ee9954b9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts @@ -0,0 +1,110 @@ +// Startup owes a latched recovery only to the chats its restart restore makes readable, the visible +// tabs: deciding it may stop a surviving agent. When the restore's own reconcile or recovery write +// meets another connection's lock, the host's retry decides it instead, round after round, until it +// lands; and a chat a reader only opened behind them (an orchestration worker's history, a mailbox +// pointer, an outline, a naming read) is never decided, so its agent is never signalled. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + scanRecord as record, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-restore-recovery-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +const busy = () => + Object.assign(new Error('database is locked'), { errcode: 5, code: 'SQLITE_BUSY' }) + +/** A chat whose agent outlived the earlier process, and whose first `locked` reconciles meet a + * lock: its lease reconciles to `recovering`, and only stopping the agent decides it. */ +async function survivingAgent(locked: number) { + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) + await seedScanJournal(root, CHAT) + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const reconcile = vi.spyOn(store, 'reconcileOnRestart') + for (let index = 0; index < locked; index += 1) { + reconcile.mockRejectedValueOnce(busy()) + } + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + const probeOwner = async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + return { current: host, stopOwnerProcess } +} + +it("never stops a hidden chat's agent that readers only read, after reconciles a lock refused", async () => { + const { current, stopOwnerProcess } = await survivingAgent(2) + await current.reconcileRestartLeases() + expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(true) + + // Every reader that opens a chat no tab shows. + await current.history({ sessionId: CHAT, direction: 'tail' }, 'own-agent') // worker-show + await current.journalSnapshot(CHAT) // a mailbox pointer + await current.journalSnapshot(CHAT) // the conversation outline + await current.journalSnapshot(CHAT) // a naming read + await vi.waitFor(() => expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(false), { + timeout: 8_000 + }) + // Rounds enough for anything that would decide it. + await new Promise((resolve) => setTimeout(resolve, 3_000)) + + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') +}, 20_000) + +it("retries a visible chat's recovery its write could not land, until it does", async () => { + // Startup's, the restore's and the first round's reconciles meet the lock. + const { current, stopOwnerProcess } = await survivingAgent(3) + const transition = store.transitionHandoff + let refused = 0 + vi.spyOn(store, 'transitionHandoff').mockImplementation((...args) => { + // The lock is back for the recovery's own write, once. + if (args[2]?.background && refused === 0) { + refused += 1 + return Promise.reject(busy()) + } + return transition(...args) + }) + + await current.reconcileRestartLeases() + await current.restoreReadableSessions([CHAT]) + + await vi.waitFor(() => expect(refused).toBe(1), { timeout: 8_000 }) + await vi.waitFor(() => expect(store.getRecord(CHAT)?.lease.handoffStage).toBeNull(), { + timeout: 8_000 + }) + expect(stopOwnerProcess).toHaveBeenCalled() + expect(current.currentWork(CHAT)?.working()).toBe(false) +}, 30_000) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts index 93a7962d4a20..2419651b7143 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts @@ -12,17 +12,32 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { randomUUID } from 'node:crypto' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionEndedChild } from './structured-agent-session-host-types' +import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' -export function isResumableStructuredAgentSessionRecord(record: AgentSessionRecord): boolean { - return agentSessionLeaseIsReleased(record.lease) +/** Released, or still held by a child this host saw end at the lease's own fence (`ended`, its + * `lastEndedChild`), whose release write did not land: the acquisition probes that owner and + * replaces it only once proven dead. Nobody else's exit widens this, so an SSH or unverifiable + * owner is refused as before. */ +export function isResumableStructuredAgentSessionRecord( + record: AgentSessionRecord, + ended?: Pick +): boolean { + return ( + agentSessionLeaseIsReleased(record.lease) || + (ended?.rootGone === true && + ended.fence === record.lease.runtimeFence && + isSurfaceReleasableAgentSessionRecord(record)) + ) } /** Attach params for a resume, or null when this record's lease is somebody else's problem. */ export function structuredAgentSessionResumeParams( record: AgentSessionRecord, - clientOperationId: string + clientOperationId: string, + ended?: Pick ): AgentSessionAttachParams | null { - if (!isResumableStructuredAgentSessionRecord(record)) { + if (!isResumableStructuredAgentSessionRecord(record, ended)) { return null } return attachParamsForRecord(record, { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts new file mode 100644 index 000000000000..3b4990e40ecb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts @@ -0,0 +1,78 @@ +// The retry's budget counts rounds, never collisions: many chats whose automatic sends meet one +// lock at once cost one failed round, not one each, and the work lands once the lock lifts. + +import { afterEach, expect, it, vi } from 'vitest' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' +import { + StructuredAgentSessionRetry, + type StructuredAgentSessionRetryContext +} from './structured-agent-session-reconciliation-retry' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +async function settleTurns(): Promise { + for (let turn = 0; turn < 60; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + +it('ten chats whose automatic sends meet one lock cost one round, and send once it lifts', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + vi.spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass').mockResolvedValue({ + failed: [], + wrote: false, + recovering: false + }) + const ids = Array.from({ length: 10 }, (_, index) => `chat-${index}`) + let locked = true + const sendQueued = vi.fn(async () => (locked ? ('contended' as const) : ('done' as const))) + const abandonSend = vi.fn() + const context = { + deps: { + store: { + onGenerationEnded: () => () => undefined, + getRecord: () => ({ lease: { unreconciled: false, runtimeFence: 1 } }), + listRecords: () => [], + readOnly: false + }, + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() } + }, + sessions: new Map( + ids.map((id) => [id, { journal: { openedAt: () => ({ epoch: 'e', sequence: 0 }) } }]) + ), + now: () => 0, + serialize: (_id: string, task: () => Promise) => task(), + laneBusy: () => false, + track: (operation: Promise) => operation, + publishGenerationEnded: vi.fn(), + reconcileOwed: () => false, + reconcile: async () => 'settled' as const, + sendQueued, + abandonSend + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retry reads only the fields above; the host's full deps would add nothing this test exercises. + const typed = context as unknown as StructuredAgentSessionRetryContext + const retry = new StructuredAgentSessionRetry(typed) + + // Each chat's own drain met the lock at the same moment. + for (const id of ids) { + retry.signal(id, { contended: true }) + } + await settleTurns() + // One round, ended by the first send that met the lock: one failure, nobody given up on. + expect(sendQueued).toHaveBeenCalledTimes(1) + expect(abandonSend).not.toHaveBeenCalled() + expect(retry['failures']).toBe(1) + + locked = false + await vi.advanceTimersByTimeAsync(2_000) + await settleTurns() + expect(sendQueued).toHaveBeenCalledTimes(11) + expect(ids.filter((id) => retryOwes(retry, id))).toEqual([]) + expect(abandonSend).not.toHaveBeenCalled() + retry.dispose() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts new file mode 100644 index 000000000000..e3810b04896a --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts @@ -0,0 +1,203 @@ +// What a round of the host's retry touches: a chat's replay is read once however many rounds a +// lock refuses, a latch that comes after the restart restore waits for its own chat, and nothing +// the store-wide step cannot decide holds the other chats back or is probed round after round. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import * as reconciliationLoad from './structured-agent-session-reconciliation-load' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' +import { + openScanHost, + scanIdle, + scanRecord, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' +const OTHER = 'chat-aaaaaaa2' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-retry-scope-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +const busy = () => + Object.assign(new Error('database is locked'), { errcode: 5, code: 'SQLITE_BUSY' }) +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** Records and journals for `ids`; `alive` names chats whose agent outlived the earlier process. */ +async function seed(ids: readonly string[], alive: readonly string[] = []) { + await seedTestAgentSessionRecordStore(root, { + records: ids.map((id) => scanRecord(id, !alive.includes(id))) + }) + for (const id of ids) { + await seedScanJournal(root, id) + } + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) +} + +/** Every pass on `contended` chats meets a lock while `held()`; the rest run as they are. */ +function lockPasses(held: () => boolean, contended: (sessionId: string) => boolean) { + const pass = reconciliationPass.runStructuredAgentSessionReconciliationPass + return vi + .spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass') + .mockImplementation(async (...args) => + held() && contended(args[1]) + ? { failed: [busy()], wrote: false, recovering: false } + : pass(...args) + ) +} + +function surviving() { + let alive = true + const probeOwner = vi.fn(async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + ) + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + return { probeOwner, stopOwnerProcess } +} + +const latch = (lease: Partial) => (record: AgentSessionRecord) => ({ + ...record, + lease: { ...record.lease, handoffStage: 'recovering' as const, ...lease } +}) + +it('replays each of 20 closed chats once while a lock refuses round after round', async () => { + const chats = Array.from({ length: 20 }, (_, i) => `chat-r${String(i).padStart(6, '0')}`) + await seed(chats) + let held = true + const passes = lockPasses( + () => held, + () => true + ) + const loads = vi.spyOn(reconciliationLoad, 'loadStructuredAgentSessionForReconciliation') + host = openScanHost(root, store) + await host.reconcileRestartLeases() + + // Rounds at 0 s, 1 s, 3 s and 5 s, each stopped by the lock. + await pause(5_500) + expect(passes.mock.calls.length).toBeGreaterThanOrEqual(3) + held = false + await scanIdle(host, chats) + + expect(loads.mock.calls.length).toBeLessThanOrEqual(20) +}, 30_000) + +it("never signals a restored chat's agent for a latch that came after the restore", async () => { + await seed([CHAT, OTHER]) + const { probeOwner, stopOwnerProcess } = surviving() + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + await host.reconcileRestartLeases() + await host.restoreReadableSessions([CHAT]) + await scanIdle(host, [CHAT, OTHER]) + const probes = probeOwner.mock.calls.length + + // Mid-session its lease latches, its owner alive; another chat's rounds run. + const { ownerProcess, reservedSpawnToken } = scanRecord(CHAT, false).lease + await store.transitionHandoff( + CHAT, + latch({ ownerProcess, reservedSpawnToken, claimStatus: 'reserved' }) + ) + let held = true + lockPasses( + () => held, + (sessionId) => sessionId === OTHER + ) + host.collaboratorsForTests().reconciliation.signal(OTHER) + await pause(3_500) + held = false + await scanIdle(host, [OTHER]) + + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(probeOwner).toHaveBeenCalledTimes(probes) + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') +}, 30_000) + +it('on a read-only store, attempts no store-wide step and still visits the other chats', async () => { + await seed([CHAT, OTHER], [CHAT]) + // A newer Orca wrote the database: a latched lease here stays latched. + Object.defineProperty(openTestJournalHostDatabase(root), 'readOnly', { value: true }) + const { probeOwner, stopOwnerProcess } = surviving() + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + await host.reconcileRestartLeases() + await host.restoreReadableSessions([CHAT]) + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') + const reconcile = vi.spyOn(store, 'reconcileOnRestart') + const probes = probeOwner.mock.calls.length + const stops = stopOwnerProcess.mock.calls.length + + host.collaboratorsForTests().reconciliation.signal(OTHER) + await scanIdle(host, [CHAT, OTHER]) + await pause(3_500) + + expect(reconcile).not.toHaveBeenCalled() + expect(probeOwner).toHaveBeenCalledTimes(probes) + expect(stopOwnerProcess).toHaveBeenCalledTimes(stops) +}, 30_000) + +it.each([ + ['a conflicted claim', { claimStatus: 'conflicted' as const }], + [ + 'an owner on another host', + { + ownerProcess: { hostId: 'other', pid: 12_000, processStartTimeMs: 0, spawnToken: 'spawn' } + } + ] +])( + 'never re-probes %s round after round', + async (_name, lease) => { + await seed([CHAT, OTHER], [CHAT]) + const { probeOwner, stopOwnerProcess } = surviving() + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + await host.reconcileRestartLeases() + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') + await store.transitionHandoff(CHAT, latch(lease)) + await host.restoreReadableSessions([CHAT]) + await scanIdle(host, [CHAT]) + const probes = probeOwner.mock.calls.length + + // Another chat's rounds, refused by a lock at 0 s, 1 s and 3 s. + let held = true + lockPasses( + () => held, + (sessionId) => sessionId === OTHER + ) + host.collaboratorsForTests().reconciliation.signal(OTHER) + await pause(3_500) + held = false + await scanIdle(host, [OTHER]) + + expect(probeOwner).toHaveBeenCalledTimes(probes) + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') + }, + 30_000 +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts new file mode 100644 index 000000000000..c7e47ef25abd --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts @@ -0,0 +1,219 @@ +// The retry's one wake and one budget: a commit after a lock refused a round runs the next one at +// once; reads during a backoff add no round and leave the budget alone; after a give-up only a +// newly owed chat or a commit starts a fresh episode. + +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as reconciliationLoad from './structured-agent-session-reconciliation-load' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' +import { RECONCILIATION_GIVE_UP_MS } from './structured-agent-session-reconciliation-backoff' +import { + StructuredAgentSessionRetry, + type StructuredAgentSessionRetryContext +} from './structured-agent-session-reconciliation-retry' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +const CHAT = 'chat-1' +const journal = { openedAt: () => ({ epoch: 'e', sequence: 0 }) } +const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) + +let lockHeld = true +let pass: ReturnType + +beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) + lockHeld = true + pass = vi + .spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass') + .mockImplementation(async () => ({ + failed: lockHeld ? [locked()] : [], + wrote: false, + recovering: false + })) +}) + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +async function settleTurns(): Promise { + for (let turn = 0; turn < 30; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + +/** `busy`: whether a person's operation holds the chat's lane. `closed`: no reader has it open. */ +function openRetry( + options: { busy?: () => boolean; closed?: true } = {} +): StructuredAgentSessionRetry { + const context = { + deps: { + store: { + onGenerationEnded: () => () => undefined, + getRecord: () => ({ lease: { unreconciled: false, runtimeFence: 1 } }), + listRecords: () => [], + readOnly: false + }, + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() } + }, + sessions: new Map(options.closed ? [] : [[CHAT, { journal }]]), + now: () => 0, + serialize: (_id: string, task: () => Promise) => task(), + laneBusy: () => options.busy?.() ?? false, + track: (operation: Promise) => operation, + publishGenerationEnded: vi.fn(), + reconcileOwed: () => false, + reconcile: async () => 'settled' as const, + sendQueued: async () => 'done' as const, + abandonSend: vi.fn() + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retry reads only the fields above; the host's full deps would add nothing these tests exercise. + const typed = context as unknown as StructuredAgentSessionRetryContext + return new StructuredAgentSessionRetry(typed) +} + +/** Rounds refused by the lock until the episode gives up: no timer is left. */ +async function exhaust(retry: StructuredAgentSessionRetry): Promise { + retry.signal(CHAT) + await settleTurns() + for (let elapsed = 0; elapsed < RECONCILIATION_GIVE_UP_MS + 4_000; elapsed += 2_000) { + await vi.advanceTimersByTimeAsync(2_000) + await settleTurns() + } + expect(retry['timer']).toBeNull() +} + +it('a commit after a lock refused a round runs the next one at once, with no timer', async () => { + const retry = openRetry() + retry.signal(CHAT) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(1) + + lockHeld = false + retry.observeCommit('another-chat', null) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(2) + expect(retryOwes(retry, CHAT)).toBe(false) + retry.dispose() +}) + +it('reads during a backoff add no round and leave the budget as it is', async () => { + const retry = openRetry() + retry.signal(CHAT) + await settleTurns() + const waited = retry['failingFor']() + + for (let read = 0; read < 20; read += 1) { + retry.opened(CHAT, journal) + } + await settleTurns() + + expect(pass.mock.calls.length).toBeLessThanOrEqual(2) + expect(retry['failingFor']()).toBe(waited) + retry.dispose() +}) + +it('before a give-up, a signal, a commit or a read leaves the budget counting', async () => { + const retry = openRetry() + retry.signal(CHAT) + await settleTurns() + await vi.advanceTimersByTimeAsync(3_000) + await settleTurns() + const waited = retry['failingFor']() + expect(waited).toBeGreaterThan(0) + + retry.signal(CHAT) + retry.observeCommit('another-chat', null) + retry.opened(CHAT, journal) + await settleTurns() + + expect(retry['failingFor']()).toBeGreaterThanOrEqual(waited) + retry.dispose() +}) + +it('after a give-up only a newly owed chat or a commit starts a fresh episode', async () => { + const retry = openRetry() + await exhaust(retry) + const given = pass.mock.calls.length + // Nothing of its own: no timer, however long. + await vi.advanceTimersByTimeAsync(600_000) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(given) + + retry.signal(CHAT) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(given + 1) + expect(retry['failingFor']()).toBeLessThan(RECONCILIATION_GIVE_UP_MS) + + await exhaust(retry) + const again = pass.mock.calls.length + lockHeld = false + retry.observeCommit('another-chat', null) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(again + 1) + expect(retryOwes(retry, CHAT)).toBe(false) + retry.dispose() +}) + +it('keeps the budget through a round whose only chat waited on a busy lane', async () => { + let busy = false + const retry = openRetry({ busy: () => busy }) + retry.signal(CHAT) + await settleTurns() + await vi.advanceTimersByTimeAsync(500) + expect(retry['failures']).toBe(1) + + // The lock lifts, but a person's operation holds the lane: no progress, so no reset. + lockHeld = false + busy = true + retry.observeCommit('another-chat', null) + await settleTurns() + + expect(pass).toHaveBeenCalledTimes(1) + expect(retry['failures']).toBe(1) + retry.dispose() +}) + +it('drops a closed chat it loaded when its episode gives up, and keeps it owed', async () => { + const close = vi.fn(async () => undefined) + const loads = vi + .spyOn(reconciliationLoad, 'loadStructuredAgentSessionForReconciliation') + .mockResolvedValue({ + kind: 'loaded', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the visit reads only the journal's open cursor and close of a loaded read. + session: { journal: { ...journal, close } } as never + }) + vi.spyOn(reconciliationLoad, 'structuredAgentSessionJournalIsCurrent').mockReturnValue(true) + const retry = openRetry({ closed: true }) + + await exhaust(retry) + + expect(loads).toHaveBeenCalledTimes(1) + expect(close).toHaveBeenCalledOnce() + expect(retry['owed'].get(CHAT)?.loaded).toBeUndefined() + expect(retryOwes(retry, CHAT)).toBe(true) + retry.dispose() +}) + +it('gives up after about 3 minutes of wall time, however often commits wake it early', async () => { + const retry = openRetry() + const warn = vi.mocked(retry['context'].deps.logger.warn) + const gaveUp = () => warn.mock.calls.filter(([message]) => message.startsWith('gave up')).length + retry.signal(CHAT) + await settleTurns() + // A commit every 100 ms: each ends the backoff and runs a refused round at once. + for (let elapsed = 0; elapsed < 170_000; elapsed += 100) { + retry.observeCommit('another-chat', null) + await settleTurns() + await vi.advanceTimersByTimeAsync(100) + } + expect(gaveUp()).toBe(0) + + for (let elapsed = 0; elapsed < 12_000; elapsed += 100) { + retry.observeCommit('another-chat', null) + await settleTurns() + await vi.advanceTimersByTimeAsync(100) + } + expect(gaveUp()).toBe(1) + retry.dispose() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts new file mode 100644 index 000000000000..bfa96e55d5f6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts @@ -0,0 +1,22 @@ +// What a test may ask of the host's retry (`StructuredAgentSessionRetry`) that no production caller +// does: whether a chat is still owed, and waiting until it is not. + +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' + +/** Whether a visit is owed: something was signalled, or failed, and has not settled. */ +export function retryOwes(retry: StructuredAgentSessionRetry, sessionId: string): boolean { + return retry['owed'].get(sessionId)?.due === true +} + +/** Resolves once the chat owes nothing, or a failed round left it waiting out a backoff. */ +export async function retryIdle( + retry: StructuredAgentSessionRetry, + sessionId: string +): Promise { + while ( + retryOwes(retry, sessionId) && + (retry['running'] || retry['timer'] === 'soon' || retry['failures'] === 0) + ) { + await retry.attempted(sessionId) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 9a6e3aca5917..b3e4c9bbfff7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -13,6 +13,13 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { journalOpenRefusal } from '../agent-session-journal/journal-open-failure' +import type { JournalWriteOptions } from '../agent-session-journal/journal-host-database' +import { + BACKGROUND_WRITE, + backgroundLeaseWrites, + type BackgroundLeaseWrites +} from './structured-agent-session-background-writes' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' import { @@ -23,6 +30,12 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal } from './structured-agent-session-host-types' +import { scanStructuredAgentSessionsAtStartup } from './structured-agent-session-startup-settlement' +import { + StructuredAgentSessionRetry, + type StructuredAgentSessionRetryContext +} from './structured-agent-session-reconciliation-retry' +import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' /** Throws its refusal as the code itself. */ export async function revealStructuredAgentSession( @@ -52,33 +65,117 @@ export async function revealStructuredAgentSession( } } -/** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. - * Its lease bookkeeping is a reader's, which never fails a read or startup; startup shares it. */ +/** The host's startup and its reconciliation: reconcile, then hand every chat to the host's retry + * in the background (no chat's tab or send waits on it), and the readable-restore sweep that + * resolves and opens each visible chat. Its lease bookkeeping is a reader's, which never fails a + * read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], 'openDeps' | 'reconcile' | 'resolveRecovery' > & { - reconcileLeases: (sessionId: string) => Promise - resolveRecovery: (sessionId: string) => Promise + reconcileLeases: ( + sessionId: string | null, + options?: JournalWriteOptions + ) => Promise + resolveRecovery: (sessionId: string, writes: BackgroundLeaseWrites) => Promise + startup: Pick & { + tasks: Pick + clientDelivery: Pick + queue: { + sendForRetry: (sessionId: string) => Promise<'contended' | 'done'> + abandon: (sessionId: string) => void + } + } } ): { reconcileRestartLeases: () => Promise + /** The startup scan the last reconcile began; resolved once every chat's first attempt ran. */ + startupSettled: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise + reconciliation: StructuredAgentSessionRetry } { - const { reconcileLeases, resolveRecovery, ...rest } = wiring + const { reconcileLeases, resolveRecovery, startup, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) - const reconcile = createReaderReconcile(reconcileLeases, failures) + // Startup's and a restored read's bookkeeping, which neither waits on: another connection's lock + // fails it at once, and the retry's round tries it again. + const reconcile = createReaderReconcile( + (sessionId) => reconcileLeases(sessionId, BACKGROUND_WRITE), + failures + ) + const recoveryWrites = backgroundLeaseWrites(deps.store) + // The visible tabs the restart restore was asked for whose leases it found latched or not yet + // reconciled: theirs is the one recovery decision startup owes. One decided, or found unlatched, + // leaves; a latch that comes later waits for its chat's own attach, start or send, as on main. + const restoreTargets = new Set() + let restoring = false + const latchedTargets = () => { + for (const sessionId of restoreTargets) { + const lease = deps.store.getRecord(sessionId)?.lease + if (!lease?.unreconciled && lease?.handoffStage !== 'recovering') { + restoreTargets.delete(sessionId) + } + } + // Only one a probe can decide: never a conflicted claim, nor an owner on another host. + return [...restoreTargets].filter((sessionId) => { + const lease = deps.store.getRecord(sessionId)?.lease + return ( + lease?.handoffStage === 'recovering' && + lease.claimStatus !== 'conflicted' && + (lease.ownerProcess?.hostId ?? deps.store.hostId) === deps.store.hostId + ) + }) + } + const reconciliation = new StructuredAgentSessionRetry({ + deps, + sessions: startup.sessions, + now: () => deps.now?.() ?? Date.now(), + serialize: rest.serialize, + laneBusy: (sessionId) => startup.tasks.busy(sessionId), + // Tracked like a start, so a quit waits for an attempt before it closes what it writes to. + track: (operation) => startup.tasks.trackAttach(operation), + publishGenerationEnded: (sessionId, options) => + startup.clientDelivery.publishGenerationEnded(sessionId, options), + // A read-only store decides nothing here; its chats' visits still run. + reconcileOwed: () => + !deps.store.readOnly && + (deps.store.listRecords().some((record) => record.lease.unreconciled) || + latchedTargets().length > 0), + reconcile: async () => { + try { + const refusal = await reconcileLeases(null, BACKGROUND_WRITE) + if (refusal) { + failures.report(refusal) + return 'failed' + } + // The restore's own decision, wherever its reconcile or its write met a lock. + for (const sessionId of latchedTargets()) { + await resolveRecovery(sessionId, recoveryWrites) + } + latchedTargets() // a decided target leaves at once + failures.clear() + return 'settled' + } catch (error) { + failures.report(error) + return isSqliteContentionFailure(error) ? 'contended' : 'failed' + } + }, + sendQueued: (sessionId) => startup.queue.sendForRetry(sessionId), + abandonSend: (sessionId) => startup.queue.abandon(sessionId) + }) + let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: deps, reconcile, // The next attach or send resolves recovery again, strictly, before it acts. resolveRecovery: (sessionId) => - resolveRecovery(sessionId).then( + resolveRecovery(sessionId, recoveryWrites).then( () => true, (error: unknown) => { failures.report(error) + // The retry's round decides it instead (`reconcileOwed`). + reconciliation.signal(sessionId) return false } ), @@ -88,7 +185,22 @@ export function createStructuredAgentSessionHostRestore( return { reconcileRestartLeases: async () => { await reconcile('startup') + startupSettled = scanStructuredAgentSessionsAtStartup(deps.store, reconciliation) + }, + startupSettled: () => startupSettled, + restoreReadableSessions: (sessionIds) => { + // Once, as the restorer restores once. + for (const { sessionId, lease } of restoring ? [] : deps.store.listRecords()) { + if ( + (lease.unreconciled || lease.handoffStage === 'recovering') && + (!sessionIds || sessionIds.includes(sessionId)) + ) { + restoreTargets.add(sessionId) + } + } + restoring = true + return gate.run(() => restorer.restore(sessionIds)) }, - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)) + reconciliation } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts new file mode 100644 index 000000000000..6ed2a1d09b77 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts @@ -0,0 +1,154 @@ +// A rewind after a failed cleanup: the dead generation's turn and prompt are still saved as +// running and pending, and the rewind rebuilds the epoch at the live fence. Through the rewind's +// own row builders and a real SQLite journal, each carries the old fold's provenance, so the host +// projection never promotes that work to the live generation's. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { + mergeRetainedHostLifecycleRows, + retainedRowReplacement, + rewindProviderRows +} from './structured-rewind-retained-host-rows' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: codexProviderHandle('thread-1') +} +const DEAD = 1 +const LIVE = 3 + +function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-1', turnId, ordinal } +} +const TURN = codexItem('turn-dead', 0) +const PROMPT = codexItem('turn-dead', 1) +const REPLY = codexItem('turn-dead', 2) +const LATER = codexItem('turn-live', 0) +const PENDING = { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null +} +const REPLY_BODY: AgentJournalItemBody = { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: 'partial' }] +} + +const LATER_BODY: AgentJournalItemBody = { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: 'mine' }] +} + +let root: string +const journals = createTrackedJournalOpener() + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-rewind-provenance-')) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +/** The dead generation's work as its cleanup left it: nothing settled. */ +async function leftByDeadGeneration(): Promise { + const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) + const scope = { fence: DEAD, ownerFence: DEAD, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + await journal.appendItem(TURN, { kind: 'turn', turnId: 'turn-dead', state: 'running' }, scope) + const inTurn = { + ...scope, + turnScope: { kind: 'turn' as const, turnItemId: agentJournalItemKey(TURN) } + } + await journal.appendItem( + PROMPT, + { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: PENDING + }, + inTurn + ) + await journal.appendItem(REPLY, REPLY_BODY, inTurn) + return journal +} + +/** The rewind's rebuild, as `rewindStructuredAgentSession` and its recovery write it. */ +async function rewindAtLiveFence(journal: AgentSessionJournal): Promise { + const retained = journal.snapshot().items.map(({ itemId, observedAt, turnScope }) => { + const ownerFence = journal.itemFence(itemId) + return { + itemId, + body: journal.itemBody(itemId)!, + observedAt, + ...(turnScope ? { turnScope } : {}), + ...(ownerFence === undefined ? {} : { ownerFence }) + } + }) + // The live agent's history names the dead turn's prompt and reply it still holds, and one of + // its own. + const provider = rewindProviderRows( + [ + { identity: PROMPT, body: journal.itemBody(agentJournalItemKey(PROMPT))! }, + { identity: REPLY, body: REPLY_BODY }, + { identity: LATER, body: LATER_BODY } + ], + 2_000, + LIVE + ) + const merged = mergeRetainedHostLifecycleRows(retained, provider) + await journal.replaceEpochItems('handle_forked', LIVE, merged.map(retainedRowReplacement)) +} + +describe('a rewind after a failed cleanup', () => { + it("does not promote the dead generation's turn, prompt or reply to the live one", async () => { + const journal = await leftByDeadGeneration() + const before = journal.epoch + + await rewindAtLiveFence(journal) + + expect(journal.epoch).not.toBe(before) + const work = new StructuredAgentSessionCurrentWork(journal, LIVE) + // Rebuilt at the live fence, still the dead generation's work. + expect(journal.itemFence(agentJournalItemKey(TURN))).toBe(DEAD) + expect(journal.itemFence(agentJournalItemKey(PROMPT))).toBe(DEAD) + expect(journal.itemFence(agentJournalItemKey(REPLY))).toBe(DEAD) + expect(work.activeTurnId()).toBeNull() + expect(work.actionablePromptIds()).toEqual([]) + expect(work.working()).toBe(false) + // What the live agent itself reported is its own. + expect(work.isCurrentItem(agentJournalItemKey(LATER))).toBe(true) + }) + + it('leaves a generation that is still live holding its own work', async () => { + const journal = await leftByDeadGeneration() + + await rewindAtLiveFence(journal) + + const work = new StructuredAgentSessionCurrentWork(journal, DEAD) + expect(work.activeTurnId()).toBe('turn-dead') + expect(work.actionablePromptIds()).toEqual([agentJournalItemKey(PROMPT)]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index bd819d09f2ca..4c48b155598c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -2,7 +2,6 @@ import { agentSessionProviderHandleChainHead } from '../../../shared/agent-sessi import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import { - agentJournalItemKey, agentJournalSubmissionKey, parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' @@ -19,11 +18,15 @@ import { openWithAgent } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' import { admitAndRunAgentSessionMutation } from './structured-agent-session-mutation-admission' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { conversationCommandBlocked } from './structured-conversation-command-admission' import { rewindRefusal } from './structured-rewind-refusal' import { renameRewindTurnOpener } from './structured-rewind-journal-body' import { persistRewindRecord, recoverStructuredRewind } from './structured-rewind-recovery' -import { mergeRetainedHostLifecycleRows } from './structured-rewind-retained-host-rows' +import { + mergeRetainedHostLifecycleRows, + rewindProviderRows +} from './structured-rewind-retained-host-rows' export async function rewindStructuredAgentSession( context: StructuredAgentSessionMutationContext, @@ -58,6 +61,8 @@ export async function rewindStructuredAgentSession( : { ok: true } }), journal: () => context.sessions.get(sessionId)?.journal, + currentWork: () => + hostStructuredAgentSessionCurrentWork({ store, sessions: context.sessions }, sessionId), publish: (journal) => context.publish(sessionId, journal), now: context.now, plan: { @@ -165,11 +170,13 @@ export async function rewindStructuredAgentSession( if (!body) { throw new Error('agent_session_rewind:missing-retained-item') } + const ownerFence = ctx.journal.itemFence(itemId) return { itemId: providerKey(itemId), body: renameRewindTurnOpener(body, providerKey), observedAt, ...(turnScope ? { turnScope } : {}), + ...(ownerFence === undefined ? {} : { ownerFence }), ...agentJournalLinkageFields(linkage) } }) @@ -204,11 +211,7 @@ export async function rewindStructuredAgentSession( onPrepared: async (items) => { const retained = mergeRetainedHostLifecycleRows( prepared.retained, - items.map(({ identity, body }) => ({ - itemId: agentJournalItemKey(identity), - body, - observedAt: ctx.now() - })) + rewindProviderRows(items, ctx.now(), ctx.fence) ) if ( retained.length > 10_000 || @@ -241,11 +244,7 @@ export async function rewindStructuredAgentSession( const confirmed = provider.items ? mergeRetainedHostLifecycleRows( prepared.retained, - provider.items.map(({ identity, body }) => ({ - itemId: agentJournalItemKey(identity), - body, - observedAt: ctx.now() - })) + rewindProviderRows(provider.items, ctx.now(), ctx.fence) ) : prepared.retained if ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts new file mode 100644 index 000000000000..ff3e56536359 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts @@ -0,0 +1,280 @@ +// A roster row and a background-task row are revised in place across generations, so each entry +// carries the generation that last observed it. Settling an ended generation settles only the +// entries it observed; a later live generation's children stay as they are. And a proof the +// retry holds for an ended generation revises what an earlier settle of it could +// only call unverifiable. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody +} from '../../../shared/agent-session-journal-types' +import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../../shared/agent-session-record.test-fixture' +import { backgroundTaskJournalBody } from '../../../shared/native-chat-background-task-row' +import { + isBackgroundTaskBlock, + isSubagentGroupBlock, + type NativeChatBackgroundTaskBlock, + type NativeChatSubagentEntry +} from '../../../shared/native-chat-types' +import { + claudeBackgroundTaskIdentity, + claudeBackgroundTaskBody +} from '../../claude/claude-background-task-row-journal' +import { claudeSubagentGroupIdentity } from '../../claude/claude-subagent-journaled-roster' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import { subagentGroupJournalBody } from '../agent-session-journal/journal-subagent-group-body' +import { stampJournalEntryProvenance } from '../agent-session-journal/journal-item-provenance' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { createDeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' +import { testEventSinkLogging } from './structured-agent-session-logger-test-support' +import { backgroundSettlementWrites } from './structured-agent-session-background-writes' + +const SESSION = 'orca-session' +const GROUP = 'claude-session:turn-a' +const ENDED = 1 +const LIVE = 3 + +const journals = createTrackedJournalOpener() +let root: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-roster-provenance-')) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +function child(id: string, extra: Partial = {}): NativeChatSubagentEntry { + return { id, label: `Child ${id}`, state: 'working', startedAt: 1, ...extra } +} + +function task( + taskId: string, + extra: Partial = {} +): NativeChatBackgroundTaskBlock { + return { + type: 'background-task', + taskId, + kind: 'command', + label: `Task ${taskId}`, + state: 'working', + startedAt: 1, + ...extra + } +} + +/** One provider process of generation `fence`, observing rows through the host's event sink. */ +async function observeAt( + journal: AgentSessionJournal, + fence: number, + rows: readonly (readonly [ReturnType, AgentJournalItemBody])[] +): Promise { + const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) + deferred.bind({ journal, fence, publish: () => {} }) + for (const [identity, body] of rows) { + deferred.sink.appendItem(identity, body, { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) + } + await expect(deferred.drained()).resolves.toEqual({ ok: true }) + deferred.close() +} + +function children(journal: AgentSessionJournal): NativeChatSubagentEntry[] { + return journal + .snapshot() + .items.flatMap((item) => + item.body.kind === 'message' + ? item.body.blocks.flatMap((block) => (isSubagentGroupBlock(block) ? block.agents : [])) + : [] + ) +} + +function tasks(journal: AgentSessionJournal): NativeChatBackgroundTaskBlock[] { + return journal + .snapshot() + .items.flatMap((item) => + item.body.kind === 'message' ? item.body.blocks.filter(isBackgroundTaskBlock) : [] + ) +} + +function openJournal(): Promise { + return journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'claude', + providerHandle: claudeProviderHandle('claude-session', null) + }, + now: () => 9_000, + stateDirectory: root + }) +} + +describe('a roster entry’s provenance', () => { + const group = (agents: NativeChatSubagentEntry[]) => subagentGroupJournalBody(GROUP, agents) + const agentsOf = (body: AgentJournalItemBody) => + body.kind === 'message' + ? body.blocks.flatMap((b) => (isSubagentGroupBlock(b) ? b.agents : [])) + : [] + + it('is the observing generation’s for a new or changed entry, and kept for one carried unchanged', () => { + const before = group([child('a', { ownerFence: ENDED }), child('b', { ownerFence: ENDED })]) + const after = stampJournalEntryProvenance( + group([child('a'), child('b', { state: 'completed' }), child('c')]), + { body: before, ownerFence: ENDED }, + LIVE + ) + expect(agentsOf(after).map(({ id, ownerFence }) => [id, ownerFence])).toEqual([ + ['a', ENDED], + ['b', LIVE], + ['c', LIVE] + ]) + }) + + it('falls back to the row’s own provenance for an entry an older build never stamped', () => { + const before = group([child('a')]) + const after = stampJournalEntryProvenance( + group([child('a')]), + { body: before, ownerFence: 2 }, + LIVE + ) + expect(agentsOf(after)[0]?.ownerFence).toBe(2) + }) + + it('keeps a stamp history carried into a new epoch already names', () => { + const after = stampJournalEntryProvenance( + group([child('a', { ownerFence: ENDED })]), + undefined, + LIVE + ) + expect(agentsOf(after)[0]?.ownerFence).toBe(ENDED) + }) +}) + +describe('settling an ended generation’s roster', () => { + it('settles only the children and tasks it observed; the live generation’s stay working', async () => { + const journal = await openJournal() + // Generation F spawns a child and starts two background commands, then dies uncleaned. + await observeAt(journal, ENDED, [ + [claudeSubagentGroupIdentity(GROUP), subagentGroupJournalBody(GROUP, [child('a')])], + [claudeBackgroundTaskIdentity('bg-1'), claudeBackgroundTaskBody(task('bg-1'))], + [claudeBackgroundTaskIdentity('bg-2'), claudeBackgroundTaskBody(task('bg-2'))] + ]) + // Generation F+1 resumes the same group row with a new child of its own, carrying F's child + // unchanged, and reports progress on one of F's commands, which it now runs. + await observeAt(journal, LIVE, [ + [ + claudeSubagentGroupIdentity(GROUP), + subagentGroupJournalBody(GROUP, [child('a'), child('b')]) + ], + [ + claudeBackgroundTaskIdentity('bg-2'), + backgroundTaskJournalBody(task('bg-2', { tokens: 50 })) + ] + ]) + expect(children(journal).map(({ id, ownerFence }) => [id, ownerFence])).toEqual([ + ['a', ENDED], + ['b', LIVE] + ]) + expect(tasks(journal).map(({ taskId, ownerFence }) => [taskId, ownerFence])).toEqual([ + ['bg-1', ENDED], + ['bg-2', LIVE] + ]) + + // F+1 holds the lease, live; the settlement judges everything below it. + const live = agentSessionRecordFixture( + agentSessionLeaseFixture({ sessionId: SESSION, runtimeFence: LIVE }) + ) + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: (sessionId) => (sessionId === SESSION ? live : null) }, + sessionId: SESSION, + journal, + writes: backgroundSettlementWrites(journal) + }) + ).toMatchObject({ ok: true }) + + expect(children(journal).map(({ id, state }) => [id, state])).toEqual([ + ['a', 'unverifiable'], + ['b', 'working'] + ]) + expect(tasks(journal).map(({ taskId, state }) => [taskId, state])).toEqual([ + ['bg-1', 'unverifiable'], + ['bg-2', 'working'] + ]) + // Settled once: a second pass finds nothing of F's left. + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => live }, + sessionId: SESSION, + journal, + writes: backgroundSettlementWrites(journal) + }) + ).toEqual({ ok: true, planned: 0 }) + }) +}) + +describe('a held proof', () => { + it('upgrades a turn an unproven settle left unverifiable to interrupted, once the lease forgot it', async () => { + const journal = await openJournal() + const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) + deferred.bind({ journal, fence: ENDED, publish: () => {} }) + deferred.sink.appendItem( + { provider: 'claude', sessionId: 'claude-session', uuid: 'turn' }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: 1_000 }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + await expect(deferred.drained()).resolves.toEqual({ ok: true }) + deferred.close() + const turn = () => + journal + .snapshot() + .items.map((item) => readAgentJournalTurn(item.body)) + .find(Boolean) + // Released with nothing proving the owner gone: less specific, never wrong. + const released = agentSessionRecordFixture( + agentSessionLeaseFixture({ + sessionId: SESSION, + runtimeFence: ENDED + 1, + claimStatus: 'released', + ownerProcess: null, + reservedSpawnToken: null + }) + ) + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => released }, + sessionId: SESSION, + journal, + writes: backgroundSettlementWrites(journal) + }) + expect(turn()).toMatchObject({ state: 'unverifiable' }) + + // A later probe proved that owner gone; a reservation since cleared the lease's copy, and the + // retry holds the proof it was handed. + const live = agentSessionRecordFixture( + agentSessionLeaseFixture({ sessionId: SESSION, runtimeFence: LIVE }) + ) + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => live }, + sessionId: SESSION, + journal, + writes: backgroundSettlementWrites(journal), + proof: { kind: 'pid-absent', detail: 'owner gone', observedAt: 2_000, ownerFence: ENDED } + }) + ).toMatchObject({ ok: true }) + expect(turn()).toMatchObject({ state: 'interrupted' }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts index 631b9cac3c37..ac865bf60276 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -1,8 +1,9 @@ import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' // A send can be what opens a conversation this process has not read yet: a chat nobody has on // screen after the app died, sent to from a phone or the CLI. Whatever that journal shows running -// belongs to a generation that is gone, so it is settled when the journal opens, not only when a -// new child starts: a start that then fails would leave the turn running for every reader. +// belongs to a generation that is gone, so the send's acquisition settles it before it reserves: +// a start that then fails leaves no turn running for any reader. A reader's open writes nothing; +// startup settles what it reads. import { cp, rm } from 'node:fs/promises' import { afterEach, expect, it, vi } from 'vitest' @@ -27,6 +28,7 @@ import { import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' /** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ function eventually(assertion: () => unknown): Promise { @@ -111,7 +113,10 @@ it.each(PROBES)( 'settles a turn a dead generation left running when a send opens the chat and its start fails, when %s', async (_when, probe, settled) => { const { host, acquire } = await relaunchAfterCrashMidTurn(probe) - expect(host.hasSession(SESSION)).toBe(false) + // The startup retry opens the chat in the background and closes it again once done. + await host.startupSettled() + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) + await eventually(() => expect(host.hasSession(SESSION)).toBe(false)) const body = hostTestMessage('sent to a chat nobody has open') const sendEnvelope = envelope('agentSession.send', { body }) @@ -133,26 +138,31 @@ it.each(PROBES)( ) it.each(PROBES)( - 'settles the same turn when a reader opens the chat, when %s', + 'settles the same turn at startup, not when a reader opens the chat, when %s', async (_when, probe, settled) => { - const { host } = await relaunchAfterCrashMidTurn(probe) + const { host } = await relaunchAfterCrashMidTurn(probe, { reconcile: false }) await host.revealSession(SESSION) + expect(await turnStates(host)).toEqual(['running']) + await host.reconcileRestartLeases() + await host.startupSettled() + await host.restoreReadableSessions([SESSION]) expect(await turnStates(host)).toEqual([settled]) await host.flushAllStreamedEvents() } ) -// On desktop the chat on screen at relaunch reads before startup reconciles the leases: nothing -// has proved its owner gone yet, and the reconcile's proof then revises what the open settled. -it('settles it when a read reaches the chat before the startup reconcile, then revises it', async () => { +// On desktop the chat on screen at relaunch reads before startup reconciles the leases: that read +// writes nothing, and the startup that follows settles the turn by the reconcile's proof. +it('settles it at startup when a read reaches the chat before the startup reconcile', async () => { const { host } = await relaunchAfterCrashMidTurn({ outcome: 'pid-absent' }, { reconcile: false }) expect(hostTestState().store.getRecord(SESSION)?.lease.claimStatus).toBe('live') await host.history({ sessionId: SESSION, direction: 'tail' }) - expect(await turnStates(host)).toEqual(['unverifiable']) + expect(await turnStates(host)).toEqual(['running']) await host.reconcileRestartLeases() + await host.startupSettled() await host.restoreReadableSessions([SESSION]) expect(await turnStates(host)).toEqual(['interrupted']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts index 677963ab3f9b..b6e7cdb4d3c5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts @@ -1,9 +1,12 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' +import { + StructuredAgentSessionSendSettlement, + type SendSettlementJournal +} from './structured-agent-session-send-settlement' -function journal(dispatchState: 'pending' | 'accepted' | 'unknown'): AgentSessionJournal { +function journal(dispatchState: 'pending' | 'accepted' | 'unknown'): SendSettlementJournal { return { + activeTurnId: () => null, cursor: () => ({ epoch: 'epoch-1', sequence: dispatchState === 'pending' ? 1 : 2 }), submissions: () => [ { @@ -17,21 +20,19 @@ function journal(dispatchState: 'pending' | 'accepted' | 'unknown'): AgentSessio resolvedAt: dispatchState === 'pending' ? null : 2 } ] - } as AgentSessionJournal + } } -function emptyJournal(): AgentSessionJournal { +function emptyJournal(): SendSettlementJournal { return { + activeTurnId: () => null, cursor: () => ({ epoch: 'epoch-1', sequence: 2 }), submissions: () => [] - } as unknown as AgentSessionJournal + } } /** A message accepted and not yet handed over, while `activeTurnId` names the running turn. */ -function queuedJournal( - activeTurnId: string | null, - handedOver = false -): Pick { +function queuedJournal(activeTurnId: string | null, handedOver = false): SendSettlementJournal { return { cursor: () => ({ epoch: 'epoch-1', sequence: handedOver ? 3 : 2 }), activeTurnId: () => activeTurnId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts index a9815b953f8f..b89d36ffe3b4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts @@ -7,8 +7,12 @@ import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-qu import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { isStructuredAgentSessionCommandTurnId } from './structured-agent-session-command-turn' -/** What a settlement read needs of a journal. */ -type SendSettlementJournal = Pick +/** What a settlement read needs: the journal's sends and position, and the live generation's + * running turn as the host's projection reads it (`structuredAgentSessionCurrentWork`), so a send + * never reads as waiting behind a command an ended generation left. */ +export type SendSettlementJournal = Pick & { + activeTurnId: () => string | null +} type SettledSend = { cursor: AgentJournalCursor diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts new file mode 100644 index 000000000000..eb6dc1c28be0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts @@ -0,0 +1,56 @@ +// Doubles for unit tests that drive a settlement without a journal database: a planned batch is +// planned at once and handed to an `appendLifecycleBatch` mock as the batch it would write. + +import type { + AgentJournalCursor, + AgentJournalRenderItem, + AgentJournalSubmission +} from '../../../shared/agent-session-journal-types' +import type { StructuredAgentSessionChildExitSession } from './structured-agent-session-child-exit' +import type { + JournalLifecycleBatchInput, + JournalPlannedLifecycleBatchInput +} from '../agent-session-journal/journal-store-contracts' + +const NO_CURSOR: AgentJournalCursor = { epoch: 'epoch-1', sequence: 0 } + +/** `appendPlannedLifecycleBatch` for a double: a plan with no row reaches no `append`, as the + * writer writes none. */ +export function plannedBatchThrough( + append: (input: JournalLifecycleBatchInput) => Promise +): (input: JournalPlannedLifecycleBatchInput) => Promise { + return async (input) => { + const { mutations, dispatches } = input.plan() + if (mutations.length === 0 && dispatches.length === 0) { + return NO_CURSOR + } + const cursor = await append({ + settlementId: input.settlementId, + fence: input.fence, + ...(input.recovered ? { recovered: input.recovered } : {}), + mutations, + dispatches + }) + return cursor + } +} + +/** An exited child's journal for a test without a database: the reads a settlement makes, with a + * planned batch handed to `appendLifecycleBatch` (`plannedBatchThrough`). */ +export function exitJournalDouble(fields: { + items: () => readonly AgentJournalRenderItem[] + appendLifecycleBatch: (input: JournalLifecycleBatchInput) => Promise + itemFence?: (itemId: string) => number | undefined + submissions?: () => readonly Partial[] +}): StructuredAgentSessionChildExitSession['journal'] { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a settlement reads only these members; the rest of the journal is unreachable from it. + return { + cursor: () => NO_CURSOR, + itemBody: () => null, + itemFence: fields.itemFence ?? (() => undefined), + snapshot: () => ({ items: fields.items() }), + submissions: fields.submissions ?? (() => []), + stopMarks: { latest: () => null, personStopDecides: () => false }, + appendPlannedLifecycleBatch: plannedBatchThrough(fields.appendLifecycleBatch) + } as unknown as StructuredAgentSessionChildExitSession['journal'] +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts index 4ed41e8c6884..7f97faa864f4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts @@ -15,6 +15,7 @@ import { type StructuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' import { projectStructuredAgentSessionStatus } from '../../../shared/structured-agent-session-projection' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' /** The full row the sidebar's fold would show, for callers that need the lead's own state beside * the working answer — the teardown snapshot records both from this one computation. */ @@ -25,9 +26,17 @@ export function structuredAgentSessionShownStatus( }, childWork: readonly AgentChildWorkView[] | undefined, /** The session's lease fence, as the status feed passes it: a send from an older one is not work. */ - fence: number | undefined + fence: number | undefined, + /** The host's projection of current work (`structuredAgentSessionCurrentWork`), as the status + * feed passes it: an ended generation's turn is not work. */ + scope?: StructuredAgentSessionWorkScope ): StructuredAgentSessionAgentStatus { - const status = projectStructuredAgentSessionStatus(journal.items, journal.submissions, fence) + const status = projectStructuredAgentSessionStatus( + journal.items, + journal.submissions, + fence, + scope + ) return structuredAgentSessionAgentStatus({ status, ...(childWork ? { childWork } : {}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts index ae09e13d8782..02a4e11d91d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts @@ -126,7 +126,6 @@ const conversation = ( child?: StructuredAgentSessionProviderChild | null items?: Row[] submissions?: ReturnType[] - activeTurnId?: string newestTurnEndedAt?: number } = {} ) => { @@ -142,7 +141,6 @@ const conversation = ( visit(item.itemId, index, item.body, item.agentId ? { agentId: item.agentId } : {}) ), itemFence: (itemId: string) => items.find((item) => item.itemId === itemId)?.fence, - activeTurnId: () => input.activeTurnId ?? null, newestTurn: (): AgentJournalTurnLifecycle | null => input.newestTurnEndedAt === undefined ? null @@ -197,6 +195,7 @@ describe('retireSignedOutStructuredAgentSessionChild', () => { invocation: { invocationId: 'spawn-subagent-1', generation: 1 } } const idle: StructuredAgentSessionChildWorkReads = { + currentWork: () => ({ activeTurnId: () => null, hasActionablePrompt: () => false }), childWork: () => [], hasOpenDispatch: () => false, providerHoldsDispatch: () => false @@ -223,7 +222,9 @@ describe('retireSignedOutStructuredAgentSessionChild', () => { it('keeps it while it owes work the idle sweep also protects', async () => { // A pending prompt is the shared check's too; the idle sweep's tests cover that case. for (const kept of [ - await retire({ ...signedOut, activeTurnId: 'turn-1' }), + await retire(signedOut, { + currentWork: () => ({ activeTurnId: () => 'turn-1', hasActionablePrompt: () => false }) + }), await retire(signedOut, { childWork: () => [liveChildWork] }), await retire(signedOut, { hasOpenDispatch: () => true }), await retire(signedOut, { providerHoldsDispatch: () => true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts index 40ea59a69eb8..a4315eeb1ca0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts @@ -16,10 +16,7 @@ import type { StructuredAgentSessionLogger } from './structured-agent-session-lo /** What the check reads of a conversation; every read skips building a snapshot. */ type SignedOutReading = Pick & { - journal: Pick< - AgentSessionJournal, - 'visitItems' | 'visitItemsWithLinkage' | 'itemFence' | 'activeTurnId' | 'newestTurn' - > & { + journal: Pick & { submissions(): readonly Pick[] } } @@ -94,7 +91,7 @@ export async function retireSignedOutStructuredAgentSessionChild( if ( !session || !structuredAgentSessionChildReportedSignedOut(session, deps.startUnavailable?.()) || - structuredAgentSessionChildHasOpenWork(session.journal, deps.work) || + structuredAgentSessionChildHasOpenWork(deps.work) || owesWakeUp(session.journal, deps.work.childWork()) ) { return diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts new file mode 100644 index 000000000000..214abbdcc60a --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts @@ -0,0 +1,224 @@ +// What a generation that can no longer write left unfinished, settled from the journal, the +// lease's death evidence and the runtimes this one replaced each time it is asked. Either proof ends +// a turn interrupted; anything else is `unverifiable`, until a proof naming its owner revises it. Working subagents and background +// tasks become `unverifiable`. Planned at the batch's own turn in the write queue, so an answer or +// a Stop queued ahead of it is what it settles from. + +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' +import type { AgentSessionReplacedRuntime } from '../../runtime/agent-session-replaced-runtime' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import type { AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' +import { journalPendingSubmissionResolutions } from '../agent-session-journal/journal-pending-submission-recovery' +import { DISPATCH_DOUBT_PROVIDER_EXITED } from '../agent-session-journal/journal-dispatch-doubt-reasons' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import type { ResolveDispatchInput } from '../agent-session-journal/journal-store-contracts' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { lostLiveWorkJournalBody } from '../agent-session-journal/journal-subagent-liveness' +import { journalEntryOwnerFence } from '../agent-session-journal/journal-item-provenance' +import { + endedUnseenMessageBody, + runningCallEnd, + terminalAgentJournalBody +} from '../agent-session-journal/journal-terminal-settlement' +import { + endedByPersonsStop, + provenUnverifiableTurnRevisions, + provenUnverifiedToolCallRevisions, + lastProvenTurnLiveAt, + runningTurnLifecycleRevisions, + turnVerdictFromDeathEvidence, + turnVerdictFromReplacedRuntime +} from './structured-agent-session-stale-turn-verdict' +import { settledRootTurnScope } from './structured-agent-session-exit-turn-scope' +import { isUnverifiableTurn, staleStopRow } from './structured-agent-session-stale-stop-row' +import { isInProgressStructuredAgentSessionItem } from './structured-agent-session-unfinished-work' + +/** What the settlement reads and writes of a chat's journal. */ +export type StaleStructuredAgentSessionStateJournal = Pick< + AgentSessionJournal, + | 'snapshot' + | 'itemFence' + | 'itemBody' + | 'submissions' + | 'stopMarks' + | 'lastProviderActivityAt' + | 'cursor' + | 'appendPlannedLifecycleBatch' +> + +export type StaleStructuredAgentSessionStateInput = { + journal: StaleStructuredAgentSessionStateJournal + sessionId: string + /** The fence the settlement's rows are written at: the lease's current one. */ + fence: number + acquisitionGeneration: string | null + deathEvidence: AgentSessionDeathEvidence | null + /** The runtimes this one replaced for this chat: every owner they held lost its pipes with them. */ + replaced?: AgentSessionReplacedRuntime + /** Who the exit row names. */ + failureTextContext?: AgentSessionFailureWordsContext + /** Only what generations below this fence wrote, which can no longer write, is settled; their + * sends handed over and never answered become doubt. Absent: every item, and no send. */ + below?: number + /** Roster entries and background tasks are settled by their own provenance + * (`journal-item-provenance.ts`): below this fence, in any row, even a live generation's. + * Absent: every entry of a row settled here, and no other row's. */ + entriesBelow?: number +} + +/** Settles it, and answers how many items it revised. */ +export async function settleStaleStructuredAgentSessionState( + input: StaleStructuredAgentSessionStateInput +): Promise { + const { journal } = input + // A successful settlement leaves terminal items; a failed transaction leaves the same prefix. + const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` + let planned = 0 + await journal.appendPlannedLifecycleBatch({ + settlementId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}`, + fence: input.fence, + recovered: true, + plan: () => { + const plan = planStaleStructuredAgentSessionState(input) + planned = plan.mutations.length + return plan + } + }) + return planned +} + +/** The settlement's rows, read from the journal as it stands. */ +export function planStaleStructuredAgentSessionState( + input: StaleStructuredAgentSessionStateInput +): { + mutations: JournalLifecycleMutationInput[] + dispatches: ResolveDispatchInput[] +} { + const { journal, below } = input + const ended = (fence: number | undefined) => + below === undefined || (fence !== undefined && fence < below) + const all = journal.snapshot().items + const items = all.filter((item) => ended(journal.itemFence(item.itemId))) + const dispatches = + below === undefined + ? [] + : journalPendingSubmissionResolutions( + journal.submissions().filter((entry) => ended(entry.fence)), + input.fence, + // Doubt, never non-delivery: what a gone owner was handed and never answered. + { reason: DISPATCH_DOUBT_PROVIDER_EXITED } + ) + const mutations = staleMutations(input, items) + const { entriesBelow } = input + if (entriesBelow !== undefined) { + // A live generation's row can still carry a child or a task an ended one ran. + for (const item of all) { + const itemFence = journal.itemFence(item.itemId) + const body = ended(itemFence) + ? null + : lostLiveWorkJournalBody(item.body, { + lost: (entry) => entryEndedBelow(entry, itemFence, entriesBelow) + }) + if (body) { + mutations.push({ + kind: 'item', + itemId: item.itemId, + body, + turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE + }) + } + } + } + return { mutations, dispatches } +} + +function entryEndedBelow( + entry: { ownerFence?: number }, + itemFence: number | undefined, + below: number +): boolean { + const fence = journalEntryOwnerFence(entry, itemFence) + return fence !== undefined && fence < below +} + +function staleMutations( + input: StaleStructuredAgentSessionStateInput, + items: readonly AgentJournalRenderItem[] +): JournalLifecycleMutationInput[] { + const { journal, replaced } = input + // Each turn is judged by the evidence only if it names that turn's owner. + const deathVerdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromDeathEvidence( + input.deathEvidence, + journal.itemFence(item.itemId), + lastProvenTurnLiveAt(journal, item) + ) + const replacedVerdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromReplacedRuntime( + replaced, + journal.itemFence(item.itemId), + lastProvenTurnLiveAt(journal, item) + ) + const verdictFor = (item: AgentJournalRenderItem) => { + const byDeath = deathVerdictFor(item) + return byDeath.state === 'interrupted' ? byDeath : replacedVerdictFor(item) + } + // Calls an earlier settle closed with no proof, revised once a proof names their owner. + const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal, replaced) + for (const item of items) { + // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running + // leaves them to the evidence. + const end = runningCallEnd(item.turnScope, (id) => journal.itemBody(id), verdictFor(item).state) + const terminal = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) + // One revision per item: its live subagents and background tasks end with it, but for any a + // live generation observed since. + const itemFence = journal.itemFence(item.itemId) + const { entriesBelow } = input + const body = + lostLiveWorkJournalBody(terminal ?? item.body, { + lost: (entry) => + entriesBelow === undefined ? true : entryEndedBelow(entry, itemFence, entriesBelow) + }) ?? terminal + if (body) { + mutations.push({ + kind: 'item', + itemId: item.itemId, + body, + turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE + }) + } + } + const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal, replaced) + const turnEnds = [ + ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), + ...proven + ] + mutations.push(...turnEnds) + const stopRow = staleStopRow(input, items, (item) => { + const unverifiable = isUnverifiableTurn(item) + if (!unverifiable && !isInProgressStructuredAgentSessionItem(item)) { + return null + } + const fence = journal.itemFence(item.itemId) + // An earlier settle's `unverifiable` turn is revised by a death naming its owner or by its + // replaced runtime. + if ( + unverifiable + ? fence !== undefined && fence === input.deathEvidence?.ownerFence + : deathVerdictFor(item).state === 'interrupted' + ) { + return { by: 'death' } + } + return fence !== undefined && replacedVerdictFor(item).state === 'interrupted' + ? { by: 'replaced', fence } + : null + }) + if (stopRow && !endedByPersonsStop(journal, turnEnds)) { + mutations.unshift({ ...stopRow, turnScope: settledRootTurnScope(items, turnEnds) }) + } + return mutations +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts index 204da81a2e6c..c2423682722c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts @@ -1,6 +1,6 @@ -// A turn an exited agent left running, whose settlement write failed, is settled by the next open of -// the conversation, not only by the next send: a reader reopening a chat the idle sweep closed, or -// reading it before the restart restore reaches it. +// A turn an exited agent left running, whose settlement write failed, is settled by the chat's +// retry once storage takes it, or by the next startup. A reader opening the chat +// in between writes nothing, and the turn holds nothing: it is an ended generation's. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' @@ -16,6 +16,7 @@ import { sweepOnce, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { retryIdle, retryOwes } from './structured-agent-session-retry.test-fixture' let rig: RestTestRig @@ -29,9 +30,8 @@ afterEach(async () => { await rig.dispose() }) -/** A turn in flight whose agent exits, and whose exit settlement the journal refuses: by default - * also the retry that recording the exit queues, so only an open is left to settle it. */ -async function exitWithUnwrittenSettlement(refusedWrites = 2): Promise { +/** A turn in flight whose agent exits, and whose exit settlement the journal refuses. */ +async function exitWithUnwrittenSettlement(refusedWrites = 1): Promise { await foundRestTestChat(rig) const open = rig.host.collaboratorsForTests().sessions.get(SESSION)! const running = open.child! @@ -43,7 +43,7 @@ async function exitWithUnwrittenSettlement(refusedWrites = 2): Promise { { turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await rig.host.flushStreamedEvents(SESSION) - const append = vi.spyOn(open.journal, 'appendLifecycleBatch') + const append = vi.spyOn(open.journal, 'appendPlannedLifecycleBatch') for (let refused = 0; refused < refusedWrites; refused += 1) { append.mockRejectedValueOnce(new Error('disk full')) } @@ -72,8 +72,8 @@ async function workingTurnState(): Promise { } describe('a turn its gone agent left running', () => { - it('is settled in place by the retry recording the exit queues, with no reopen', async () => { - await exitWithUnwrittenSettlement(1) + it('is settled in place by the exit, with no reopen', async () => { + await exitWithUnwrittenSettlement(0) expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(true) expect(await workingTurnState()).toBe('interrupted') @@ -84,30 +84,44 @@ describe('a turn its gone agent left running', () => { expect(rig.adapter.acquire).toHaveBeenCalledOnce() }) - it('is settled when the idle-closed conversation is opened again, and its reader sees it', async () => { - await exitWithUnwrittenSettlement() + it('is settled by the host retry once storage takes the write, with no send; a reader writes nothing meanwhile', async () => { + // The exit's own write and the retry's first attempt both fail; the backoff's retry lands. + await exitWithUnwrittenSettlement(2) + const { reconciliation } = rig.host.collaboratorsForTests() + await retryIdle(reconciliation, SESSION) + expect(retryOwes(reconciliation, SESSION)).toBe(true) expect(await workingTurnState()).toBe('running') - rig.clock.now += IDLE_MS + 1 - await sweepOnce(rig.host) - expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) + const cursor = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal.cursor() const reader = collectSubscriber() await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) + expect(rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal.cursor()).toEqual(cursor) - expect(await workingTurnState()).toBe('interrupted') + await vi.waitFor(async () => expect(await workingTurnState()).toBe('interrupted'), { + timeout: 5_000 + }) // The death evidence is Orca's log text: the row says only that the provider stopped. expect(JSON.stringify(reader.events)).toContain( 'Codex stopped while this response was in progress. You can continue in this conversation.' ) expect(rig.adapter.acquire).toHaveBeenCalledOnce() + // Nothing owed once it landed: the idle sweep closes the chat as it would any other. + await retryIdle(reconciliation, SESSION) + rig.clock.now += IDLE_MS + 1 + await sweepOnce(rig.host) + expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) }) - it('is settled by a read after a restart that reaches the chat before the restore does', async () => { - await exitWithUnwrittenSettlement() + it('is settled by the startup after a restart, even when a read reaches the chat first', async () => { + // Storage refuses it for the rest of this process. + await exitWithUnwrittenSettlement(100) await rig.restart() setStructuredAgentSessionHost(rig.host) - // The client's read opens the chat first; the restore then finds it open and skips it. + // The client's read opens the chat first and writes nothing. + expect(await workingTurnState()).toBe('running') + await rig.host.reconcileRestartLeases() + await rig.host.startupSettled() expect(await workingTurnState()).toBe('interrupted') await rig.host.restoreReadableSessions([SESSION]) expect(await workingTurnState()).toBe('interrupted') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index 9efb8202f880..0d3f5620bad7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -17,6 +17,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { plannedBatchThrough } from './structured-agent-session-settlement-double.test-fixture' import { runningTurnLifecycleRevisions, turnVerdictFromDeathEvidence, @@ -314,7 +315,8 @@ describe('stale session state on a cold acquire', () => { lastProviderActivityAt: () => undefined, stopMarks: { latest: () => null }, cursor: () => ({ epoch: 'epoch-1', sequence: 8 }), - appendLifecycleBatch + submissions: () => [], + appendPlannedLifecycleBatch: plannedBatchThrough(appendLifecycleBatch) } as unknown as AgentSessionJournal return { journal, appendLifecycleBatch } } @@ -339,6 +341,7 @@ describe('stale session state on a cold acquire', () => { settlementId: 'stale-session:session-1:14:generation-2', fence: 14, recovered: true, + dispatches: [], mutations: [ { kind: 'item', @@ -369,6 +372,7 @@ describe('stale session state on a cold acquire', () => { settlementId: 'stale-session:session-1:14:generation-2', fence: 14, recovered: true, + dispatches: [], mutations: [ { kind: 'item', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts index 9206f95fde0b..1ca8d381e323 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -1,10 +1,10 @@ import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { structuredAgentSessionStartFailureRowIdentity } from '../../../shared/structured-agent-session-start-failure-row-key' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionStartFailureWords } from './structured-agent-session-failure-text' +import { structuredAgentSessionHandsOver } from './structured-agent-session-current-work' /** A start that failed, keyed like its row, in the words `structuredAgentSessionStartFailure` gave. */ export type StructuredAgentSessionStartFailure = StructuredAgentSessionStartFailureWords & { @@ -55,22 +55,27 @@ export async function recordStructuredAgentSessionStartFailure( return } const startKey = failure.startKey ?? oldest.clientMessageId - await session.journal.appendLifecycleBatch({ + const { journal } = session + await journal.appendLifecycleBatch({ settlementId: `start-failure:${startKey}`, fence: session.fence, recovered: true, mutations: [structuredAgentSessionStartFailureRow(startKey, failure)], - rejectsQueued: { reason: failure.reason, rejection: failure.rejection } + rejectsQueued: { reason: failure.reason, rejection: failure.rejection }, + // The start was for this process's sends; an earlier process's is the reconciliation pass's to keep. + rejectsQueuedOnly: (submission) => structuredAgentSessionHandsOver(journal, submission) }) } +/** The oldest send this process may hand over (`structuredAgentSessionHandsOver`): an earlier host + * process's unhanded send is never handed over, whether or not its bookkeeping landed. */ export function oldestQueuedSubmission( session: Pick ): ReturnType[number] | undefined { let oldest: ReturnType for (const submission of session.journal.submissions()) { if ( - isQueuedAgentJournalSubmission(submission) && + structuredAgentSessionHandsOver(session.journal, submission) && (oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0)) ) { oldest = submission diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index fed25d7dc9b0..cf49ea0b7934 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -11,25 +11,29 @@ import { type StructuredAgentSessionChildExitSession } from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { exitJournalDouble } from './structured-agent-session-settlement-double.test-fixture' const SESSION = 'session-1' const GENERATION = 'generation-1' const REASON = 'claude stream-json exited (code 1): session limit reached' const STARTUP_TEXT = 'Claude stopped before it finished starting. Send your message to try again.' -function startedSession(): StructuredAgentSessionChildExitSession & { - journal: { appendLifecycleBatch: ReturnType } +function startedSession( + submissions?: () => readonly Partial[] +): StructuredAgentSessionChildExitSession & { + appendLifecycleBatch: ReturnType } { + const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })) return { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - // Nothing ran: the start failed before any response or acknowledged prompt. - snapshot: () => ({ items: [] }), - appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })) - } + appendLifecycleBatch, + // Nothing ran: the start failed before any response or acknowledged prompt. + journal: exitJournalDouble({ + items: () => [], + appendLifecycleBatch, + ...(submissions ? { submissions } : {}) + }) } } @@ -57,7 +61,7 @@ function contextFor(session: StructuredAgentSessionChildExitSession) { }, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } @@ -84,7 +88,7 @@ describe('a provider that ends before it finished starting', () => { startupUnproven: true }) - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect(session.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ mutations: [ expect.objectContaining({ @@ -108,7 +112,7 @@ describe('a provider that ends before it finished starting', () => { await settleStructuredAgentSessionChildExit(contextFor(session), ended) expect(session.child).toBeNull() - expect(session.journal.appendLifecycleBatch).not.toHaveBeenCalled() + expect(session.appendLifecycleBatch).not.toHaveBeenCalled() }) it("reads a start that failed off the host's own phase when the provider omits the flag", async () => { @@ -122,7 +126,7 @@ describe('a provider that ends before it finished starting', () => { failure: { kind: 'providerExited', detail: { text: REASON, audience: 'log' } } }) - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect(session.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ mutations: [ expect.objectContaining({ @@ -145,27 +149,26 @@ describe('a provider that ends before it finished starting', () => { }) it('names /compact as the next step when the start that failed was carrying it', async () => { - const base = startedSession() + const base = startedSession(() => [ + { clientMessageId: 'compact-1', dispatchState: 'pending' as const } + ]) const session = { ...base, child: { generation: GENERATION, fence: 7, phase: 'starting' as const }, - journal: { - ...base.journal, - submissions: () => [{ clientMessageId: 'compact-1', dispatchState: 'pending' as const }], - itemBody: () => structuredAgentSessionCompactBody() - } + journal: { ...base.journal, itemBody: () => structuredAgentSessionCompactBody() } } await settleStructuredAgentSessionChildExit(contextFor(session), ended) const text = 'Claude stopped before it finished starting. Run /compact again.' - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect(session.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ dispatches: [ expect.objectContaining({ clientMessageId: 'compact-1', state: 'rejected', reason: text, + // At the exited child's own fence, before its release moves it. fence: 7, recovered: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts new file mode 100644 index 000000000000..51170ce2431c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts @@ -0,0 +1,166 @@ +// What startup decides about a lease latched in recovery: nothing, for a chat nobody looks at. +// Deciding it signals a process that may still run, so only the visible-tab restore, a start or an +// attach does; that decision's release then wakes the host's retry, which settles what it left. +// What the earlier process left unsent is kept as a card meanwhile, as every open did on main. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + scanIdle as idle, + scanRecord as record, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' +const OWNER_PID = 12_000 + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +/** The earlier process's agent outlived it, until something stops it. */ +async function openWithSurvivingOwner(options: Parameters[2] = {}) { + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) + await seedScanJournal(root, CHAT, options) + store = await openTestAgentSessionRecordStore(root) + let alive = true + const stopOwnerProcess = vi.fn((_pid: number, _signal: 'SIGTERM' | 'SIGKILL') => { + alive = false + }) + const probeOwner = async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + return { current: host, stopOwnerProcess } +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-recovery-scope-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +describe('a hidden chat whose agent outlived the earlier process', () => { + it('is left alone at startup, and settled once an attach decides its recovery', async () => { + const { current, stopOwnerProcess } = await openWithSurvivingOwner() + + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [CHAT]) + + // No signal to a process that may still run, and no lease change. + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease).toMatchObject({ + handoffStage: 'recovering', + runtimeFence: 13, + ownerProcess: { pid: OWNER_PID } + }) + + // The person attaches: recovery is decided first, as on every build, stopping the owner. + const params = attachParamsForRecord(store.getRecord(CHAT)!, { + clientOperationId: 'attach-1', + expectedRuntimeFence: 13 + }) + await current.attach({ callerKey: 'client-1' }, params).catch(() => undefined) + + expect(stopOwnerProcess).toHaveBeenCalledWith(OWNER_PID, 'SIGTERM') + expect(store.getRecord(CHAT)?.lease).toMatchObject({ handoffStage: null, ownerProcess: null }) + // Its release woke the retry, which settles the turn the owner left by that proof. + await idle(current, [CHAT]) + expect(await turnState(current, CHAT)).toBe('interrupted') + }) +}) + +describe("a hidden chat's send the earlier process never handed over, while its lease recovers", () => { + it('is a card once the chat is read, with no attach and no decision about the owner', async () => { + // Accepted at the owner's own fence, which the lease still holds. + const { current, stopOwnerProcess } = await openWithSurvivingOwner({ + queued: true, + queuedFence: 13 + }) + + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [CHAT]) + // Desktop opens a chat by reading it. + const page = await current.history({ sessionId: CHAT, direction: 'tail' }) + const { submissions } = await current.journalSnapshot(CHAT) + + expect(page.ok && page.page.queuedMessages?.map((card) => card.messageId)).toEqual([ + `${CHAT}-queued` + ]) + expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-queued`)).toMatchObject({ + dispatchState: 'rejected' + }) + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease).toMatchObject({ + handoffStage: 'recovering', + runtimeFence: 13, + ownerProcess: { pid: OWNER_PID } + }) + // The runtime that held its owner was replaced: that turn is over whatever the recovery + // decides, so startup settles it, stopping nothing and leaving the lease as it is. + expect(await turnState(current, CHAT)).toBe('interrupted') + }) +}) + +describe('a visible chat whose agent outlived the earlier process', () => { + it('has its owner stopped by the visible-tab restore, as before', async () => { + const { current, stopOwnerProcess } = await openWithSurvivingOwner() + + await current.reconcileRestartLeases() + await current.restoreReadableSessions([CHAT]) + + expect(stopOwnerProcess).toHaveBeenCalledWith(OWNER_PID, 'SIGTERM') + expect(store.getRecord(CHAT)?.lease).toMatchObject({ handoffStage: null, ownerProcess: null }) + await current.startupSettled() + await idle(current, [CHAT]) + expect(await turnState(current, CHAT)).toBe('interrupted') + }) +}) + +describe('a chat whose startup lease reconcile failed (storage was busy)', () => { + it('is reconciled by the retry after a backoff, and its turn settles with no attach or send', async () => { + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) + await seedScanJournal(root, CHAT) + store = await openTestAgentSessionRecordStore(root) + const busy = () => new Error('database is locked') + // Startup's reconcile and the retry's first retry meet a locked store; the next one lands. + const reconcile = vi + .spyOn(store, 'reconcileOnRestart') + .mockRejectedValueOnce(busy()) + .mockRejectedValueOnce(busy()) + host = openScanHost(root, store) + const current = host + + await current.reconcileRestartLeases() + expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(true) + await vi.waitFor(() => expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(false), { + timeout: 5_000 + }) + await idle(current, [CHAT]) + + expect(reconcile.mock.calls.length).toBeGreaterThanOrEqual(3) + // The owner was proven gone: the turn ends, and nothing reads working. + expect(await turnState(current, CHAT)).toBe('interrupted') + expect(current.currentWork(CHAT)?.working()).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts new file mode 100644 index 000000000000..3f20f3b751e8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts @@ -0,0 +1,252 @@ +// The startup scan's retrys over their whole life: what a quit cuts short, what +// they never publish or close, and what ends a retry that cannot succeed. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionStatusEvent } from '../../../shared/agent-session-wire' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { publishJournalSessionEpoch } from '../agent-session-journal/journal-row-table' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' +import * as reconciliationLoad from './structured-agent-session-reconciliation-load' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { + openScanHost, + scanIdle as idle, + scanRecord as record, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +const CHAT = 'chat-aaaaaaa1' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined +const warnings: string[] = [] + +function openHost(options: Parameters[2] = {}): StructuredAgentSessionHost { + const logger = createStructuredAgentSessionLogger() + host = openScanHost(root, store, { + logger: { ...logger, warn: (message) => warnings.push(message) }, + ...options + }) + return host +} + +async function seed(sessionIds: readonly string[], extra: Parameters[2] = {}) { + await seedTestAgentSessionRecordStore(root, { + records: sessionIds.map((id) => record(id, true, extra)) + }) + for (const sessionId of sessionIds) { + await seedScanJournal(root, sessionId) + } + store = await openTestAgentSessionRecordStore(root) +} + +/** Counts every journal open from now on; each waits for `hold` first. */ +function countOpens(hold: () => Promise = async () => undefined): { opens: number } { + const counter = { opens: 0 } + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + counter.opens += 1 + await hold() + return open.call(this) + }) + return counter +} + +beforeEach(async () => { + warnings.length = 0 + root = await mkdtemp(join(tmpdir(), 'orca-startup-scan-lifetime-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +describe('a quit during the startup scan', () => { + it('waits only for work already started: no chat still waiting for a slot opens or recovers', async () => { + const chats = Array.from( + { length: 10 }, + (_, index) => `chat-q${String(index).padStart(7, '0')}` + ) + await seed(chats) + const current = openHost() + let release: () => void = () => undefined + const gate = new Promise((resolve) => (release = resolve)) + const counter = countOpens(() => gate) + const passes = vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune') + await current.reconcileRestartLeases() + await vi.waitFor(() => expect(counter.opens).toBe(4)) + + const quit = current.flushAllStreamedEvents() + release() + await quit + host = undefined + + // The four already reading finished that read and went no further; the six still waiting + // for a slot never started. + expect(counter.opens).toBe(4) + expect(passes).not.toHaveBeenCalled() + }) +}) + +describe('a chat the startup scan settles', () => { + it('is published to no status surface unless a reader or the restorer opened it', async () => { + const visible = 'chat-visible1' + const hidden = ['chat-hidden01', 'chat-hidden02', 'chat-hidden03'] + await seed([visible, ...hidden]) + const published: string[] = [] + const current = openHost({ + statusSink: { + publish: (summary) => published.push(summary.sessionId), + forget: () => undefined + } + }) + await current.reconcileRestartLeases() + await current.restoreReadableSessions([visible]) + await current.startupSettled() + await idle(current, [visible, ...hidden]) + + const snapshots: string[][] = [] + const unsubscribe = current.subscribeStatus({ + id: 'renderer', + emit: (event: AgentSessionStatusEvent) => { + if (event.type === 'snapshot') { + snapshots.push(event.sessions.map((session) => session.sessionId)) + } + } + }) + unsubscribe() + expect(new Set(published)).toEqual(new Set([visible])) + expect(snapshots.at(-1)).toEqual([visible]) + // Settled all the same. + expect(current.collaboratorsForTests().sessions.has('chat-hidden01')).toBe(false) + expect(await turnState(current, 'chat-hidden01')).toBe('interrupted') + }) + + it('stays open when the restorer opened it while the scan was settling it', async () => { + await seed([CHAT]) + const current = openHost() + const { sessions } = current.collaboratorsForTests() + // The restore lands between the scan's read and its pass. + let restore: Promise | null = null + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + await open.call(this) + restore ??= current.restoreReadableSessions([CHAT]) + }) + await current.reconcileRestartLeases() + await current.startupSettled() + await restore + await idle(current, [CHAT]) + await new Promise((resolve) => setTimeout(resolve, 20)) + + expect(sessions.has(CHAT)).toBe(true) + expect(await turnState(current, CHAT)).toBe('interrupted') + }) +}) + +describe('a startup retry whose step cannot succeed', () => { + it('retires at once on a journal no retry can load, and replays it once', async () => { + await seed([CHAT]) + // The last row is one no build wrote. + openTestJournalHostDatabase(root) + .db.prepare( + `UPDATE journal_rows SET row_json = '{"garbage":true}' WHERE session_id = ? AND seq = (SELECT MAX(seq) FROM journal_rows WHERE session_id = ?)` + ) + .run(CHAT, CHAT) + const counter = countOpens() + const current = openHost() + await current.reconcileRestartLeases() + await current.startupSettled() + + expect(retryOwes(current.collaboratorsForTests().reconciliation, CHAT)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(counter.opens).toBe(1) + expect(warnings).toContain("a chat's history cannot be loaded, so nothing is settled") + }) + + it('leaves a rewind only its provider can prove for the next acquisition, quietly, and retires', async () => { + const rewind = { + operationId: 'op-rewind', + callerKey: 'caller', + itemId: agentJournalItemKey({ + provider: 'codex', + threadId: 'thread', + turnId: 'turn-1', + ordinal: 0 + }), + expectedEpoch: 'epoch-x', + phase: 'prepared' as const, + retained: [] + } + await seed([CHAT], { rewind }) + const current = openHost() + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [CHAT]) + + expect(store.getRecord(CHAT)?.rewind?.phase).toBe('prepared') + expect(current.collaboratorsForTests().sessions.has(CHAT)).toBe(false) + // Not tried here at all: without its provider it cannot succeed. + expect(warnings).toEqual([]) + expect(await turnState(current, CHAT)).toBe('interrupted') + }) + + it('retires at once on a journal whose named epoch holds no row, reading it once', async () => { + // An older build's repair crashed after naming the epoch: nothing durable is there to settle. + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, true)] }) + publishJournalSessionEpoch( + openTestJournalHostDatabase(root).db, + { sessionId: CHAT, workspaceId: LOCATION.workspaceId }, + 'empty-epoch' + ) + store = await openTestAgentSessionRecordStore(root) + const counter = countOpens() + const current = openHost() + await current.reconcileRestartLeases() + await current.startupSettled() + + expect(retryOwes(current.collaboratorsForTests().reconciliation, CHAT)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(counter.opens).toBe(1) + }) + + it('backs off a read that goes stale on every attempt instead of re-reading at once', async () => { + await seed([CHAT]) + vi.spyOn(reconciliationLoad, 'structuredAgentSessionJournalIsCurrent').mockReturnValue(false) + const counter = countOpens() + const current = openHost() + await current.reconcileRestartLeases() + await new Promise((resolve) => setTimeout(resolve, 2_500)) + + // Attempts at 0 s, 1 s and 3 s: a read each, never a loop. + expect(counter.opens).toBeGreaterThanOrEqual(2) + expect(counter.opens).toBeLessThanOrEqual(3) + expect(retryOwes(current.collaboratorsForTests().reconciliation, CHAT)).toBe(true) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts new file mode 100644 index 000000000000..4f840a9b1063 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts @@ -0,0 +1,186 @@ +// Chats an earlier process left, for the startup-scan suites: records on disk, each journal with a +// running turn at fence 13, and a host over them with no provider. + +import { vi, expect } from 'vitest' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' +import { + createStructuredAgentSessionLogger, + type StructuredAgentSessionLogger +} from './structured-agent-session-logger' +import type { StructuredAgentSessionStatusSink } from './structured-agent-session-status-ownership' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +export const SCAN_NOW = 1_800_000_000_000 +export const SCAN_CHATS = [ + 'chat-aaaaaaa1', + 'chat-aaaaaaa2', + 'chat-aaaaaaa3', + 'chat-aaaaaaa4', + 'chat-aaaaaaa5' +] +export const SCAN_LATE = 'chat-aaaaaaa6' +/** Released long ago with nothing proving its owner gone: no restart moves its fence. */ +export const SCAN_RELEASED = 'chat-released1' + +export function scanRecord( + sessionId: string, + released: boolean, + extra: Partial = {} +): AgentSessionRecord { + const linkId = `${sessionId}-link` + return { + schemaVersion: 2, + sessionId, + location: LOCATION, + provider: 'claude', + providerHandleChain: [ + { + linkId, + handle: claudeProviderHandle(`provider-${sessionId}`, null), + origin: 'created', + mintedAtFence: 13, + observedAt: SCAN_NOW - 60_000 + } + ], + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + createdAt: SCAN_NOW - 60_000, + updatedAt: SCAN_NOW, + lease: { + sessionId, + runtimeKind: 'native', + runtimeFence: released ? 14 : 13, + handoffStage: null, + provenHandleLinkId: linkId, + ownerProcess: released + ? null + : { + hostId: 'local', + pid: 12_000, + processStartTimeMs: SCAN_NOW - 60_000, + spawnToken: 'spawn' + }, + reservedSpawnToken: released ? null : 'spawn', + leaseDeadlineAt: SCAN_NOW + 30_000, + lastRenewedAt: SCAN_NOW, + handoffOperationId: null, + journalCheckpoint: null, + claimKeyId: 'key-1', + claimStatus: released ? 'released' : 'live', + unreconciled: false, + deathEvidence: null + }, + ...extra + } +} + +/** The chat's journal as the crashed process left it: a running turn at fence 13, and, when + * asked, a send it accepted and never handed over (at `queuedFence`, the released lease's 14 by + * default), or one it handed to the generation at `handedOverAt` that never answered it. */ +export async function seedScanJournal( + root: string, + sessionId: string, + options: { queued?: true; queuedFence?: number; handedOverAt?: number } = {} +): Promise { + const journal = await openAgentSessionJournal({ + identity: { + sessionId, + workspaceId: LOCATION.workspaceId, + hostId: LOCATION.executionHostId, + agent: 'claude', + providerHandle: claudeProviderHandle(`provider-${sessionId}`, null) + }, + database: openTestJournalHostDatabase(root), + now: () => SCAN_NOW + }) + await journal.appendItem( + { provider: 'claude', sessionId: `provider-${sessionId}`, uuid: 'turn' }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: SCAN_NOW - 30_000 }, + { fence: 13, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + if (options.queued) { + await journal.appendSubmission({ + clientMessageId: `${sessionId}-queued`, + payloadFingerprint: '0'.repeat(64), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, + fence: options.queuedFence ?? 14, + handoverRecorded: true, + origin: 'client', + source: { kind: 'user' } + }) + } + const fence = options.handedOverAt + if (fence !== undefined) { + const clientMessageId = `${sessionId}-handed` + await journal.appendSubmission({ + clientMessageId, + payloadFingerprint: '1'.repeat(64), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'handed over' }] }, + fence, + handoverRecorded: true, + origin: 'client', + source: { kind: 'user' } + }) + await journal.resolveDispatch({ + clientMessageId, + state: 'pending', + fence, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + } + await journal.close() +} + +export function openScanHost( + root: string, + store: AgentSessionRecordStore, + options: { + logger?: StructuredAgentSessionLogger + statusSink?: StructuredAgentSessionStatusSink + } & Partial> = {} +): StructuredAgentSessionHost { + return new StructuredAgentSessionHost({ + agents: NO_STRUCTURED_AGENTS, + logger: options.logger ?? createStructuredAgentSessionLogger(), + store, + adapter: { + acquire: vi.fn(), + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + ...(options.statusSink ? { statusSink: options.statusSink } : {}), + ...(options.stopOwnerProcess ? { stopOwnerProcess: options.stopOwnerProcess } : {}), + probeOwner: options.probeOwner ?? (async () => ({ outcome: 'pid-absent' })), + now: () => SCAN_NOW + }) +} + +/** The turn's state as a reader sees it; the read opens the chat. */ +export async function scanTurnState(current: StructuredAgentSessionHost, sessionId: string) { + return (await current.journalSnapshot(sessionId)).items + .map((item) => readAgentJournalTurn(item.body)) + .find(Boolean)?.state +} + +export async function scanIdle(current: StructuredAgentSessionHost, sessionIds: readonly string[]) { + const { reconciliation } = current.collaboratorsForTests() + await vi.waitFor( + () => + expect(sessionIds.filter((sessionId) => retryOwes(reconciliation, sessionId))).toEqual([]), + { timeout: 10_000 } + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts new file mode 100644 index 000000000000..71d86a5f8e6d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts @@ -0,0 +1,210 @@ +// Startup's scan of every chat on record: in the background, never ahead of a person's own action +// on a chat, with no stored mark saying which chats are owed anything. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + SCAN_CHATS as CHATS, + SCAN_LATE as LATE, + SCAN_RELEASED as RELEASED, + scanIdle as idle, + scanRecord as record, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +function openHost(): StructuredAgentSessionHost { + host = openScanHost(root, store) + return host +} + +/** The chats each pass ran on, in order: every pass repairs the cards an earlier handle left. */ +function watchPasses(): string[] { + const passes: string[] = [] + const repair = JournalQueuedMessages.prototype.repairAndPrune + vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune').mockImplementation(function ( + this: JournalQueuedMessages + ) { + passes.push(this.sessionId) + return repair.call(this) + }) + return passes +} + +/** Holds the first four journal opens (every background slot); `opened` lists each chat whose + * open began, in order. */ +function gateFirstOpens(): { opened: string[]; releaseOne: () => void; release: () => void } { + const held: (() => void)[] = [] + const opened: string[] = [] + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + opened.push(this.queuedMessages.sessionId) + if (opened.length <= 4) { + await new Promise((resolve) => held.push(resolve)) + } + return open.call(this) + }) + return { + opened, + releaseOne: () => held.shift()?.(), + release: () => held.splice(0).forEach((resolve) => resolve()) + } +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-scan-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +describe('the startup scan', () => { + it('never puts a chat late in the scan behind its own share, and settles it without publishing it', async () => { + const all = [...CHATS, LATE] + await seedTestAgentSessionRecordStore(root, { records: all.map((id) => record(id, false)) }) + for (const sessionId of all) { + await seedScanJournal(root, sessionId) + } + store = await openTestAgentSessionRecordStore(root) + const { opened, release } = gateFirstOpens() + const current = openHost() + const indexed: string[] = [] + const { sessions, serialize } = current.collaboratorsForTests() + const set = sessions.set.bind(sessions) + vi.spyOn(sessions, 'set').mockImplementation((sessionId, session) => { + indexed.push(sessionId) + return set(sessionId, session) + }) + const shares = watchPasses() + + await current.reconcileRestartLeases() + await vi.waitFor(() => expect(opened).toHaveLength(4)) + + // The person's own actions on the late chat run now, ahead of its share. + const order: string[] = [] + await serialize(LATE, async () => { + order.push('lane') + }) + expect(await turnState(current, LATE)).toBe('running') + order.push('read') + expect(shares).not.toContain(LATE) + + release() + await current.startupSettled() + await idle(current, all) + expect(order).toEqual(['lane', 'read']) + expect(new Set(shares)).toEqual(new Set(all)) + // Only the chat a reader opened is a conversation; the scan's own reads published nothing. + expect(indexed).toEqual([LATE]) + for (const sessionId of all) { + expect(await turnState(current, sessionId)).toBe('interrupted') + } + }) + + it('takes a chat a reader opens ahead of the rest of the scan', async () => { + const all = [...CHATS, LATE] + // Released: the restart reconcile ends nothing, so each chat's first attempt is its share. + await seedTestAgentSessionRecordStore(root, { records: all.map((id) => record(id, true)) }) + for (const sessionId of all) { + await seedScanJournal(root, sessionId) + } + store = await openTestAgentSessionRecordStore(root) + const { opened, releaseOne, release } = gateFirstOpens() + const current = openHost() + const passes = watchPasses() + + await current.reconcileRestartLeases() + await vi.waitFor(() => expect(opened).toHaveLength(4)) + expect(opened).not.toContain(LATE) + // The person opens the late chat while two chats still wait for a slot, it among them. + expect(await turnState(current, LATE)).toBe('running') + + // One slot comes free: the late chat takes it ahead of the chat queued before it. + releaseOne() + await vi.waitFor(() => expect(passes).toContain(LATE)) + release() + await current.startupSettled() + await idle(current, all) + expect(passes.indexOf(LATE)).toBeLessThan(passes.indexOf(CHATS[4]!)) + expect(await turnState(current, LATE)).toBe('interrupted') + }) + + it('finds what an earlier process left with no fence move and no stored mark, and keeps its send as a card', async () => { + await seedTestAgentSessionRecordStore(root, { records: [record(RELEASED, true)] }) + await seedScanJournal(root, RELEASED, { queued: true }) + store = await openTestAgentSessionRecordStore(root) + const current = openHost() + const ended = vi.fn() + store.onGenerationEnded(ended) + + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [RELEASED]) + + // Already released: the restart moved nothing and ended no generation. + expect(ended).not.toHaveBeenCalled() + expect(store.getRecord(RELEASED)?.lease.runtimeFence).toBe(14) + // Nothing proved its owner gone, but the runtime that held it was replaced, which ends it. + expect(await turnState(current, RELEASED)).toBe('interrupted') + const { journal } = current.collaboratorsForTests().sessions.get(RELEASED)! + expect(journal.submission(`${RELEASED}-queued`)).toMatchObject({ dispatchState: 'rejected' }) + expect(journal.queuedMessages.list()).toEqual([ + expect.objectContaining({ body: expect.objectContaining({ role: 'user' }) }) + ]) + }) + + it('retries a conversion that failed instead of marking it done', async () => { + await seedTestAgentSessionRecordStore(root, { records: [record(RELEASED, true)] }) + await seedScanJournal(root, RELEASED, { queued: true }) + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const database = openTestJournalHostDatabase(root) + database.db.exec(` + CREATE TEMP TRIGGER reject_conversion BEFORE INSERT ON journal_rows + WHEN json_extract(NEW.row_json, '$.kind') = 'dispatch' + AND json_extract(NEW.row_json, '$.state') = 'rejected' + BEGIN SELECT RAISE(ABORT, 'temporary storage failure'); END; + `) + const current = openHost() + + await current.reconcileRestartLeases() + await current.startupSettled() + const { reconciliation } = current.collaboratorsForTests() + expect(retryOwes(reconciliation, RELEASED)).toBe(true) + + database.db.exec('DROP TRIGGER reject_conversion') + await idle(current, [RELEASED]) + + expect(await turnState(current, RELEASED)).toBe('interrupted') + const { journal } = current.collaboratorsForTests().sessions.get(RELEASED)! + expect(journal.submission(`${RELEASED}-queued`)).toMatchObject({ dispatchState: 'rejected' }) + expect(journal.queuedMessages.list()).toHaveLength(1) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts new file mode 100644 index 000000000000..87fd4618723e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -0,0 +1,27 @@ +// What the host owes its chats when it comes up as a new instance: what the earlier process left. +// Run after the whole-host restart reconcile, in the background, for EVERY chat on record: no +// stored "settled" mark says which are owed, so the host's retry derives each from its journal +// (`structured-agent-session-reconciliation-pass.ts`). The scan only wakes the retry. +// +// It never decides a lease latched in recovery: that signals a process that may still run, which +// only the visible-tab restore, a start or an attach does, as on every build before this one. +// Their release then wakes the retry again. The retry replays a closed chat in a background slot as +// its own read, published to no status surface, and takes the chat's lane only to write, so a send, +// start or read of a chat late in the scan never waits behind it, and an opened chat goes first. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' + +/** Resolves once every chat's first attempt finished, whatever it found. */ +export function scanStructuredAgentSessionsAtStartup( + store: Pick, + reconciliation: Pick +): Promise { + const sessionIds = store.listRecords().map((record) => record.sessionId) + for (const sessionId of sessionIds) { + reconciliation.signal(sessionId) + } + return Promise.all(sessionIds.map((sessionId) => reconciliation.attempted(sessionId))).then( + () => undefined + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 9fa62fd170e3..0a24a6b805b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -360,6 +360,8 @@ describe('StructuredAgentSessionStatusFeed', () => { it('carries the record model and the running tool line the sidebar row shows', async () => { const journal = await openJournal() + // Written by the record's current generation: work an ended one left never reads working. + const fence = agentSessionRecordFixture().lease.runtimeFence const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), { location: indexed({ journal }).params.location, options: { model: 'gpt-5-codex' }, @@ -368,12 +370,12 @@ describe('StructuredAgentSessionStatusFeed', () => { await journal.appendItem( USER_IDENTITY, { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'run the tests' }] }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await journal.appendItem( TURN_IDENTITY, { kind: 'status', text: 'Working', turnLifecycle: { turnId: 'turn-1', state: 'running' } }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) feed.publish(SESSION) expect(events.at(-1)).toEqual({ @@ -384,7 +386,7 @@ describe('StructuredAgentSessionStatusFeed', () => { await journal.appendItem( { ...USER_IDENTITY, ordinal: 2 }, { kind: 'tool-call', name: 'shell', input: { command: 'pnpm test' }, state: 'running' }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) feed.publish(SESSION) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index 2a26af994dcb..81b9a273c464 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -19,7 +19,10 @@ import type { import type { AgentChildWorkEvidence } from '../../../shared/agent-status-child-work-evidence' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionProviderChild +} from './structured-agent-session-host-types' import { structuredAgentSessionStatusSummary } from './structured-agent-session-status-summary' import { structuredStatusChildWork } from './structured-agent-session-status-child-work' import { @@ -46,12 +49,17 @@ type StatusFeedSession = { journal: AgentSessionJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } child?: Pick | null + /** What the current-work projection reads of the last child this host saw end. */ + lastEndedChild?: Pick + operationalRevision?: number restartResume?: AgentSessionStatusSummary['restartResume'] } export type StructuredAgentSessionStatusFeedDeps = { sessions: ReadonlyMap getRecord: (sessionId: string) => AgentSessionRecord | null + /** `AgentSessionRecordStore.replacedRuntime`; absent where no runtime was replaced. */ + replacedRuntime?: (sessionId: string) => { fence: number } | undefined now: () => number /** Where a failing sink or observer is reported; neither may cost subscribers their event. */ logger: StructuredAgentSessionLogger @@ -86,7 +94,10 @@ export class StructuredAgentSessionStatusFeed { /** The user's newest accepted send each session was last projected with; a new one retires * settled children. */ private readonly acceptedSends = new Map() - private readonly projections = new StructuredAgentSessionJournalProjections() + // Read per call: field initializers run before the constructor assigns `deps`. + private readonly projections = new StructuredAgentSessionJournalProjections((sessionId) => + this.deps.replacedRuntime?.(sessionId) + ) constructor(private readonly deps: StructuredAgentSessionStatusFeedDeps) {} @@ -208,8 +219,9 @@ export class StructuredAgentSessionStatusFeed { sessionId: string, journal?: AgentSessionJournal ): StructuredAgentSessionJournalProjection | null { - const source = journal ?? this.deps.sessions.get(sessionId)?.journal - return source ? this.projections.read(source, this.deps.getRecord(sessionId)) : null + const session = this.deps.sessions.get(sessionId) + const source = journal ?? session?.journal + return source ? this.projections.read(source, this.deps.getRecord(sessionId), session) : null } /** Re-projects one session after its journal changed; equal projections are not re-sent. */ @@ -220,7 +232,7 @@ export class StructuredAgentSessionStatusFeed { } const source = journal ?? session.journal const record = this.deps.getRecord(sessionId) - const projection = this.projections.read(source, record) + const projection = this.projections.read(source, record, session) this.retireSettledChildrenOnNewTurn(sessionId, session, projection.acceptedSendKey) const summary = structuredAgentSessionStatusSummary({ sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index f28a49c81aea..23347ef6948b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -7,6 +7,10 @@ import { isStructuredAgentSessionCommandEntry } from '../../../shared/structured import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionWorkEvidence +} from './structured-agent-session-current-work' import { newestAcceptedSendKey } from './structured-agent-session-status-child-work' import { structuredAgentSessionStopping } from './structured-agent-session-stopping' @@ -18,6 +22,12 @@ export type StructuredAgentSessionJournalProjection = { epoch: string sequence: number fence: number | undefined + /** The live generation's fence (`structuredAgentSessionLiveFence`): an exit this host saw, or a + * release, changes what is current without a journal row. */ + liveFence: number | null + /** `StructuredAgentSessionCurrentWork.revision`: what this host saw end with no row or lease + * write still changes the row. */ + workRevision: number /** The Stop marks' settle revision: a settle edge writes no row, so it is a key of its own. */ stopRevision: number state: StructuredAgentSessionStatusState @@ -35,13 +45,34 @@ export class StructuredAgentSessionJournalProjections { StructuredAgentSessionJournalProjection >() + /** `AgentSessionRecordStore.replacedRuntime`, which ends what a replaced runtime held. */ + constructor( + private readonly replacedRuntime?: ( + sessionId: string + ) => StructuredAgentSessionWorkEvidence['replaced'] + ) {} + read( journal: AgentSessionJournal, - record: AgentSessionRecord | null + record: AgentSessionRecord | null, + /** What this host saw end (`StructuredAgentSessionHostSession`). */ + seen?: { + lastEndedChild?: StructuredAgentSessionWorkEvidence['ended'] + operationalRevision?: number + child?: StructuredAgentSessionWorkEvidence['child'] + } ): StructuredAgentSessionJournalProjection { const cursor = journal.cursor() // The conversation's fence, which a child's end moves: its unanswered sends stop counting. const fence = record?.lease.runtimeFence + const ended = seen?.lastEndedChild + const work = structuredAgentSessionCurrentWork(journal, { + record, + replaced: record ? this.replacedRuntime?.(record.sessionId) : undefined, + child: seen?.child ?? null, + ...(ended ? { ended } : {}), + revision: seen?.operationalRevision ?? 0 + }) const stopRevision = journal.stopMarks.revision() let projection = this.byJournal.get(journal) if ( @@ -49,6 +80,8 @@ export class StructuredAgentSessionJournalProjections { projection.epoch !== cursor.epoch || projection.sequence !== cursor.sequence || projection.fence !== fence || + projection.liveFence !== work.liveFence || + projection.workRevision !== work.revision || projection.stopRevision !== stopRevision ) { // A journalled submission bumps `lastSequence`, so the send-time working @@ -81,17 +114,45 @@ export class StructuredAgentSessionJournalProjections { firstInputSubmissionKey, submissionCount: snapshot.submissions.length, fence, + liveFence: work.liveFence, + workRevision: work.revision, stopRevision, - state: projectStructuredAgentSessionStatusState( - snapshot.items, - snapshot.submissions, - fence + // The host's projection of current work: an ended generation's is not Working. + state: withDiedTurnVerdict( + projectStructuredAgentSessionStatusState( + snapshot.items, + snapshot.submissions, + fence, + work.scope + ), + work.diedTurn() ), acceptedSendKey: newestAcceptedSendKey(cursor.epoch, snapshot.submissions), - stopping: structuredAgentSessionStopping(journal, snapshot.items, snapshot.submissions) + stopping: structuredAgentSessionStopping( + journal, + snapshot.items, + snapshot.submissions, + work.scope + ) } this.byJournal.set(journal, projection) } return projection } } + +/** A turn whose generation died on its own reads as its settlement writes it, interrupted, before + * that lands: never a finish with no verdict (`StructuredAgentSessionCurrentWork.diedTurn`). */ +function withDiedTurnVerdict( + state: StructuredAgentSessionStatusState, + died: { turnId: string } | null +): StructuredAgentSessionStatusState { + const request = state.latestRequest + return died && + state.summary.status === 'idle' && + request?.kind === 'turn' && + request.id === died.turnId && + request.turnState === 'running' + ? { ...state, summary: { ...state.summary, turnOutcome: 'interruption' } } + : state +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts index c09d2eba456c..cfd68b683c17 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts @@ -122,7 +122,7 @@ async function stopOfStart(options: { held?: true } = {}): Promise { const handedOver = rig.dispatch.mock.calls.length - const mail = rig.send('mail for the worker') + const mail = rig.send('mail for the retry') await mail.result await rig.proveStart() await eventually(() => expect(rig.dispatch).toHaveBeenCalledTimes(handedOver + 1)) @@ -167,7 +167,7 @@ describe('a Stop of a start that never landed binds no later turn', () => { it('reads a mail turn the child end cut, with no verdict of its own, as news', async () => { await stopOfStart() - const mail = rig.send('mail for the worker') + const mail = rig.send('mail for the retry') await mail.result await turnOpenedBy(mail.id) @@ -180,7 +180,7 @@ describe('a Stop of a start that never landed binds no later turn', () => { await stopOfStart() await mailTurn() const owner = fence() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.host.journalSnapshot(HOST_TEST_SESSION) await settleStaleStructuredAgentSessionState({ @@ -248,7 +248,7 @@ describe('a Stop pressed before its send opened a turn binds only the turn it st expect(laterTurnEndRows()).toEqual([ expect.objectContaining({ state: 'interrupted', outcome: 'cancellation' }) ]) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() }) it('binds no turn that opens after a Stop that stopped nothing', async () => { @@ -346,7 +346,7 @@ describe("a Stop's settle that ends the turn its interrupt took", () => { expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) expect(await laterTurn()).toMatchObject({ state: 'interrupted', outcome: 'cancellation' }) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() }) }) @@ -443,7 +443,8 @@ describe('a rewind that restates a turnless Stop', () => { await journal().replaceEpochItems('handle_forked', 1, [ { identity: stoppedTurn, - body: { ...ended, completedAt: Date.now() + 1, outcome: 'cancellation' } + body: { ...ended, completedAt: Date.now() + 1, outcome: 'cancellation' }, + ownerFence: 1 } ]) const mail = rig.send('mail after the rewind') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts index 17a274d6f5fc..9bba98fcf3da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts @@ -269,7 +269,7 @@ describe('every Stop entry writes its event, with its reason, before it ends the { kind: 'turn', turnId: 'turn-1', state: 'interrupted', completedAt: Date.now() }, { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() const sink = rig.host['runtimeState'].eventSinkFor(HOST_TEST_SESSION) const drained = sink.drained.bind(sink) vi.spyOn(sink, 'drained') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts index 55eb9080a5b1..17c9350dd750 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts @@ -74,7 +74,7 @@ async function runningTurn( async function restartAndSettle( proof: 'pid-absent' | 'exit-observed' | 'unproven' = 'pid-absent' ): Promise { - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.host.journalSnapshot(HOST_TEST_SESSION) const now = Date.now() const deathEvidence: AgentSessionDeathEvidence | null = @@ -364,7 +364,7 @@ describe('a restart between a Stop and its turn end', () => { return new Promise(() => undefined) }) void rig.stop() - await expect.poll(() => journal().activeTurnId()).toBe(TURN) + await expect.poll(() => journal().runningTurn()?.turnId ?? null).toBe(TURN) expect(journal().stopMarks.latest()?.event.turnId).toBeUndefined() await restartAndSettle('exit-observed') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts index ea0dae59199e..f243af3ba3c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts @@ -258,7 +258,7 @@ describe("a Stop's event", () => { await rig.settleAccepted(working, 'working') await turnRow('turn-1', 'interrupted') await eventually(async () => expect((await rig.handoff(next))?.handedOverAt).toBeDefined()) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() const fields = { turnId: 'turn-1' } await rig.host.cancel(QUEUED_RIG_CALLER, { envelope: rig.envelope(fields, 'agentSession.cancel', hostTestOperationId()), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts index 4cbc39073c17..526c82de0b25 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts @@ -45,7 +45,7 @@ async function runningTurn(): Promise { throw new Error('no open journal') } // The turn row landed when the provider's event was handed over. - expect(journal.activeTurnId()).toBe('turn-1') + expect(journal.runningTurn()?.turnId ?? null).toBe('turn-1') return journal } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts index a3fee6b2ed40..ed528949a952 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts @@ -66,7 +66,7 @@ function openedTurn() { function nothingRuns(): boolean { return !isStructuredAgentSessionMainAgentWorking( - journal().activeTurnId(), + journal().runningTurn()?.turnId ?? null, journal().submissions() ) } @@ -169,11 +169,13 @@ describe("a person's close pressed before its send's turn showed", () => { await rig.host['lifetime'].stopAgent(HOST_TEST_SESSION, { cause: 'user-close' }) expect(journal().stopMarks.latest()?.event).toMatchObject({ reason: 'user-close' }) expect(journal().stopMarks.latest()?.event).not.toHaveProperty('turnId') + // The next child's turn, at the fence the stopped one's end moved to. + const fence = rig.store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence await journal().appendItem( LATER, { kind: 'turn', turnId: 'turn-later', state: 'running', startedAt: Date.now() }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await journal().appendItem( LATER, @@ -184,7 +186,7 @@ describe("a person's close pressed before its send's turn showed", () => { startedAt: Date.now(), completedAt: Date.now() + 5 }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) const later = journal() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts index 8750fe909a0a..a6dc5d103b06 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts @@ -11,6 +11,7 @@ import { type AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { isMainAgentWorking } from './structured-agent-session-turns-cancel' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { AgentSessionTurnContext } from './structured-agent-session-turns' import type { JournalStopFailedOn } from '../agent-session-journal/queued-message-pause' import { structuredAgentSessionFailedStopMark } from './structured-agent-session-stopping' @@ -38,7 +39,7 @@ export type StructuredAgentSessionStopWindDown = { * keeps the child on record, and the next operation that reaches the agent joins that close. */ export async function endStoppedStructuredAgentSession( - ctx: Pick, + ctx: Pick, windDown: StructuredAgentSessionStopWindDown, stopChild: () => Promise, onError: (error: unknown) => void @@ -51,7 +52,10 @@ export async function endStoppedStructuredAgentSession( await stopChild() } catch (error) { onError(error) - failedOn = structuredAgentSessionFailedStopMark(ctx.journal) + failedOn = structuredAgentSessionFailedStopMark( + contextStructuredAgentSessionCurrentWork(ctx), + ctx.journal + ) await reviseStopNoteUnconfirmed(ctx, windDown.stopNote).catch(onError) } finally { windDown.settled?.(failedOn) @@ -61,7 +65,7 @@ export async function endStoppedStructuredAgentSession( /** While the work it stopped runs on, the Stop's own note, if it wrote one, says what a lost * interrupt's says. Work that ended took the Stop, whatever became of the child. */ async function reviseStopNoteUnconfirmed( - ctx: Pick, + ctx: Pick, identity: AgentJournalItemIdentity ): Promise { if (!isMainAgentWorking(ctx)) { @@ -85,7 +89,7 @@ async function reviseStopNoteUnconfirmed( // Stop of that turn writes it: its proven end finds it there. A row keeps the scope it was created // with, so a note a Stop wrote before the turn showed is re-keyed, in one batch. Known limit: with // no turn open yet, the note keeps its key and scope, and no turn's end revises it. - const running = ctx.journal.activeTurnId() + const running = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() const turnScope = running !== null ? structuredAgentSessionNamedTurnScope(ctx.journal, running) : null const onTurn = running !== null ? structuredAgentSessionStopNoteIdentity(running) : identity diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts index 92cd6f574331..1912d3485098 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts @@ -13,6 +13,7 @@ import { type AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' import type { AgentSessionCancelOutcome } from './structured-agent-session-adapter' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionStatusEvent, AgentSessionStatusSummary @@ -98,7 +99,12 @@ function stopAnswers(): (string | undefined)[] { } /** A send whose turn `turn-1` is running, and the session's status. */ -async function runningTurn(options: { stopEndsSession?: true } = {}) { +async function runningTurn( + options: Pick< + NonNullable[0]>, + 'stopEndsSession' | 'probeOwner' + > = {} +) { rig = await createQueuedMessageTestRig(options) const sent = await rig.workingSend() await rig.settleAccepted(sent, 'sent') @@ -268,7 +274,7 @@ describe('a Stop that failed before the turn it meant to stop opened', () => { expect(await rig.stop()).toMatchObject({ ok: true }) // A Stop that ends its session ends the child on the session's next step. await rig.host['tasks'].serialize(HOST_TEST_SESSION, async () => {}) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() return { sent, status } } @@ -283,7 +289,7 @@ describe('a Stop that failed before the turn it meant to stop opened', () => { await rig.settleAccepted(sent, 'sent') await turn('turn-1', sent, 'running') - await eventually(() => expect(journal().activeTurnId()).toBe('turn-1')) + await eventually(() => expect(journal().runningTurn()?.turnId ?? null).toBe('turn-1')) expect(status()).toMatchObject({ status: 'working', stopping: true }) await turn('turn-1', sent, 'interrupted') await eventually(() => expect(status()?.status).toBe('idle')) @@ -351,7 +357,7 @@ describe('a Stop pressed before its send opened a turn', () => { it('ends the turn its interrupt took at the settle, and Stopping with it', async () => { const { status } = await stopAsTheTurnOpens({ cancelled: true, turnId: 'turn-1' }) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() await eventually(() => expect(status()?.status).toBe('idle')) expect(status()).not.toHaveProperty('stopping') }) @@ -390,7 +396,7 @@ describe('a Stop pressed before its send opened a turn', () => { expect(status()).not.toHaveProperty('stopping') await rig.settleAccepted(sent, 'sent') await turn('turn-1', sent, 'running') - await eventually(() => expect(journal().activeTurnId()).toBe('turn-1')) + await eventually(() => expect(journal().runningTurn()?.turnId ?? null).toBe('turn-1')) expect(status()).not.toHaveProperty('stopping') }) @@ -468,12 +474,12 @@ describe('a Stop whose provider ends its session', () => { // The close lives on the child, in memory, and dies with the host. The new host's settlement, // which ends the turn on a proof of the old owner's death, is what says the Stop took. - async function crashAfterUnconfirmedStop(): Promise { - await runningTurn({ stopEndsSession: true }) + async function crashAfterUnconfirmedStop(proof: AgentSessionOwnerProbe): Promise { + await runningTurn({ stopEndsSession: true, probeOwner: async () => proof }) rig.closeSession.mockRejectedValueOnce(new Error('the kill timed out')) expect(await rig.stop()).toMatchObject({ ok: true }) await eventually(() => expect(stopAnswers()).toEqual(['cancelUnconfirmed'])) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() } function turnOneState(): string | undefined { @@ -483,21 +489,9 @@ describe('a Stop whose provider ends its session', () => { .find((turn) => turn?.turnId === 'turn-1')?.state } - /** What the host's restart reconciliation or recovery writes once a probe finds the old pid gone. */ - async function proveOldOwnerGone(): Promise { - const record = rig.store.getRecord(HOST_TEST_SESSION)! - await rig.store.evictProvenDeadOwner({ - sessionId: HOST_TEST_SESSION, - expectedFence: record.lease.runtimeFence, - probe: { outcome: 'pid-absent' }, - now: Date.now() - }) - } - it("says the Stop took once the new host proves the old child's exit after a crash", async () => { - await crashAfterUnconfirmedStop() - // Proven before the chat opens, as the restart's reconciliation does on most machines. - await proveOldOwnerGone() + // Proven by the restart's reconciliation, as on most machines; its startup settles the chat. + await crashAfterUnconfirmedStop({ outcome: 'pid-absent' }) await rig.queuePause() expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ @@ -506,22 +500,22 @@ describe('a Stop whose provider ends its session', () => { }) }) - it('keeps the note unconfirmed while the old exit stays unverifiable, then says it took once proven', async () => { - await crashAfterUnconfirmedStop() + it('keeps the note unconfirmed while the old exit stays unverifiable, until a later restart replaces its runtime', async () => { + // The restart cannot prove the old owner gone: it releases it unproven, and startup settles. + await crashAfterUnconfirmedStop({ outcome: 'indeterminate', reason: 'no answer' }) await rig.queuePause() expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ turn: 'unverifiable', notes: ['cancelUnconfirmed'] }) - // A later proof naming the old owner, as recovery writes it, revises the open chat. - await proveOldOwnerGone() - await eventually(() => - expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ - turn: 'interrupted', - notes: ['took'] - }) - ) + // A later restart replaced the runtime that held the old owner, which ends its turn: the Stop took. + await rig.crashRestartHostProcess() + await rig.queuePause() + expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ + turn: 'interrupted', + notes: ['took'] + }) }) // A Stop pressed before its turn showed keys its note by itself, with no turn to sit on; once it is diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts index 284a474b9c65..c491814f542b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts @@ -10,6 +10,8 @@ import type { } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-live-turn' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { stopIsAPersons } from '../agent-session-journal/journal-stop-turn-end' import type { JournalStopFailedOn } from '../agent-session-journal/queued-message-pause' @@ -19,7 +21,9 @@ import type { JournalStopFailedOn } from '../agent-session-journal/queued-messag export function structuredAgentSessionStopping( journal: Pick, items: readonly AgentJournalRenderItem[], - submissions: readonly AgentJournalSubmission[] = [] + submissions: readonly AgentJournalSubmission[] = [], + /** The host's projection of current work: a Stop bound to an ended generation's turn is over. */ + scope?: StructuredAgentSessionWorkScope ): boolean { const stop = journal.stopMarks.latest() if (stop === null || !stopIsAPersons(stop.event.reason)) { @@ -33,7 +37,10 @@ export function structuredAgentSessionStopping( stop.event.turnId ?? stop.settle?.turnId ?? (failed && 'turnId' in failed ? failed.turnId : undefined) - const live = activeStructuredAgentSessionTurnId(items) + const live = activeStructuredAgentSessionTurnId( + items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ) if (bound !== undefined) { return bound === live } @@ -64,9 +71,10 @@ function firstSendHandedOverAfter( /** Where a Stop that failed leaves "Stopping…" (display only): the turn running now, or, with * none, the first that opens after this position. */ export function structuredAgentSessionFailedStopMark( - journal: Pick + work: Pick, + journal: Pick ): JournalStopFailedOn { - const live = journal.activeTurnId() + const live = work.activeTurnId() return live !== null ? { turnId: live } : { openedAfter: journal.cursor().sequence } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index dcd3e07c4f2a..5e0470430e6e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -22,6 +22,7 @@ import { emptyAgentSessionBatch } from './agent-session-empty-batch' import { readAgentSessionHydrationPage } from './agent-session-history-page' import { rememberSessionActivity } from './structured-agent-session-activity-retention' import { refreshDerivedStopNotes } from './agent-session-stop-note-refresh' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' export type AgentSessionSubscriberEmit = (event: AgentSessionSubscribeEvent) => void export type AgentSessionSubscribeInput = { @@ -43,6 +44,9 @@ export type Subscriber = { queuePublication?: QueuePublication /** Fingerprint of the background-task roster last SENT. */ backgroundTasks?: string + /** The current-work view the last frame's fields were read under + * (`StructuredAgentSessionCurrentWork.viewKey`); a change with no row still owes a frame. */ + workView?: string } export type AgentSessionSubscribersHooks = { @@ -53,6 +57,12 @@ export type AgentSessionSubscribersHooks = { readBackgroundTasks?: SubscriberFieldHooks['readBackgroundTasks'] /** Fires after publications that can change journal content. */ onJournalPublished?: (sessionId: string, journal: AgentSessionJournal) => void + /** The host's projection of current work (`structuredAgentSessionCurrentWork`), which every + * page and frame states as its turn and prompt fields. */ + readCurrentWork?: ( + sessionId: string, + journal: AgentSessionJournal + ) => StructuredAgentSessionCurrentWork | undefined now?: () => number } @@ -100,10 +110,13 @@ export class AgentSessionSubscribers { }, subscriber, input.journal, - hostNow + hostNow, + this.hooks.readCurrentWork?.(input.sessionId, input.journal) ) } else { - const page = readAgentSessionHydrationPage(input.journal, input.fence) + const work = this.hooks.readCurrentWork?.(input.sessionId, input.journal) + const page = readAgentSessionHydrationPage(input.journal, input.fence, work) + subscriber.workView = work?.viewKey() this.emit(subscriber, { type: 'snapshot', sessionId: input.sessionId, @@ -144,20 +157,36 @@ export class AgentSessionSubscribers { this.activityBySession.delete(sessionId) } } + this.deliverFrames(sessionId, journal, activity) + if (activity === undefined) { + this.hooks.onJournalPublished?.(sessionId, journal) + } + } + + /** The queue as frames publish it, for a history page to read the same. */ + readQueuePublication(sessionId: string): QueuePublication | undefined { + return this.hooks.readQueuePublication?.(sessionId) + } + + /** Each subscriber's frame for what changed, with none of a journal write's other edges. */ + deliverFrames( + sessionId: string, + journal: AgentSessionJournal, + activity?: AgentSessionTurnActivity | null + ): void { const hostNow = this.now() for (const subscriber of this.subscribers(sessionId)) { this.deliver(subscriber, journal, hostNow, false, activity) } - if (activity === undefined) { - this.hooks.onJournalPublished?.(sessionId, journal) - } } /** Every subscriber back to a bounded tail page. */ snapshot(sessionId: string, journal: AgentSessionJournal, fence: number): void { - const page = readAgentSessionHydrationPage(journal, fence) + const work = this.hooks.readCurrentWork?.(sessionId, journal) + const page = readAgentSessionHydrationPage(journal, fence, work) const hostNow = this.now() for (const subscriber of this.subscribers(sessionId)) { + subscriber.workView = work?.viewKey() this.emit(subscriber, { type: 'snapshot', sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts index f0237835c756..2bc2ed3ab304 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts @@ -8,6 +8,11 @@ export class StructuredAgentSessionTaskQueue { return runKeyedSerializedOperation(this.chains, sessionId, task) } + /** Whether anything holds or waits for the chat's lane now. */ + busy(sessionId: string): boolean { + return this.chains.has(sessionId) + } + trackAttach(operation: Promise): Promise { this.attaching.add(operation) void operation.then( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts index 4af804263aec..1310663c9989 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts @@ -13,6 +13,7 @@ import type { } from '../../../shared/agent-session-wire' import type { MutationPlan } from './structured-agent-session-mutation-plans' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' function refused( reason: AgentSessionRefusalReason<'agent_session_operation_invalid'>, @@ -70,7 +71,7 @@ export async function performThreadGoalChange( sentAs: 'goal' }, // Accepting a goal is delivering it, so it joins whatever turn runs now. - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) } const withdrawObjective = async (): Promise => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index d1cfdec43356..a6b5b9b30be9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -119,6 +119,7 @@ function transitionAppend( batch: { settlementId: step.settlementId, fence, + ownerFence: fence, recovered: step.recovered, resolve: () => step.resolve(journal) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts index 0f86c053ad66..4a954f7e8fa1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts @@ -5,15 +5,16 @@ import type { AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' export const STOP_NOTE_CANCELLATION_REQUESTED = 'Cancellation requested.' -/** The turn a Stop is about: the one it named, else the one running when it is read. */ +/** The turn a Stop is about: the one it named, else the live generation's running turn. */ export function structuredAgentSessionStoppedTurnId( - journal: Pick, + work: Pick, namedTurnId: string | undefined ): string | null { - return namedTurnId ?? journal.activeTurnId() + return namedTurnId ?? work.activeTurnId() } /** A Stop names a turn the journal does not show running, as a phone does once that turn ended. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index a588fa7168af..ab65c4512cc8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -25,7 +25,7 @@ import { structuredAgentSessionStopNamesTurnNotLive, structuredAgentSessionStoppedTurnId } from './structured-agent-session-turn-stop-notes' -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { StructuredAgentSessionStopWindDown } from './structured-agent-session-stop-wind-down' import type { JournalStopFailedOn } from '../agent-session-journal/queued-message-pause' import { structuredAgentSessionFailedStopMark } from './structured-agent-session-stopping' @@ -33,16 +33,12 @@ import { sendStopCanTakeBack } from './structured-agent-session-unopened-send-wi import { settleTakenStop } from './structured-agent-session-taken-stop-settle' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' -/** Whether the fold reads working. Every write has landed by its call's return, so a Stop reads it - * without waiting on the write queue. */ +/** Whether the host's projection reads working (`structuredAgentSessionCurrentWork`). Every write + * has landed by its call's return, so a Stop reads it without waiting on the write queue. */ export function isMainAgentWorking( - ctx: Pick + ctx: Pick ): boolean { - return isStructuredAgentSessionMainAgentWorking( - ctx.journal.activeTurnId(), - ctx.journal.submissions(), - ctx.fence - ) + return contextStructuredAgentSessionCurrentWork(ctx).working() } /** @@ -51,14 +47,15 @@ export function isMainAgentWorking( * its exit ends its turn, and the host's own exit handling waits behind this step. */ function stillRunsStoppedTurn( - ctx: Pick, + ctx: Pick, stoppedTurnId: string | null ): boolean { - if (!isMainAgentWorking(ctx)) { + const work = contextStructuredAgentSessionCurrentWork(ctx) + if (!work.working()) { return false } // Working with no turn open after the Stop's turn is a later send whose turn has not opened. - return stoppedTurnId === null || ctx.journal.activeTurnId() === stoppedTurnId + return stoppedTurnId === null || work.activeTurnId() === stoppedTurnId } /** Whether the Stop took back every send it found in flight, none of them having run @@ -148,6 +145,8 @@ type PerformCancelInput = { childWork?: () => readonly AgentChildWorkView[] | undefined /** The latest Stop event is this press's own, or the in-force one it repeats: its settle binds. */ opensSettle?: true + /** The fence the note is written at: a child end this Stop caused moved it past `ctx.fence`. */ + noteFence?: () => number } async function cancelAndNote( @@ -161,17 +160,20 @@ async function cancelAndNote( kind: 'status', text: STOP_NOTE_CANCELLATION_REQUESTED } + // Every read of what runs goes through the host's projection: an ended generation's turn is no + // Stop's target, whether or not its cleanup landed. + const work = () => contextStructuredAgentSessionCurrentWork(ctx) // The turn the Stop names, read before the cancel settles it: the note reports on that turn. const turnScope = (input.turnId !== undefined && !input.scope ? structuredAgentSessionNamedTurnScope(ctx.journal, input.turnId) - : null) ?? ctx.journal.liveTurnScope() + : null) ?? work().turnScope() // Only the provider's end or the child's ends a command. A command the provider has not opened a // turn for, would not interrupt, or was already asked to stop, ends with its child; that child's // dead-generation settlement writes the command's verdict. - const liveTurnId = ctx.journal.activeTurnId() + const liveTurnId = work().activeTurnId() // Read with the live turn, before the cancel settles it: the note is keyed by this turn. - const stoppedTurnId = structuredAgentSessionStoppedTurnId(ctx.journal, input.turnId) + const stoppedTurnId = structuredAgentSessionStoppedTurnId(work(), input.turnId) const namesTurnNotLive = structuredAgentSessionStopNamesTurnNotLive(input.turnId, liveTurnId) const runningCommand = input.stopChild !== undefined && @@ -223,7 +225,7 @@ async function cancelAndNote( ...(input.turnId !== undefined ? { turnId: input.turnId } : {}), fence: ctx.fence, // The journal is what the client read to name a turn, so it is what judges the request. - resolveLiveTurnId: () => ctx.journal.activeTurnId(), + resolveLiveTurnId: () => work().activeTurnId(), ...(dispatchStatus ? { dispatchStatus } : {}), ...(input.prompt ? { prompt: { itemId: input.prompt.itemId } } : {}) }) @@ -330,7 +332,7 @@ async function cancelAndNote( binding.turnId = stoppedTurn ?? stoppedTurnId ?? undefined } else if (interruptFailed) { // A Stop that failed reads "Stopping…" through the turn it could not stop; its end stays its own. - binding.failedOn = structuredAgentSessionFailedStopMark(ctx.journal) + binding.failedOn = structuredAgentSessionFailedStopMark(work(), ctx.journal) } if (taken === true && stoppedTurn !== undefined && !binding.closedByWindDown && !input.scope) { const opened = await settleTakenStop(ctx, stoppedTurn) @@ -349,6 +351,7 @@ async function cancelAndNote( (stoppedTurn !== undefined ? structuredAgentSessionNamedTurnScope(ctx.journal, stoppedTurn) : null) ?? turnScope - await ctx.journal.appendItem(noteIdentity, note, { fence: ctx.fence, turnScope: noteScope }) + const fence = Math.max(ctx.fence, input.noteFence?.() ?? ctx.fence) + await ctx.journal.appendItem(noteIdentity, note, { fence, turnScope: noteScope }) return { ok: true, value } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts index a7bbf523c3ce..5b8f090b22ea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts @@ -26,6 +26,7 @@ import { import type { CommandReceiptResult } from '../agent-session-journal/command-receipt-schema' import type { JournalRow } from '../agent-session-journal/journal-row-schema' import { settledPrompt, validatePendingPrompt } from './structured-agent-session-prompt-state' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' export type AgentSessionPromptRequest = { @@ -99,6 +100,14 @@ export async function performPrompt( return held ? { ok: true, value: held } : validated } const { prompt } = validated + const work = contextStructuredAgentSessionCurrentWork(ctx) + if (!work.isCurrentItem(input.itemId)) { + // The agent that asked has ended: an answer would reach whichever agent runs now, or none. + return invalid( + 'promptOwnerEnded', + `Item ${input.itemId} was raised by an agent that has ended.` + ) + } const choice = readPromptChoice(prompt, input) if (!choice) { return invalid( @@ -134,7 +143,7 @@ export async function performPrompt( identity, { ...prompt, resolution }, // A revision: the prompt keeps the turn it was raised in. - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() }, + { fence: ctx.fence, turnScope: work.turnScope() }, // The answer's identity commits here, before the adapter tells the provider. ctx.operationReceipt ) @@ -159,7 +168,7 @@ export async function performPrompt( surface: 'row' }) }, - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) } const appended = committed.item diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index cb3d87e72e95..b9c6b5e0bbc5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -24,6 +24,10 @@ import { } from '../../../shared/agent-session-wire' import { isAgentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { DISPATCH_DOUBT_PERSISTENCE_FAILED } from '../agent-session-journal/journal-dispatch-doubt-reasons' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { @@ -53,6 +57,9 @@ export type AgentSessionTurnContext = { sessionId: string journal: AgentSessionJournal fence: number + /** The host's projection of what work is current (`structuredAgentSessionCurrentWork`), read + * fresh per call: a child's end can move it mid-operation. */ + currentWork?: () => StructuredAgentSessionCurrentWork | null adapter: StructuredAgentSessionAdapter /** What each agent declares; the session's own answer is `agents.capabilities(agent)`. */ agents: StructuredAgentRegistry @@ -219,12 +226,13 @@ export async function handOverSubmission( await handOverStructuredAgentSessionCommand(ctx, submission, body) return } - // The message joins the turn running at handover, a steer, or opens its own. + // The message joins the turn the child it goes to is running, a steer, or opens its own; never a + // turn an ended generation left. await ctx.journal.resolveDispatch({ clientMessageId, state: 'pending', fence: ctx.fence, - turnScope: ctx.journal.liveTurnScope() + turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() }) // The handover row's instant on the host clock; the turn this dispatch opens records it so the // live counter never re-anchors at turn-open. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts new file mode 100644 index 000000000000..94bbf5a82260 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts @@ -0,0 +1,202 @@ +// After a restart, an owner no adjudication has concluded about (unreconciled: its probe failed; +// recovering: proven alive, same runtime) on a chat no tab shows. Every host reader reads its +// leftover turn as the chat shows it: unverifiable, not working. The earlier process's card waits +// under the restart's pause; a person's own send goes to a start, which decides the owner. + +import { afterEach, expect, it, vi } from 'vitest' +import { + HOST_TEST_SESSION as SESSION, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + currentJournal, + leaveUnfinishedWork, + sendText +} from './structured-agent-session-leftover-settlement.test-fixture' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' +import { setStructuredAgentSessionHost } from './structured-agent-session-registry' +import { readStructuredSessionGateFacts } from '../../runtime/orchestration/structured-mailbox-pointer-host' +import { structuredWorkerAgentStatus } from '../../runtime/orchestration/structured-worker-group-addressing' + +const MODES = ['unreconciled', 'recovering'] as const + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + setStructuredAgentSessionHost(null) + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** The chat reopened in a new process with its owner undecided; `earlier` is a card the earlier + * process queued while it worked. Answers the owner-stop spy and the probe's call count. */ +async function reopened( + mode: (typeof MODES)[number], + options: { leftoverTurn: boolean; earlier?: 'message' | 'compact' } +) { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const probes = { calls: 0 } + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => { + probes.calls += 1 + if (mode === 'unreconciled') { + throw new Error('the probe could not run') + } + return { outcome: 'identity-matched', matchedOn: ['process-start-time'] } + } + }) + const current = rig + await current.workingSend() + let earlierCard: string | undefined + if (options.earlier === 'message') { + const queued = await sendText(current, 'queued before the crash').result + earlierCard = queued.ok && 'queued' in queued.value ? queued.value.queued.messageId : undefined + } else if (options.earlier === 'compact') { + earlierCard = await queueCompact(current) + } + if (options.leftoverTurn) { + await leaveUnfinishedWork(current) + } + // No tab restore: nothing at startup decides the owner. + await current.crashReloadHostProcess(() => undefined) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + const stopOwnerProcess = vi.fn() + current.host.deps.stopOwnerProcess = stopOwnerProcess + const lease = current.store.getRecord(SESSION)?.lease + expect(mode === 'unreconciled' ? lease?.unreconciled : lease?.handoffStage).toBe( + mode === 'unreconciled' ? true : 'recovering' + ) + return { current, stopOwnerProcess, probes, earlierCard } +} + +async function queueCompact(current: QueuedMessageTestRig): Promise { + const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const } + const id = hostTestOperationId() + const result = await current.host.conversationCommand(CALLER, { + envelope: current.envelope(fields, 'agentSession.conversationCommand', id), + ...fields, + userSend: true + }) + return result.ok && result.value.queued ? result.value.queued.messageId : undefined +} + +async function page(current: QueuedMessageTestRig) { + const read = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + if (!read.ok) { + throw new Error('history refused') + } + return read.page +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 3_000)) + +it.each(MODES)( + "holds the earlier process's card under the restart's pause while the owner is %s: nothing sent or stopped", + async (mode) => { + const { current, stopOwnerProcess, earlierCard } = await reopened(mode, { + leftoverTurn: true, + earlier: 'message' + }) + expect(earlierCard).toBeDefined() + const starts = current.starts.length + const dispatches = current.dispatch.mock.calls.length + + await settle() + + expect(structuredQueuePauses(currentJournal(current)).map((pause) => pause.reason)).toContain( + 'restarted' + ) + const held = await page(current) + expect(held.working).toBe(false) + expect(held.nextQueuedMessageId ?? null).toBeNull() + expect(held.queuedMessages?.map((card) => [card.messageId, card.state])).toEqual([ + [earlierCard, 'waiting'] + ]) + expect(current.starts.length).toBe(starts) + expect(current.dispatch.mock.calls.length).toBe(dispatches) + expect(stopOwnerProcess).not.toHaveBeenCalled() + + // The person's own send is no card the queue could send for them later: it goes to a start. + const sent = await sendText(current, 'while it may still run').result + expect(sent).toMatchObject({ ok: true, value: { submission: expect.anything() } }) + await settle() + const after = await page(current) + expect(after.queuedMessages?.map((card) => [card.messageId, card.state])).toEqual([ + [earlierCard, 'waiting'] + ]) + expect(after.queuedMessages?.[0]?.returnedReason).toBeUndefined() + expect(await current.handoff(earlierCard ?? '')).toBeUndefined() + }, + 30_000 +) + +it.each(MODES.flatMap((mode) => [true, false].map((leftoverTurn) => ({ mode, leftoverTurn }))))( + "sends a person's queue-if-active message to a start that decides the owner ($mode, leftover turn: $leftoverTurn)", + async ({ mode, leftoverTurn }) => { + const { current, stopOwnerProcess, probes } = await reopened(mode, { leftoverTurn }) + expect((await page(current)).working).toBe(false) + const probed = probes.calls + + const sent = await sendText(current, 'while it may still run').result + + expect(sent).toMatchObject({ ok: true, value: { submission: expect.anything() } }) + expect(sent.ok && 'queued' in sent.value).toBe(false) + await vi.waitFor(() => expect(probes.calls).toBeGreaterThan(probed), { timeout: 10_000 }) + expect((await page(current)).queuedMessages ?? []).toEqual([]) + // A recovery the start resolves ends the owner it proved alive, as on main; no proof, no stop. + if (mode === 'recovering') { + await vi.waitFor(() => expect(stopOwnerProcess).toHaveBeenCalledOnce(), { timeout: 10_000 }) + } else { + expect(stopOwnerProcess).not.toHaveBeenCalled() + } + }, + 30_000 +) + +it.each(MODES)( + "sends a /compact card on Send while the chat shows the %s owner's turn as not working", + async (mode) => { + const { current, earlierCard } = await reopened(mode, { + leftoverTurn: true, + earlier: 'compact' + }) + expect(earlierCard).toBeDefined() + expect((await page(current)).working).toBe(false) + + const sent = await current.sendNow(earlierCard ?? '') + + expect(sent).toMatchObject({ + ok: true, + value: { submission: { queuedMessageId: earlierCard } } + }) + }, + 30_000 +) + +it.each(MODES)( + "reads the %s owner's leftover turn for @idle as the chat shows it", + async (mode) => { + const { current } = await reopened(mode, { leftoverTurn: true }) + setStructuredAgentSessionHost(current.host) + const record = current.store.getRecord(SESSION) + if (!record) { + throw new Error('expected the record') + } + const { working } = await page(current) + + const facts = await readStructuredSessionGateFacts(SESSION) + const status = await structuredWorkerAgentStatus({ sessionId: SESSION, record, lineage: [] }) + + expect(working).toBe(false) + expect(facts?.turnRunning).toBe(working) + expect(status).toBe('idle') + }, + 30_000 +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index b1fdcb5d8605..7b5e7c9c1163 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -14,6 +14,11 @@ import { type StructuredAgentSessionChildExitSession } from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import { + exitJournalDouble, + plannedBatchThrough +} from './structured-agent-session-settlement-double.test-fixture' +import type { JournalLifecycleBatchInput } from '../agent-session-journal/journal-store-contracts' const exitOutcome = (agent: string): string => `${agent} stopped while this response was in progress. You can continue in this conversation.` @@ -101,7 +106,8 @@ describe('provider-exit settlement', () => { itemFence: () => 7, lastProviderActivityAt: () => undefined, stopMarks: { latest: () => null, personStopDecides: () => false }, - appendLifecycleBatch + submissions: () => [], + appendPlannedLifecycleBatch: plannedBatchThrough(appendLifecycleBatch) } } as unknown as StructuredAgentSessionHostSession @@ -115,7 +121,7 @@ describe('provider-exit settlement', () => { now = 60_000 return { ok: false, error: new Error('sink unavailable') } }, - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId, task) => task(), now: () => now } as never, @@ -173,7 +179,8 @@ describe('provider-exit settlement', () => { itemBody: () => null, itemFence: () => undefined, snapshot: () => ({ items }), - appendLifecycleBatch + submissions: () => [], + appendPlannedLifecycleBatch: plannedBatchThrough(appendLifecycleBatch) } } as unknown as StructuredAgentSessionHostSession const store = { @@ -191,11 +198,12 @@ describe('provider-exit settlement', () => { } await settleStructuredAgentSessionChildExit( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit settlement reads only these members of its context; the store double is partial. { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId, task) => task(), now: () => 1_234 } as never, @@ -278,19 +286,13 @@ describe('provider-exit settlement', () => { ...(initialState === 'running' ? {} : { completedAt: 40 }) }) ] - const appendLifecycleBatch = vi.fn(async (_input: { mutations: readonly unknown[] }) => ({ + const appendLifecycleBatch = vi.fn(async (_input: JournalLifecycleBatchInput) => ({ epoch: 'epoch-1', sequence: 3 })) const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - snapshot: () => ({ items }), - appendLifecycleBatch - } + journal: exitJournalDouble({ items: () => items, appendLifecycleBatch }) } const { store } = mutableStore() @@ -308,7 +310,7 @@ describe('provider-exit settlement', () => { ] return { ok: true } }, - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1_234 } @@ -339,16 +341,15 @@ describe('provider-exit settlement', () => { ) it('settles a submission the dead child never acknowledged', async () => { + const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })) const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - snapshot: () => ({ items: [] }), - appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), - submissions: () => [{ clientMessageId: 'client-1', dispatchState: 'pending' }] - } + journal: exitJournalDouble({ + items: () => [], + appendLifecycleBatch, + // Handed to the child that exited, at its fence. + submissions: () => [{ clientMessageId: 'client-1', dispatchState: 'pending', fence: 7 }] + }) } const { store } = mutableStore() @@ -357,7 +358,7 @@ describe('provider-exit settlement', () => { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } @@ -370,8 +371,11 @@ describe('provider-exit settlement', () => { acquisitionGeneration: GENERATION }) - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + // Written at the exited child's own fence, before its release moves it, as is every row its + // settlement writes. + expect(appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ + fence: 7, dispatches: [ { clientMessageId: 'client-1', @@ -398,20 +402,15 @@ describe('provider-exit settlement', () => { it('releases without offering a restart while terminal settlement is failing', async () => { const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - snapshot: () => ({ - items: [lifecycleItem('turn-failing', 1, { state: 'running', startedAt: 1 })] - }), + journal: exitJournalDouble({ + items: () => [lifecycleItem('turn-failing', 1, { state: 'running', startedAt: 1 })], appendLifecycleBatch: vi.fn(async () => { throw new Error('journal still unavailable') }) - } + }) } const log = recordingStructuredAgentSessionLogger() - const publishFence = vi.fn() + const generationEnded = vi.fn() const event = { type: 'ended' as const, sessionId: SESSION, @@ -425,7 +424,7 @@ describe('provider-exit settlement', () => { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: false, error: new Error('sink failed') }), - publishFence, + generationEnded, serialize: async (_sessionId, task) => task(), now: () => 1, logger: log.logger @@ -433,7 +432,14 @@ describe('provider-exit settlement', () => { await settleStructuredAgentSessionChildExit(context, event) expect(session.child).toBeNull() - expect(publishFence).toHaveBeenCalledTimes(1) + // The death first, before anything is written; then the exit's own account, for the retry to + // write once the journal takes it. + expect(generationEnded).toHaveBeenCalledTimes(2) + expect(generationEnded).toHaveBeenNthCalledWith(1, SESSION) + expect(generationEnded).toHaveBeenLastCalledWith(SESSION, { + restate: true, + exit: expect.objectContaining({ ownerFence: 7, showUnexpectedExitOutcome: true }) + }) expect(log.scopes()).toEqual(['exit-lifecycle-barrier', 'exit-settlement']) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts index ed1e642da39e..e0a98e7d7716 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts @@ -11,7 +11,7 @@ import { import type { UnopenedSendJournal } from './structured-agent-session-unopened-send-withdrawal' export type DeadGenerationJournal = UnopenedSendJournal & { - appendLifecycleBatch: AgentSessionJournal['appendLifecycleBatch'] + appendPlannedLifecycleBatch: AgentSessionJournal['appendPlannedLifecycleBatch'] pendingSubmissions?: AgentSessionJournal['pendingSubmissions'] itemFence: AgentSessionJournal['itemFence'] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts index e1022722955f..deeb87fb23e9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts @@ -24,7 +24,7 @@ export type UnopenedSendJournal = { | 'handoverRecorded' | 'handedOverAt' | 'acceptedSequence' - > & { reason?: string | null })[] + > & { reason?: string | null; fence?: number })[] resolveDispatch?: AgentSessionJournal['resolveDispatch'] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts index e4de581bf067..1a2f7b1f2bb1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts @@ -106,7 +106,7 @@ describe('a message held behind a start that never answered, then the chat ends' async (cause) => { const id = await heldBehindHungStart('kept at close') await rig.host.close(SESSION, cause) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts index 872d08c1da40..d4b653c4326f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts @@ -110,9 +110,10 @@ async function closeWhileStarting(cause: 'user-close' | 'evict'): Promise await closed } -/** The app quits: delivery stops first, so the held spawn finishing hands nothing over. */ -async function quitRestart(): Promise { - const quit = rig.quitRestartHostProcess() +/** The app quits: delivery stops first, so the held spawn finishing hands nothing over. + * `beforeStartup` reads the state directory before the next process starts. */ +async function quitRestart(beforeStartup?: () => unknown): Promise { + const quit = rig.quitRestartHostProcess(beforeStartup) releaseStart() releaseStart = () => {} await quit @@ -161,17 +162,19 @@ describe('a message accepted while the agent starts, then Orca stops', () => { }) // Nothing is sent on its own, however many times the chat reopens. await rig.host.close(SESSION, 'evict') - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }]) expect(rig.dispatch).not.toHaveBeenCalled() } ) - it('quit writes nothing for it: no closed-chat rejection, still queued until the next open', async () => { + it('quit writes nothing for it: no closed-chat rejection, still pending until the next startup', async () => { const id = await acceptWhileStarting(sendRequest('kept through quit')) - await quitRestart() - // Read beside the host, without opening the chat through it: the row the quit left. - const leftover = await peekSubmission(id) + let leftover: Awaited> | undefined + // Read beside the host, between the two processes: the row the quit left. + await quitRestart(async () => { + leftover = await peekSubmission(id) + }) expect(leftover).toMatchObject({ dispatchState: 'pending', handoverRecorded: true }) expect(leftover?.handedOverAt).toBeUndefined() }) @@ -290,7 +293,7 @@ describe('the next accepted turn releases a kept card', () => { const id = await acceptWhileStarting(sendRequest('kept twice over')) await quitRestart() expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }]) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await new Promise((resolve) => setTimeout(resolve, 100)) expect(rig.dispatch).not.toHaveBeenCalled() const working = await rig.workingSend() @@ -352,11 +355,7 @@ describe('cards queued behind a working turn, then Orca stops', () => { ok: true, value: { queued: { state: 'waiting' } } }) - if (how === 'quit') { - await quitRestart() - } else { - rig.crashRestartHostProcess() - } + await (how === 'quit' ? quitRestart() : rig.crashRestartHostProcess()) expect(await rig.drafts()).toEqual([ { messageId: first.id, state: 'waiting' }, @@ -429,7 +428,7 @@ describe('the queue at a quit', () => { } owed.mockRestore() healing.mockRestore() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.drafts()).toEqual([{ messageId: queued.id, state: 'waiting' }]) expect(await rig.queuePause()).toBeNull() @@ -470,7 +469,7 @@ describe('a message accepted while the agent starts, then the chat closes', () = async (cause) => { const id = await acceptWhileStarting(sendRequest('kept at close')) await closeWhileStarting(cause) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', ...CHAT_CLOSED }) expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }]) @@ -583,7 +582,7 @@ describe('what is not kept', () => { vi.spyOn(JournalQueuedMessages.prototype, 'holdInTransaction').mockImplementation(() => { throw new Error('disk full') }) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await eventually(async () => expect(await rig.submission(id)).toMatchObject({ @@ -600,7 +599,7 @@ describe('what is not kept', () => { ) }) - it('the delivery step keeps one the open could not settle, and hands nothing over', async () => { + it('a failed keep rejects nothing: a fresh send is delivered, the earlier one is never handed over, and the retry keeps it', async () => { const id = await acceptWhileStarting(sendRequest('open write failed')) const resolve = AgentSessionJournal.prototype.resolveDispatch // The open's keep and its plain-rejection fallback both fail. @@ -617,12 +616,20 @@ describe('what is not kept', () => { return resolve.apply(this, args) }) await quitRestart() - // The open's write failed; a new send wakes the delivery loop, whose first step settles it. + // The open's keep failed. Delivery does not wait on that bookkeeping: what may be handed over + // is derived from where each send sits, so only this process's own send goes. const next = rig.send('wakes the loop') - await next.result - await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) + expect(await next.result).toMatchObject({ ok: true }) await eventually(() => expect(dispatchedTexts()).toEqual(['wakes the loop'])) - expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', ...HOST_RESTARTED }) + // The retry's next round lands the bookkeeping in this process and keeps it as a card. + await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) + const earlier = await rig.submission(id) + expect(earlier).toMatchObject({ dispatchState: 'rejected', ...HOST_RESTARTED }) + expect(earlier).not.toHaveProperty('handedOverAt') + // The next open finds it kept, and nothing hands it over. + await quitRestart() + await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) + expect(dispatchedTexts()).toEqual(['wakes the loop']) }) }) @@ -758,7 +765,7 @@ describe("the reopen's mark", () => { await queued.result void working const sent = rig.dispatch.mock.calls.length - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.host.restoreReadableSessions([SESSION]) expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(true) await new Promise((resolve) => setTimeout(resolve, 100)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 6727b0144c03..1fec1ca9831a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -38,7 +38,8 @@ import { } from '../../runtime/agent-session-record-store-test-harness' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { JournalLifecycleBatchAppender } from '../agent-session-journal/journal-lifecycle-batch-appender' import * as recoveryResolution from './structured-agent-session-recovery-resolution' import { StructuredAgentSessionHost } from './structured-agent-session-host' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' @@ -57,6 +58,7 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const CALLER = { callerKey: 'client-1' } const DEAD_OWNER: AgentSessionProcessIdentity = { @@ -149,6 +151,23 @@ function openHost(overrides: Partial = {}): void }) } +/** Startup as the runtime runs it: the restart reconcile, the retry's visit to every chat in the + * background, and the boot sweep that opens each chat for its readers once the retry closed it. A + * lease latched in recovery is decided by the sweep, whose release wakes the retry again. */ +async function startUp(): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() + await reconciled() + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) + await host.restoreReadableSessions() + await reconciled() +} + +/** The host's retry has retired, or waits out a backoff. */ +function reconciled(): Promise { + return retryIdle(host.collaboratorsForTests().reconciliation, SESSION) +} + /** The host starting the agent with no message to deliver, as an operation that needs it does. */ function startAgent(): Promise { return host['serialize'](SESSION, () => host['mutationContext']().ensureAgent(SESSION)) @@ -338,7 +357,7 @@ describe('already-wedged profiles become usable on load', () => { : {}) }) - await host.restoreReadableSessions() + await startUp() expect(acquire).not.toHaveBeenCalled() expect(turnLifecycle('turn-1')).toEqual({ @@ -364,8 +383,12 @@ describe('already-wedged profiles become usable on load', () => { expect(published.map((summary) => summary.status)).not.toContain('working') // A crash is not something the user did: a proven one reads as an interruption and an // unprovable one as unconfirmed, so no reader files it as a cancellation the user knows about. - expect(published.map((summary) => summary.turnOutcome)).toEqual( - published.map(() => (verdict.state === 'interrupted' ? 'interruption' : 'unconfirmed')) + // The chat is published before its settlement lands, as not working with no outcome yet. + const outcome = verdict.state === 'interrupted' ? 'interruption' : 'unconfirmed' + const outcomes = published.map((summary) => summary.turnOutcome) + expect(outcomes.at(-1)).toBe(outcome) + expect(new Set(outcomes.filter((reported) => reported !== undefined))).toEqual( + new Set([outcome]) ) } ) @@ -383,6 +406,8 @@ describe('already-wedged profiles become usable on load', () => { const fence = stale.ok ? 13 : (stale.refusal.currentFence ?? 13) expect(stale.ok || stale.refusal.code === 'agent_session_checkpoint_stale').toBe(true) expect(stale.ok || (await host.attach(CALLER, hostTestAttachParams(fence))).ok).toBe(true) + // The reservation over the proven-dead owner ended its generation: the retry settles it. + await reconciled() expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) @@ -400,7 +425,7 @@ describe('already-wedged profiles become usable on load', () => { }) }) - it('settles an observed-exit latch through attach before the boot sweep', async () => { + it('an attach that beats the boot sweep clears the latch; the sweep then settles the turn interrupted', async () => { const record = wedgedRecord({ claimStatus: 'released', handoffStage: 'recovering' }) // The settlement latch an older build wrote; this build derives the settlement instead. const olderBuildLatch = { @@ -421,15 +446,18 @@ describe('already-wedged profiles become usable on load', () => { openHost() expect(await host.attach(CALLER, hostTestAttachParams(13))).toMatchObject({ ok: true }) + // Acquiring a released lease ends nothing (design §9.8): the boot sweep settles what it left. + host.collaboratorsForTests().reconciliation.signal(SESSION) + await reconciled() expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) - // The exit was observed, so its receipt is the turn's end. - expect(turnLifecycle('turn-1')).toEqual({ + // The reservation cleared the release's proof before the sweep ran; the runtime this one + // replaced held the turn's owner, which still ends it. + expect(turnLifecycle('turn-1')).toMatchObject({ turnId: 'turn-1', state: 'interrupted', startedAt: NOW - 5_000, - completedAt: NOW - 1_000, recovered: true }) expect(store.getRecord(SESSION)?.lease).toMatchObject({ @@ -445,7 +473,7 @@ describe('already-wedged profiles become usable on load', () => { ['a restart eviction', false], ['a proven eviction by recovery', true] ] as const)( - 'settles the turn %s left at the next acquire when the read restore could not write it', + 'settles the turn %s left once storage takes it, when its first settlement could not write it', async (_origin, ownerOutlivedRestart) => { await seedStore( wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }) @@ -462,9 +490,9 @@ describe('already-wedged profiles become usable on load', () => { : { outcome: 'pid-absent' }, stopOwnerProcess }) - // The read restore's settlement fails, and nothing retries it. + // The retry's settlement fails until storage takes it again; it backs off and retries. const failing = vi - .spyOn(AgentSessionJournal.prototype, 'appendLifecycleBatch') + .spyOn(JournalLifecycleBatchAppender.prototype, 'appendPlanned') .mockRejectedValue(new Error('journal unavailable')) await host.restoreReadableSessions() failing.mockRestore() @@ -476,11 +504,15 @@ describe('already-wedged profiles become usable on load', () => { deathEvidence: { kind: 'pid-absent' } }) + // The next start waits on none of it. await startAgent() expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) // A pid probe proved the owner gone, so the turn was cut short when it was last proven alive. + await vi.waitFor(() => expect(turnLifecycle('turn-1')?.state).toBe('interrupted'), { + timeout: 5_000 + }) expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', state: 'interrupted', @@ -552,13 +584,14 @@ describe('already-wedged profiles become usable on load', () => { } ) - it('interrupts a running turn left behind by a released lease on a cold acquire', async () => { + it('interrupts a running turn left behind by a released lease at startup, and a cold acquire starts', async () => { // No settlement latch: the record was released cleanly, but the journal still says a turn is // running. Nothing observed the child's exit, but the runtime that held it was replaced, and // nothing after its start proves it alive. await seedStore(wedgedRecord({ claimStatus: 'released', handoffStage: null })) await seedRunningTurn() openHost() + await startUp() expect(await host.attach(CALLER, hostTestAttachParams(13))).toMatchObject({ ok: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index aae177bbd650..329f4405597f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -52,6 +52,8 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { isUnansweredHandedOverSubmission } from '../agent-session-journal/journal-unsent-send-hold' import { structuredAgentSessionShownStatus } from './structured-agent-session-shown-work' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' /** A send Orca journaled that the provider has neither opened a turn for nor refused. Mirrors the * projection's own unanswered-dispatch rule, which is what makes that window read as `working`. */ @@ -71,13 +73,18 @@ function pendingSubmissionInFlight( return null } -/** The work identity to record: a running turn if one exists, else the send still awaiting one. */ +/** The work identity to record: a running turn if one exists, else the send still awaiting one. + * `scope`: the host's (`structuredAgentSessionCurrentWork`), so an ended generation's turn is none. */ export function structuredAgentSessionWorkInFlight( items: readonly AgentJournalRenderItem[], - submissions: readonly AgentJournalSubmission[] + submissions: readonly AgentJournalSubmission[], + scope?: StructuredAgentSessionWorkScope ): AgentSessionResumeWork | null { const current = agentSessionCurrentContextRows(items, submissions) - const turnId = activeStructuredAgentSessionTurnId(current.items) + const turnId = activeStructuredAgentSessionTurnId( + current.items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ) if (turnId) { return { kind: 'turn', id: turnId } } @@ -91,7 +98,8 @@ export function structuredAgentSessionWorkInFlight( */ function structuredAgentSessionResumeWork( items: readonly AgentJournalRenderItem[], - submissions: readonly AgentJournalSubmission[] + submissions: readonly AgentJournalSubmission[], + scope: StructuredAgentSessionWorkScope ): AgentSessionResumeWork | null { const bodies = new Map(items.map((item) => [item.itemId, item.body])) const bodyOf = (itemId: string) => bodies.get(itemId) @@ -107,7 +115,7 @@ function structuredAgentSessionResumeWork( ) { return null } - const inFlight = structuredAgentSessionWorkInFlight(items, submissions) + const inFlight = structuredAgentSessionWorkInFlight(items, submissions, scope) if (inFlight) { return inFlight } @@ -129,13 +137,17 @@ function boundedLabel(text: string | undefined): string { /** The prompts the session is blocked on, from the same items the projection called `attention` * over — one read, so the recorded state and the prompts it stands for cannot disagree. */ -function pendingPrompts(items: readonly AgentJournalRenderItem[]): AgentSessionRestartPrompt[] { +function pendingPrompts( + items: readonly AgentJournalRenderItem[], + scope: StructuredAgentSessionWorkScope +): AgentSessionRestartPrompt[] { const prompts: AgentSessionRestartPrompt[] = [] for (const item of items) { const body = item.body if ( (body.kind !== 'approval' && body.kind !== 'question') || - body.resolution.state !== 'pending' + body.resolution.state !== 'pending' || + !scope.isCurrentItem(item.itemId) ) { continue } @@ -197,16 +209,19 @@ export function structuredAgentSessionWorkingAtStop(input: { !(unanswered && isUnansweredHandedOverSubmission(submission)) ) const childWork = input.childWork(sessionId) + const record = input.getRecord(sessionId) + // The live generation is this host's own child, read first-hand like the rest, never the lease. + const { scope } = new StructuredAgentSessionCurrentWork(session.journal, session.child.fence) const status = structuredAgentSessionShownStatus( { items: current.items, submissions: handedOver }, childWork, - session.child.fence + session.child.fence, + scope ) if (status.state === 'done') { return null } - const work = structuredAgentSessionResumeWork(current.items, handedOver) - const record = input.getRecord(sessionId) + const work = structuredAgentSessionResumeWork(current.items, handedOver, scope) const head = record ? agentSessionProviderHandleChainHead(record.providerHandleChain) : null if (!work || !head) { return null @@ -215,7 +230,7 @@ export function structuredAgentSessionWorkingAtStop(input: { // The lead's OWN state, not the fold: a settled lead with running children reads `done` here // and carries them in `tasks`, which is how the dialog tells the two apart. state: status.mainAgent.state, - prompts: pendingPrompts(current.items), + prompts: pendingPrompts(current.items, scope), tasks: liveTasks(childWork) } return { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts new file mode 100644 index 000000000000..8cbc919e6faf --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts @@ -0,0 +1,68 @@ +// Another connection's write lock, held from its own thread, and a watch on the main thread while +// this one meets it: for tests of bookkeeping that must fail at once instead of waiting it out. + +import { Worker } from 'node:worker_threads' +import { journalDatabasePath } from '../agent-session-journal/journal-host-database' + +const lockers = new Set() + +/** Another connection, on its own thread, holds the write lock for `ms`: this thread's waits + * cannot release it, as another process's could not. Resolves once the lock is held. */ +export function holdWriteLock( + stateDirectory: string, + ms: number +): Promise<{ released: Promise }> { + const worker = new Worker( + `const { workerData, parentPort } = require('node:worker_threads') + const { DatabaseSync } = require('node:sqlite') + const db = new DatabaseSync(workerData.file) + db.exec('BEGIN IMMEDIATE') + parentPort.postMessage('locked') + setTimeout(() => { db.exec('ROLLBACK'); db.close(); parentPort.postMessage('released') }, workerData.ms)`, + { eval: true, workerData: { file: journalDatabasePath(stateDirectory), ms } } + ) + lockers.add(worker) + let release: () => void = () => undefined + const released = new Promise((resolve) => (release = resolve)) + return new Promise((resolve) => + worker.on('message', (message) => (message === 'locked' ? resolve({ released }) : release())) + ) +} + +/** Ends every lock a test still holds. */ +export async function releaseWriteLocks(): Promise { + await Promise.all([...lockers].map((worker) => worker.terminate())) + lockers.clear() +} + +/** Each gap between the main thread's timer turns while `run` ran: how long, and when it ended. */ +async function timerGaps(run: () => Promise): Promise<{ ms: number; endedAt: number }[]> { + let last = performance.now() + const gaps: { ms: number; endedAt: number }[] = [] + const gap = (): void => { + const now = performance.now() + gaps.push({ ms: now - last, endedAt: now }) + last = now + } + const probe = setInterval(gap, 2) + try { + await run() + gap() + return gaps + } finally { + clearInterval(probe) + } +} + +/** The longest the main thread went without running a timer, while `run` ran. */ +export async function longestStall(run: () => Promise): Promise { + return Math.max(0, ...(await timerGaps(run)).map((gap) => gap.ms)) +} + +/** Every time the main thread went longer than `ms` without running a timer, while `run` ran. */ +export async function stallsOver( + ms: number, + run: () => Promise +): Promise<{ ms: number; endedAt: number }[]> { + return (await timerGaps(run)).filter((gap) => gap.ms > ms) +} diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts index 50ddcbc08e9e..9a0d655c771a 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts @@ -27,7 +27,14 @@ function contextWith( return { sessionId: 'session-1', fence: 1, - journal: { snapshot: () => ({ items: [] }), submissions: () => [] }, + journal: { + snapshot: () => ({ items: [] }), + submissions: () => [], + runningTurn: () => null, + itemFence: () => undefined, + visitItems: () => undefined, + wroteBeforeOpen: () => false + }, adapter } } @@ -59,6 +66,8 @@ const SETTLED = child({ settledAt: 300 }) const RECORD = { lease: {} } as unknown as AgentSessionRecord +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: admission reads only the running turn's id and item id. +const RUNNING_TURN = { turnId: 'turn-1', item: { itemId: 'turn-1' } } as never const STOP = 'Stop background tasks before using this command.' const WAIT = 'Wait for background tasks to finish before using this command.' @@ -120,6 +129,7 @@ describe('conversationCommandBlocked background work', () => { } ] }) as unknown as ReturnType + ctx.journal.runningTurn = () => RUNNING_TURN expect(conversationCommandBlocked(ctx, RECORD, [child()])?.message).toBe( 'Wait for the current turn to finish before using this command.' ) @@ -135,6 +145,7 @@ describe('conversationCommandBlocked for a command sent at rest (C6, B3)', () => const ctx = contextWith(undefined) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the admission reads only each item's body. ctx.journal.snapshot = () => staleTurn as never + ctx.journal.runningTurn = () => RUNNING_TURN expect(conversationCommandBlocked(ctx, RECORD, [], 'at-rest')).toBeNull() expect(conversationCommandBlocked(ctx, RECORD, [])).toMatchObject({ details: { reason: 'turnActive' }, diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 7d81e4a77180..b40265c16b4b 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -6,10 +6,13 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' -import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection' import { agentSessionCurrentContextRows } from '../../../shared/agent-session-context-clear' -import { isUnansweredStructuredAgentSessionDispatch } from '../../../shared/structured-agent-session-unanswered-dispatch' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkJournal +} from './structured-agent-session-current-work' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { refuse, @@ -21,10 +24,11 @@ import { export type ConversationCommandAdmissionContext = { sessionId: string fence: number - journal: { + journal: StructuredAgentSessionCurrentWorkJournal & { snapshot(): Pick, 'items'> - submissions: AgentSessionJournal['submissions'] } + /** The host's projection of current work; without one, the generation at `fence` is live. */ + currentWork?: () => StructuredAgentSessionCurrentWork | null adapter: Pick } @@ -64,14 +68,18 @@ export function conversationCommandBlocked( if (record.lease.handoffStage || record.lease.handoffOperationId) { return blocked('handoffInFlight', 'Wait for the session handoff to finish.') } - if (admission !== 'at-rest' && activeStructuredAgentSessionTurnId(items)) { + // What is current is the host projection's to say: an ended generation's turn, prompt or send + // refuses nothing. + const work = contextStructuredAgentSessionCurrentWork(ctx) + if (admission !== 'at-rest' && work.activeTurnId()) { return blocked('turnActive', 'Wait for the current turn to finish before using this command.') } if ( items.some( (item) => (item.body.kind === 'approval' || item.body.kind === 'question') && - item.body.resolution.state === 'pending' + item.body.resolution.state === 'pending' && + work.isCurrentItem(item.itemId) ) ) { return blocked( @@ -80,7 +88,9 @@ export function conversationCommandBlocked( ) } // The same liveness fold the strip's monitoring indicator reads: settled rows block nothing. - if (agentChildWorkLiveness(childWork) !== null) { + // Child records are the live generation's: its adapter ends them on every close, so with no + // generation live, any still running are an ended one's leftovers and hold nothing. + if (work.liveFence !== null && agentChildWorkLiveness(childWork) !== null) { return blocked( 'backgroundTasksRunning', stripOffersStop(childWork ?? [], ctx.adapter.backgroundTaskStops?.(ctx.sessionId)) @@ -93,8 +103,7 @@ export function conversationCommandBlocked( if ( submissions.some( (entry) => - !(admission === 'handover' && isQueuedAgentJournalSubmission(entry)) && - isUnansweredStructuredAgentSessionDispatch(entry, ctx.fence) + !(admission === 'handover' && isQueuedAgentJournalSubmission(entry)) && work.owesSend(entry) ) ) { return blocked( diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index ab6df634d13f..606d2093c93b 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -11,6 +11,7 @@ import { agentSessionOperationKey } from '../../../shared/agent-session-operatio import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' import { sendPreparation } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { conversationCommandBlocked } from './structured-conversation-command-admission' import type { AgentSessionFailureFact } from '../../../shared/agent-session-failure' import { @@ -69,6 +70,8 @@ export function runStructuredConversationCommand( // Starts the agent only to settle a rewind in doubt, as a send does; a /clear itself starts nothing. prepareSession: sendPreparation(context, envelope, { refusesInRun: true }), journal: () => context.sessions.get(sessionId)?.journal, + currentWork: () => + hostStructuredAgentSessionCurrentWork({ store, sessions: context.sessions }, sessionId), publish: (journal) => context.publish(sessionId, journal), now: context.now, plan: { diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts index 35e5b4fda216..cbd1197d511e 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts @@ -248,15 +248,16 @@ it('hands over a message held behind the command when the command ends just as t await state.host.conversationCommand(CALLER, compactParams()) await vi.waitFor(() => expect(compact).toHaveBeenCalledOnce(), { interval: 1 }) const { journal: live } = state.host['sessions'].get(SESSION)! - const activeTurnId = live.activeTurnId + const runningTurn = live.runningTurn let ended = false - vi.spyOn(live, 'activeTurnId').mockImplementation(() => { - const read = activeTurnId() + // The loop reads the command through the host's current-work projection, which asks this. + vi.spyOn(live, 'runningTurn').mockImplementation(() => { + const read = runningTurn() // The provider's end lands the moment the loop's own step reads the command as running and // stops; no other reader's view of it matters here. if ( !ended && - read?.startsWith('compact:') && + read?.turnId.startsWith('compact:') && /at (?:StructuredAgentSessionDeliveryLoop\.)?prepare \(.*structured-agent-session-delivery-loop/.test( new Error('who reads').stack ?? '' ) diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts index b0320ffdcbcf..25bbf9905678 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts @@ -16,11 +16,21 @@ import { rewindRefusal } from './structured-rewind-refusal' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import type { + BackgroundJournalWrites, + BackgroundStoreWrites +} from './structured-agent-session-background-writes' -type RewindRecoveryDeps = { store: AgentSessionRecordStore; logger: StructuredAgentSessionLogger } +type RewindRecoveryDeps = { + store: AgentSessionRecordStore + logger: StructuredAgentSessionLogger + /** The retry's own recovery writes through its handles: another connection's lock throws. */ + background?: { store: BackgroundStoreWrites; journal: BackgroundJournalWrites } +} export function persistRewindRecord( - store: AgentSessionRecordStore, + store: Pick, sessionId: string, fence: number, rewind: AgentSessionRewindRecord @@ -38,12 +48,13 @@ export function persistRewindRecord( * settled refused rather than proven. Bookkeeping only: the chat is already attached either way. */ async function settleUnsupportedClaudeRewind( - { store, logger }: RewindRecoveryDeps, + { store: reads, logger, background }: RewindRecoveryDeps, sessionId: string, fence: number, rewind: AgentSessionRewindRecord ): Promise { const refusal = rewindRefusal('unsupported').refusal + const store = background?.store ?? reads try { await persistRewindRecord(store, sessionId, fence, { ...rewind, @@ -57,6 +68,9 @@ async function settleUnsupportedClaudeRewind( outcome: { status: 'failed', code: refusal.code, rewindReason: 'unsupported' } }) } catch (error) { + if (background && isSqliteContentionFailure(error)) { + throw error + } logger.warn('a pending Claude rewind was not settled', { scope: 'rewind-unsupported-settlement', sessionId, @@ -66,6 +80,17 @@ async function settleUnsupportedClaudeRewind( } } +/** A rewind in doubt that only its provider's history can settle: an unverified Codex revert. */ +export function structuredRewindNeedsProvider( + rewind: AgentSessionRewindRecord | undefined +): boolean { + if (rewind?.phase !== 'provider-succeeded' && rewind?.phase !== 'prepared') { + return false + } + const target = parseAgentJournalItemKey(rewind.providerItemId ?? rewind.itemId) + return target?.provider === 'codex' && !rewind.hydrationVerified +} + /** Recovery observes provider state; it never repeats an ambiguous native mutation. */ export async function recoverStructuredRewind( deps: RewindRecoveryDeps, @@ -80,8 +105,10 @@ export async function recoverStructuredRewind( cursor: AgentJournalCursor ) => JournalOperationReceipt ): Promise { - const { store } = deps - let rewind = store.getRecord(sessionId)?.rewind + const { store: reads } = deps + const store = deps.background?.store ?? reads + const writes = deps.background?.journal ?? journal + let rewind = reads.getRecord(sessionId)?.rewind if (rewind?.phase !== 'provider-succeeded' && rewind?.phase !== 'prepared') { return } @@ -156,7 +183,7 @@ export async function recoverStructuredRewind( const replacement = rewind.retained.map(retainedRowReplacement) if (rewind.contextClearOperationId && rewind.contextClearSequence) { if ( - store.getRecord(sessionId)?.providerContextBoundary?.operationId !== + reads.getRecord(sessionId)?.providerContextBoundary?.operationId !== rewind.contextClearOperationId || journal.cursor().epoch !== rewind.expectedEpoch || journal.context.floor()?.sequence !== rewind.contextClearSequence @@ -164,13 +191,13 @@ export async function recoverStructuredRewind( throw new Error('agent_session_rewind:stale-context') } const completed = rewind - await journal.context.rewind( + await writes.context.rewind( { epoch: completed.expectedEpoch, sequence: completed.contextClearSequence! }, fence, replacement, (cursor) => receipt?.(completed, fence, cursor) ?? - store.conversationReceipts.rewind(sessionId, fence, completed, cursor) + reads.conversationReceipts.rewind(sessionId, fence, completed, cursor) ) return } @@ -190,7 +217,7 @@ export async function recoverStructuredRewind( } const cursor = alreadyReplaced ? journal.cursor() - : await journal.replaceEpochItems('handle_forked', fence, replacement) + : await writes.replaceEpochItems('handle_forked', fence, replacement) await persistRewindRecord(store, sessionId, fence, { ...rewind, phase: 'completed', diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts b/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts index f41e83522d2c..5fc0f7f184a7 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts @@ -5,7 +5,10 @@ // each keeps the ones its retained row held. import { parseCodexGoalJournalItemId } from '../../codex/codex-goal-journal-identity' -import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + agentJournalItemKey, + parseAgentJournalItemKey +} from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_ITEM_BODY_KINDS, isAdmissibleAgentJournalMessageBody @@ -64,12 +67,28 @@ export function mergeRetainedHostLifecycleRows( ) } +/** The live agent's rewound history as retained rows: its own observation, at the fence it + * answered at. A row the old epoch held keeps that row's provenance when merged. */ +export function rewindProviderRows( + items: readonly { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[], + observedAt: number, + ownerFence: number +): RetainedRow[] { + return items.map(({ identity, body }) => ({ + itemId: agentJournalItemKey(identity), + body, + observedAt, + ownerFence + })) +} + /** The rebuilt epoch's item for one retained row, with the scope and producer it was written with. */ export function retainedRowReplacement(row: RetainedRow): AgentJournalProducerLinkage & { identity: AgentJournalItemIdentity body: AgentJournalItemBody observedAt: number turnScope?: AgentJournalTurnScope + ownerFence: number } { const identity = parseAgentJournalItemKey(row.itemId) if (!identity) { @@ -87,6 +106,9 @@ export function retainedRowReplacement(row: RetainedRow): AgentJournalProducerLi body: restoreRewindJournalBody(row.body), observedAt: row.observedAt, ...(scope ? { turnScope: scope } : {}), + // The old fold's provenance, so a rewind never makes a dead generation's work current. A record + // from before rows carried it states none: 0, which is never current. + ownerFence: row.ownerFence ?? 0, ...(agentId === undefined ? {} : { agentId }), ...(parentAgentId === undefined ? {} : { parentAgentId }), ...(providerParentRef === undefined ? {} : { providerParentRef }), diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts index 73d01b5b88e8..a025d5d1621c 100644 --- a/src/main/runtime/agent-session-death-evidence-persistence.test.ts +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -82,14 +82,14 @@ describe('death evidence on disk', () => { }) }) -describe('who is told a proof of death landed', () => { - it('tells a listener once the proof is committed, and never for a write that proves nothing', async () => { +describe('who is told a generation ended', () => { + it('tells a listener once the end is committed, with its proof, and never for a write that ends nothing', async () => { await seed(null) const store = await open() const told: unknown[] = [] - store.onDeathEvidence((sessionId) => told.push(store.getRecord(sessionId)?.lease.deathEvidence)) + store.onGenerationEnded((ended) => told.push(ended)) const unsubscribed = vi.fn() - store.onDeathEvidence(unsubscribed)() + store.onGenerationEnded(unsubscribed)() await expect( store.evictProvenDeadOwner({ @@ -103,7 +103,13 @@ describe('who is told a proof of death landed', () => { // The proof already on the record is not news to a later write. await store.setSessionTabVisibility(SESSION, true) - expect(told).toEqual([expect.objectContaining({ kind: 'pid-absent', ownerFence: 7 })]) + expect(told).toEqual([ + { + sessionId: SESSION, + endedFence: 7, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: 7 }) + } + ]) expect(unsubscribed).not.toHaveBeenCalled() }) }) diff --git a/src/main/runtime/agent-session-generation-end.test.ts b/src/main/runtime/agent-session-generation-end.test.ts new file mode 100644 index 000000000000..17dffef026f4 --- /dev/null +++ b/src/main/runtime/agent-session-generation-end.test.ts @@ -0,0 +1,268 @@ +// Every lease write that revokes or supersedes a generation is told once, after commit, from the +// store's one transaction path, with the proof the end carried; a write that ends nothing is not. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { claudeProviderHandle } from '../../shared/agent-session-provider-handle-encoding' +import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' +import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { releaseStoredStructuredAgentSessionOwnerAfterExit } from '../native-chat/agent-session-wire/structured-agent-session-lease-release' +import type { AgentSessionFailedAcquisitionSettlement } from './agent-session-acquisition-failure-settlement' +import type { AgentSessionGenerationEnd } from './agent-session-generation-end' +import { releaseUnprovenAgentSessionOwner } from './agent-session-lease-transitions' +import type { AgentSessionRecordStore } from './agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from './agent-session-record-store-test-harness' + +const SESSION = 'session-alpha-1' +const NOW = 1_800_000_000_000 +let directory: string +let store: AgentSessionRecordStore +let ended: AgentSessionGenerationEnd[] +let operations = 0 + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-generation-end-')) + await seedTestAgentSessionRecordStore(directory, { records: [] }) + store = await openTestAgentSessionRecordStore(directory) + ended = [] + store.onGenerationEnded((end) => ended.push(end)) +}) + +afterEach(async () => { + closeTestJournalHostDatabases() + await rm(directory, { recursive: true, force: true }) +}) + +function operationId(): string { + operations += 1 + return `${NOW}-${operations.toString(16).padStart(32, '0')}` +} + +/** Reserves at the next fence, as an attach does, over what the probe says of the recorded owner. */ +async function reserve( + probe: Parameters[0]['probe'] +): Promise<{ fence: number; operation: string }> { + const operation = operationId() + const record = store.getRecord(SESSION) + const reserved = await store.reserveOwner({ + sessionId: SESSION, + location: agentSessionRecordFixture().location, + provider: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + expectedFence: record ? record.lease.runtimeFence : null, + spawnToken: `spawn-${operation}`, + claimKeyId: 'key-1', + handoffOperationId: operation, + probe, + operation: { callerKey: 'client-1', operationId: operation, fingerprint: 'fp-1' }, + now: NOW + }) + return { fence: reserved.record.lease.runtimeFence, operation } +} + +/** A reservation whose child spawned and proved its handle: a live owner. */ +async function liveOwner(): Promise<{ fence: number; operation: string }> { + const reserved = await reserve({ outcome: 'indeterminate', reason: 'no owner yet' }) + await store.commitProcessIdentity({ + sessionId: SESSION, + fence: reserved.fence, + process: { + hostId: 'local', + pid: 4_000 + reserved.fence, + processStartTimeMs: NOW, + spawnToken: `spawn-${reserved.operation}` + }, + now: NOW + }) + await store.proveOwner({ + sessionId: SESSION, + fence: reserved.fence, + link: { + linkId: `link-${reserved.fence}`, + handle: claudeProviderHandle('provider-session-1', null), + // The first start creates the conversation; each later one resumes it. + origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', + mintedAtFence: reserved.fence, + observedAt: NOW + }, + now: NOW + }) + return reserved +} + +function failedAcquisition( + reserved: { fence: number; operation: string }, + exitProof: AgentSessionFailedAcquisitionSettlement['exitProof'] +): AgentSessionFailedAcquisitionSettlement { + return { + sessionId: SESSION, + fence: reserved.fence, + spawnToken: `spawn-${reserved.operation}`, + callerKey: 'client-1', + operationId: reserved.operation, + outcome: { status: 'failed', code: 'agent_session_operation_invalid', message: 'failed' }, + exitProof, + now: NOW + 1_000 + } +} + +describe('a generation that ends', () => { + it('is told once for an observed exit’s release, with the exit’s proof', async () => { + const { fence } = await liveOwner() + ended = [] + + await releaseStoredStructuredAgentSessionOwnerAfterExit({ + store, + writes: store, + sessionId: SESSION, + expectedFence: fence, + now: NOW + 1_000, + exitObservedAt: NOW + 900 + }) + + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: fence, + evidence: expect.objectContaining({ kind: 'exit-observed', ownerFence: fence }) + } + ]) + }) + + it('is told once for each failed acquisition’s release', async () => { + const first = await reserve({ outcome: 'indeterminate', reason: 'no owner yet' }) + ended = [] + await store.settleFailedAcquisition(failedAcquisition(first, 'processless')) + expect(ended).toEqual([ + { sessionId: SESSION, endedFence: first.fence, evidence: expect.anything() } + ]) + + const second = await liveOwner() + ended = [] + await store.settleFailedPostAcquisitionAttachment( + failedAcquisition(second, 'root-exit-observed') + ) + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: second.fence, + evidence: expect.objectContaining({ ownerFence: second.fence }) + } + ]) + }) + + it('is told once for an eviction, and once for a release nothing proved', async () => { + const first = await liveOwner() + await store.settleFailedPostAcquisitionAttachment(failedAcquisition(first, 'unproven')) + // Parked in recovery: nothing has ended yet. + expect(ended).toEqual([]) + await store.evictProvenDeadOwner({ + sessionId: SESSION, + expectedFence: first.fence, + probe: { outcome: 'pid-absent' }, + now: NOW + 2_000 + }) + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: first.fence, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: first.fence }) + } + ]) + + const second = await liveOwner() + await store.settleFailedPostAcquisitionAttachment(failedAcquisition(second, 'unproven')) + ended = [] + await store.transitionHandoff(SESSION, (record) => + releaseUnprovenAgentSessionOwner({ record, expectedFence: second.fence, now: NOW + 3_000 }) + ) + expect(ended).toEqual([{ sessionId: SESSION, endedFence: second.fence, evidence: null }]) + }) + + it('is told once for a reservation granted over an owner the probe proved gone, with the probe’s proof the reservation clears', async () => { + const { fence } = await liveOwner() + ended = [] + + const replaced = await reserve({ outcome: 'pid-absent' }) + + expect(replaced.fence).toBe(fence + 1) + expect(store.getRecord(SESSION)?.lease.deathEvidence).toBeNull() + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: fence, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: fence }) + } + ]) + }) + + it('is told once for a restart adjudication that releases it', async () => { + const { fence } = await liveOwner() + closeTestJournalHostDatabases() + store = await openTestAgentSessionRecordStore(directory) + ended = [] + store.onGenerationEnded((end) => ended.push(end)) + + await store.reconcileOnRestart({ probe: async () => ({ outcome: 'pid-absent' }), now: NOW }) + + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: fence, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: fence }) + } + ]) + }) +}) + +describe('a write that ends nothing', () => { + it('tells no one: a reservation over a released lease with no proof, a spawn, a proof, a renewal', async () => { + await store.reconcileOnRestart({ probe: async () => ({ outcome: 'pid-absent' }), now: NOW }) + const first = await reserve({ outcome: 'indeterminate', reason: 'no owner yet' }) + await store.settleFailedAcquisition(failedAcquisition(first, 'processless')) + const released = store.getRecord(SESSION)!.lease + // Strip the release's proof, as one an earlier build wrote without any. + await store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { ...record.lease, deathEvidence: null } + })) + expect(released.claimStatus).toBe('released') + ended = [] + + const { fence } = await liveOwner() + await store.renewLease({ + sessionId: SESSION, + fence, + childProbe: { outcome: 'identity-matched', matchedOn: ['held-child'] }, + now: NOW + 5_000 + }) + await store.setSessionTabVisibility(SESSION, true) + + expect(ended).toEqual([]) + }) + + // Design §9.8: that generation was told at its release; the reservation's clearing of the + // release's proof is no second end. + it('tells no one for a reservation over a lease an observed exit released with its proof', async () => { + const first = await liveOwner() + await releaseStoredStructuredAgentSessionOwnerAfterExit({ + store, + writes: store, + sessionId: SESSION, + expectedFence: first.fence, + now: NOW + 1_000, + exitObservedAt: NOW + 900 + }) + ended = [] + + await liveOwner() + + expect(store.getRecord(SESSION)?.lease.deathEvidence).toBeNull() + expect(ended).toEqual([]) + }) +}) diff --git a/src/main/runtime/agent-session-generation-end.ts b/src/main/runtime/agent-session-generation-end.ts new file mode 100644 index 000000000000..e7468e9420ed --- /dev/null +++ b/src/main/runtime/agent-session-generation-end.ts @@ -0,0 +1,77 @@ +// Which agent generations a committed store transaction ended: every lease write that revokes or +// supersedes a generation that could still act. A release of any kind (exit, surface, failed +// acquisition, eviction, unproven, restart adjudication) moves the fence and leaves no owner; a +// reservation granted over an owner a probe proved gone supersedes it directly. Reserving a lease +// already released ends nothing new: that generation was told at its release. + +import { + agentSessionDeathEvidenceFromProbe, + type AgentSessionOwnerProbe +} from '../../shared/agent-session-lease-adjudication' +import type { + AgentSessionDeathEvidence, + AgentSessionLease, + AgentSessionRecord +} from '../../shared/agent-session-record' + +export type AgentSessionGenerationEnd = { + sessionId: string + /** The generation that can no longer act. */ + endedFence: number + /** The proof of how it ended: the release's own, or, for a reservation that replaced it, the + * probe's, which the reservation clears from the lease. Null when nothing proved it gone. */ + evidence: AgentSessionDeathEvidence | null +} + +/** A transaction's own proof for a generation its reservation superseded. */ +export type AgentSessionSupersessionEvidence = ( + ended: AgentSessionLease +) => AgentSessionDeathEvidence | null + +/** What a store transaction may say of the generations it ends. */ +export type AgentSessionGenerationEndOptions = { + supersessionEvidence?: AgentSessionSupersessionEvidence +} + +/** A reservation granted over an owner the probe proved gone ends it with that probe's proof, which + * the reservation clears from the lease. */ +export function reservationSupersessionEvidence( + probe: AgentSessionOwnerProbe, + now: number +): AgentSessionSupersessionEvidence { + return (ended) => agentSessionDeathEvidenceFromProbe(probe, now, ended) +} + +export function agentSessionLeasesBefore( + records: ReadonlyMap +): Map { + return new Map([...records].map(([sessionId, record]) => [sessionId, record.lease])) +} + +export function agentSessionGenerationEnds( + before: ReadonlyMap, + after: ReadonlyMap, + supersessionEvidence?: AgentSessionSupersessionEvidence +): AgentSessionGenerationEnd[] { + const ends: AgentSessionGenerationEnd[] = [] + for (const [sessionId, { lease }] of after) { + const prior = before.get(sessionId) + if (!prior || prior === lease) { + continue + } + // A released lease's generation ended at its release; acquiring it again ends nothing. + if (prior.claimStatus === 'released') { + continue + } + const released = lease.claimStatus === 'released' + if (!released && lease.runtimeFence === prior.runtimeFence) { + continue + } + ends.push({ + sessionId, + endedFence: prior.runtimeFence, + evidence: released ? lease.deathEvidence : (supersessionEvidence?.(prior) ?? null) + }) + } + return ends +} diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 5601d2a2e00c..af852adfbcf6 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -1,6 +1,7 @@ /** Durable single-writer session records and their operation ledger, as rows in the host's chat * journal database. */ +import type { JournalWriteOptions } from '../native-chat/agent-session-journal/journal-database' import { agentSessionRefusalError } from '../../shared/agent-session-wire-refusals' import { createAgentSessionConversationReceipts } from './agent-session-conversation-receipts' import { commitConversationCommandRecord } from './agent-session-conversation-command-record' @@ -27,6 +28,13 @@ import { retireAgentSessionClaimKey } from './agent-session-claim-key-retention' import type { AgentSessionOwnerProbe } from '../../shared/agent-session-lease-adjudication' +import { + agentSessionGenerationEnds, + agentSessionLeasesBefore, + reservationSupersessionEvidence, + type AgentSessionGenerationEnd, + type AgentSessionGenerationEndOptions +} from './agent-session-generation-end' import type { AgentSessionProviderHandleLink } from '../../shared/agent-session-provider-handle' import { agentSessionScopeKey, @@ -69,7 +77,6 @@ import { showAgentSessionTabs } from './agent-session-tab-table' import type { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' -import type { JournalOperationReceipt } from '../native-chat/agent-session-journal/journal-row-writer' import { agentSessionOperationOutcomeReceipt } from './agent-session-operation-receipt' import { loadAgentSessionStoreRows } from './agent-session-record-rows' import { AgentSessionStoreTransactions } from './agent-session-store-transactions' @@ -84,7 +91,7 @@ export const AGENT_SESSION_LEASE_TTL_MS = 30_000, AGENT_SESSION_LEASE_RENEW_INTERVAL_MS = 10_000 export class AgentSessionRecordStore { - private readonly deathEvidenceListeners = new Set<(sessionId: string) => void>() + private readonly generationEndListeners = new Set<(ended: AgentSessionGenerationEnd) => void>() private readonly firstRecordListeners = new Set<() => void>() readonly conversationReceipts: ReturnType @@ -148,8 +155,8 @@ export class AgentSessionRecordStore { /** Shows each session that still has a record, in one write: an index written part way would * read as complete at the next launch and drop the rest. */ - showSessionTabs(sessionIds: readonly string[]): Promise { - return this.transact((draft) => showAgentSessionTabs(draft, sessionIds)) + showSessionTabs(sessionIds: readonly string[], options?: JournalWriteOptions): Promise { + return this.transact((draft) => showAgentSessionTabs(draft, sessionIds), options) } listByScope(location: AgentSessionExecutionLocation): AgentSessionRecord[] { @@ -195,8 +202,9 @@ export class AgentSessionRecordStore { isAgentSessionClaimKeyVerifiable(this.state, keyId, now) reserveOwner = (request: AgentSessionReserveRequest): Promise => - this.transact((draft) => - commitAgentSessionReservation(draft, request, AGENT_SESSION_LEASE_TTL_MS) + this.transact( + (draft) => commitAgentSessionReservation(draft, request, AGENT_SESSION_LEASE_TTL_MS), + { supersessionEvidence: reservationSupersessionEvidence(request.probe, request.now) } ) readonly founding: ReturnType @@ -252,12 +260,11 @@ export class AgentSessionRecordStore { return this.mutate(args.sessionId, (record) => evictAgentSessionOwner({ ...args, record })) } - async transitionHandoff( + transitionHandoff = ( sessionId: string, - transition: (record: AgentSessionRecord) => AgentSessionRecord - ): Promise { - return this.mutate(sessionId, transition) - } + transition: (record: AgentSessionRecord) => AgentSessionRecord, + options?: JournalWriteOptions + ): Promise => this.mutate(sessionId, transition, options) /** * Adjudicate every lease this host loaded. No lease grants a writer until it appears here. On a @@ -265,13 +272,13 @@ export class AgentSessionRecordStore { * start, and none grants a writer there, since every grant is a write. */ async reconcileOnRestart( - args: AgentSessionRestartProbeArgs + args: AgentSessionRestartProbeArgs, + options?: JournalWriteOptions ): Promise> { const pending = this.listRecords().filter((record) => record.lease.unreconciled) const probes = await collectAgentSessionRestartProbes(pending, args) - return this.transact((draft) => applyAgentSessionRestartProbes(draft, probes, args.now), { - inMemoryWhenReadOnly: true - }) + const write = { inMemoryWhenReadOnly: true, ...options } + return this.transact((draft) => applyAgentSessionRestartProbes(draft, probes, args.now), write) } /** Admits one non-reservation mutation through the durable ledger. */ @@ -304,16 +311,17 @@ export class AgentSessionRecordStore { ) => this.transact((draft) => admitAndClaimAgentSessionOperationInto(draft, args, claimAfter)) recordOperationOutcome = ( - args: Parameters[1] - ): Promise => this.transact((draft) => settleAgentSessionOperationInto(draft, args)) + args: Parameters[1], + options?: JournalWriteOptions + ) => this.transact((draft) => settleAgentSessionOperationInto(draft, args), options) /** That settlement, or an accepted row inserted if absent, committed by the journal write that * makes it true. It changes only the ledger, so no record listener is owed. */ operationOutcomeReceipt = (args: Parameters[1]) => agentSessionOperationOutcomeReceipt(this.transactions, args) - commitOperationReceipt = (receipt: JournalOperationReceipt) => - this.transactions.commitReceipt(receipt) + commitOperationReceipt = (...args: Parameters) => + this.transactions.commitReceipt(...args) readCommandReceipt = (...args: Parameters) => this.transactions.readCommandReceipt(...args) @@ -327,7 +335,8 @@ export class AgentSessionRecordStore { private async mutate( sessionId: string, - apply: (record: AgentSessionRecord) => AgentSessionRecord + apply: (record: AgentSessionRecord) => AgentSessionRecord, + options?: JournalWriteOptions ): Promise { return this.transact((draft) => { const record = draft.records.get(sessionId) @@ -339,14 +348,15 @@ export class AgentSessionRecordStore { const next = apply(record) draft.records.set(sessionId, next) return next - }) + }, options) } - /** Told, once committed, of each session a transaction wrote a proof of death for — whichever - * transition wrote it, since every one lands here. Must not throw. */ - onDeathEvidence(listener: (sessionId: string) => void): () => void { - this.deathEvidenceListeners.add(listener) - return () => this.deathEvidenceListeners.delete(listener) + /** Told once, after commit, of each generation a transaction revoked or superseded, whichever + * transition wrote it, since every lease write lands here (`agentSessionGenerationEnds`). Must not + * throw. */ + onGenerationEnded(listener: (ended: AgentSessionGenerationEnd) => void): () => void { + this.generationEndListeners.add(listener) + return () => this.generationEndListeners.delete(listener) } /** Told, once committed, when the store records its first chat. Must not throw. */ @@ -359,26 +369,20 @@ export class AgentSessionRecordStore { * once its rows have committed. */ private transact = async ( apply: (draft: AgentSessionStoreState) => T, - options?: { inMemoryWhenReadOnly?: boolean } + options?: { inMemoryWhenReadOnly?: boolean } & AgentSessionGenerationEndOptions & + JournalWriteOptions ): Promise => { - let proven: string[] = [] + let ended: AgentSessionGenerationEnd[] = [] let heldBefore = true const result = await this.transactions.transact((draft) => { heldBefore = this.holdsRecords() - if (this.deathEvidenceListeners.size === 0) { - return apply(draft) - } - const before = new Map( - [...draft.records].map(([id, record]) => [id, record.lease.deathEvidence]) - ) + const before = agentSessionLeasesBefore(draft.records) const applied = apply(draft) - proven = [...draft.records] - .filter(([id, { lease }]) => lease.deathEvidence && lease.deathEvidence !== before.get(id)) - .map(([id]) => id) + ended = agentSessionGenerationEnds(before, draft.records, options?.supersessionEvidence) return applied }, options) - for (const sessionId of proven) { - this.deathEvidenceListeners.forEach((listener) => listener(sessionId)) + for (const end of ended) { + this.generationEndListeners.forEach((listener) => listener(end)) } if (!heldBefore && this.holdsRecords()) { this.firstRecordListeners.forEach((listener) => listener()) diff --git a/src/main/runtime/agent-session-store-transactions.ts b/src/main/runtime/agent-session-store-transactions.ts index f68b9a1b6a82..a1ca00829b8a 100644 --- a/src/main/runtime/agent-session-store-transactions.ts +++ b/src/main/runtime/agent-session-store-transactions.ts @@ -8,6 +8,7 @@ // the async boundary every awaiting caller was written against. import type { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' +import type { JournalWriteOptions } from '../native-chat/agent-session-journal/journal-database' import type { JournalOperationReceipt } from '../native-chat/agent-session-journal/journal-row-writer' import { journalOpenRefusalError } from '../native-chat/agent-session-journal/journal-open-failure' import { AgentSessionJournalError } from '../native-chat/agent-session-journal/journal-write-guards' @@ -135,9 +136,11 @@ export class AgentSessionStoreTransactions { */ transact( apply: (draft: AgentSessionStoreState) => T, - options: { inMemoryWhenReadOnly?: boolean } = {} + options: { inMemoryWhenReadOnly?: boolean } & JournalWriteOptions = {} ): Promise { - const run = this.queue.then(() => this.commit(apply, options.inMemoryWhenReadOnly === true)) + const run = this.queue.then(() => + this.commit(apply, options.inMemoryWhenReadOnly === true, options.background) + ) this.queue = run.catch(() => {}) return run } @@ -185,7 +188,11 @@ export class AgentSessionStoreTransactions { return run } - private commit(apply: (draft: AgentSessionStoreState) => T, inMemoryWhenReadOnly: boolean): T { + private commit( + apply: (draft: AgentSessionStoreState) => T, + inMemoryWhenReadOnly: boolean, + background?: true + ): T { const readOnly = this.journalDatabase.readOnly if (readOnly && !inMemoryWhenReadOnly) { throw readOnlyStoreRefusal() @@ -193,7 +200,10 @@ export class AgentSessionStoreTransactions { const staged = this.stage(apply) const writes = staged.writes if (writes && !readOnly) { - this.journalDatabase.transaction((db) => writeAgentSessionStoreRows(db, writes)) + this.journalDatabase.transaction( + (db) => writeAgentSessionStoreRows(db, writes), + background ? { background } : undefined + ) } staged.adopt() return staged.result diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index d3bf795a19c6..05683fcc505c 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -73,7 +73,10 @@ describe('structured mailbox pointer host', () => { // far from the tail, so any page-sized read reports a busy worker as idle — and `@idle` then // wakes it mid-turn. const items = [runningTurn(), ...transcript(500)] - hostRef.current = { journalSnapshot: () => ({ items, submissions: [] }) } + hostRef.current = { + journalSnapshot: () => ({ items, submissions: [] }), + currentWork: () => null + } expect(await readStructuredSessionGateFacts('s1')).toEqual({ turnRunning: true, awaitingHuman: false diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index e51734b5b85e..92b1e0594823 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -47,7 +47,12 @@ export async function readStructuredSessionGateFacts( sessionId: string ): Promise { const snapshot = await readSessionJournal(sessionId) - return snapshot ? structuredSessionGateFacts(snapshot.items) : null + if (!snapshot) { + return null + } + // The host's projection of current work: an agent that ended is idle, whatever its rows say. + const work = getStructuredAgentSessionHost()?.currentWork(sessionId) + return structuredSessionGateFacts(snapshot.items, work?.scope) } /** What each recorded send settled as. */ diff --git a/src/main/runtime/orchestration/structured-session-pointer-delivery.ts b/src/main/runtime/orchestration/structured-session-pointer-delivery.ts index fc7758e07a19..8a99deb39950 100644 --- a/src/main/runtime/orchestration/structured-session-pointer-delivery.ts +++ b/src/main/runtime/orchestration/structured-session-pointer-delivery.ts @@ -8,6 +8,7 @@ */ import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' import { activeStructuredAgentSessionTurnId, projectStructuredAgentSessionStatus @@ -46,11 +47,17 @@ export type StructuredSessionGateFacts = { * and idle-with-history is the normal steady state of a working agent. */ export function structuredSessionGateFacts( - items: readonly AgentJournalRenderItem[] + items: readonly AgentJournalRenderItem[], + /** The host's projection of current work (`structuredAgentSessionCurrentWork`). */ + scope?: StructuredAgentSessionWorkScope ): StructuredSessionGateFacts { return { - turnRunning: activeStructuredAgentSessionTurnId(items) !== null, - awaitingHuman: projectStructuredAgentSessionStatus(items) === 'attention' + turnRunning: + activeStructuredAgentSessionTurnId( + items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ) !== null, + awaitingHuman: projectStructuredAgentSessionStatus(items, [], undefined, scope) === 'attention' } } diff --git a/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts b/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts index e255f5eefc80..dd1a7ec78dc9 100644 --- a/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts +++ b/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts @@ -83,7 +83,8 @@ function installHost(options: { present: true, sessionIds: options.tabListed ? [SESSION_ID] : [] }), - journalSnapshot: async () => ({ items: options.items ?? [idleTurn()] }) + journalSnapshot: async () => ({ items: options.items ?? [idleTurn()] }), + currentWork: () => null } } diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index dfda59b17b3f..ce9b35c8f3c3 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -101,12 +101,14 @@ function statusFeed(): StructuredAgentSessionStatusFeed { [ STATUS_SESSION, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot(), itemFence() and runningTurn() of a journal with no sends. journal: { cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), lastActivityAt: () => 2, stopMarks: { latest: () => null, revision: () => 0 }, - snapshot: () => ({ items: STATUS_ITEMS, submissions: [] }) + snapshot: () => ({ items: STATUS_ITEMS, submissions: [] }), + itemFence: () => undefined, + runningTurn: () => null } as unknown as AgentSessionJournal, params: { location: { @@ -143,11 +145,7 @@ export function hostStub(): StructuredAgentSessionHost { items: [], removedItemIds: [], submissions: [], - window: { - oldest: null, - newest: null, - nextCursor: { epoch: 'epoch-a', sequence: 0 } - }, + window: { oldest: null, newest: null, nextCursor: { epoch: 'epoch-a', sequence: 0 } }, liveCursor: { epoch: 'epoch-a', sequence: 0 }, hasOlder: false, hasNewer: false diff --git a/src/main/runtime/structured-agent-session-mobile-restore.test.ts b/src/main/runtime/structured-agent-session-mobile-restore.test.ts index 0e3033f000b2..2891a0361419 100644 --- a/src/main/runtime/structured-agent-session-mobile-restore.test.ts +++ b/src/main/runtime/structured-agent-session-mobile-restore.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { hostTestState, adapter, @@ -80,6 +80,13 @@ describe('prompt delivery after host recovery', () => { notificationSeq: delivered.notificationSeq } unsubscribe() + // The crashed host's last write landed before the new process opens the chat: two live + // processes never share one journal. + await vi.waitFor(async () => + expect((await h.host.journalSnapshot(SESSION)).submissions).toEqual([ + expect.objectContaining({ dispatchState: 'accepted' }) + ]) + ) const restarted = new RuntimeMobileNotificationController() restarted.configureDismissalStore(h.root) const events: MobileNotificationEvent[] = [] @@ -100,6 +107,7 @@ describe('prompt delivery after host recovery', () => { try { await host.reconcileRestartLeases() await host.restoreReadableSessions() + await host.startupSettled() const history = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(history.ok).toBe(true) expect( diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index 039b80631dda..d21aad8619aa 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -339,6 +339,8 @@ describe('restoring the chat tabs open at quit', () => { const tabWrites = spyOnTabWrites() const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B] }) await runtime.prepareStructuredAgentSessionStartupRestoration() + // Startup's settlement of what the legacy chats' last process left lands first. + await getStructuredAgentSessionHost()?.startupSettled() // One more write, then failing: a seed written chat by chat would stop part way. writes.grants = 1 diff --git a/src/main/runtime/structured-agent-session-tab-index-seed.ts b/src/main/runtime/structured-agent-session-tab-index-seed.ts index 3e2245148a26..cd1063490211 100644 --- a/src/main/runtime/structured-agent-session-tab-index-seed.ts +++ b/src/main/runtime/structured-agent-session-tab-index-seed.ts @@ -3,6 +3,7 @@ // the next launch and drop the rest. With an index present, every restored chat is already listed. import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { BACKGROUND_WRITE } from '../native-chat/agent-session-wire/structured-agent-session-background-writes' type TabIndexHost = Pick< StructuredAgentSessionHost, @@ -26,7 +27,8 @@ export async function seedStructuredAgentSessionTabIndex( (sessionId) => opened.has(sessionId) && !listed.has(sessionId) ) if (unlisted.length > 0) { - await host.showSessionTabs(unlisted).catch((error: unknown) => + // Startup's bookkeeping: another connection's lock fails it at once, and the next launch seeds. + await host.showSessionTabs(unlisted, BACKGROUND_WRITE).catch((error: unknown) => host.deps.logger.warn('recording restored chat tabs failed', { scope: 'tab-index-seed', sessionIds: unlisted, diff --git a/src/main/runtime/structured-worker-cleared-session.test.ts b/src/main/runtime/structured-worker-cleared-session.test.ts index c4d07df01fb8..037ec7beb14c 100644 --- a/src/main/runtime/structured-worker-cleared-session.test.ts +++ b/src/main/runtime/structured-worker-cleared-session.test.ts @@ -207,6 +207,8 @@ function installHost(): void { open.add(id) return { items: [message(`${id}-1`, 'idle')], submissions: [] } }, + // No projection held: the gate reads the timeline as it stands. + currentWork: () => null, setSessionTabVisibility: async (id: string, visible: boolean) => { if (id === failHideOf) { throw new Error('the durable tab index is wedged') diff --git a/src/main/sqlite/sqlite-read-failure.ts b/src/main/sqlite/sqlite-read-failure.ts index 498bf5c1de12..a1a6767eec1c 100644 --- a/src/main/sqlite/sqlite-read-failure.ts +++ b/src/main/sqlite/sqlite-read-failure.ts @@ -53,6 +53,19 @@ export function isTransientSqliteContention(error: unknown): boolean { return CONTENTION_MESSAGE.test(errorText(error)) } +/** `isTransientSqliteContention` for a failure that wraps others (`cause`, `AggregateError`). */ +export function isSqliteContentionFailure(error: unknown, depth = 0): boolean { + if (depth > 4 || error === undefined || error === null) { + return false + } + if (isTransientSqliteContention(error)) { + return true + } + const wrapped = error instanceof AggregateError ? error.errors : [] + const cause = error instanceof Error ? error.cause : undefined + return [cause, ...wrapped].some((inner) => isSqliteContentionFailure(inner, depth + 1)) +} + export type SqliteReadFailureKind = /** Another connection holds the lock; the same read can succeed later. */ | 'contended' diff --git a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx index 3ae1545874c1..10b3e7414e31 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx @@ -27,6 +27,7 @@ import { NativeChatWaitingTranscriptItems } from './NativeChatTranscriptItems' import type { NativeChatTranscriptRowContext } from './NativeChatTranscriptRow' +import type { StructuredAgentSessionHostWork } from './use-structured-agent-session-transport-state' import type { NativeChatDeliveryNotice } from './NativeChatMessageRow' import { splitNativeChatSlotsWaitingBehindLiveTurn, @@ -67,7 +68,7 @@ import { structuredAgentLabel } from '@/lib/structured-agent-session-launch-labe * `unshown`: it cannot (the prompt is only in the agent's terminal). */ export type NativeChatAwaitingInput = 'shown' | 'unshown' -type NativeChatNavigationRequest = +type NavigationRequest = | { kind: 'diff'; target: NativeChatDiffReveal } | { kind: 'rail'; messageId: string; requestId: number } @@ -77,6 +78,7 @@ export function NativeChatMessageList({ journalItems, journalSubmissions, journalLatestTurn: latestTurn, + journalHostWork: hostWork, subagentRoster, railOutline = null, isVisible = true, @@ -99,6 +101,8 @@ export function NativeChatMessageList({ journalSubmissions?: readonly AgentJournalSubmission[] /** The host's newest turn record, which places a live turn whose record is not loaded. */ journalLatestTurn?: AgentSessionLatestTurn | null + /** The host's prompts still waiting on the person and its working answer (an older host: none). */ + journalHostWork?: StructuredAgentSessionHostWork /** Every subagent the session's rosters named, whether or not its roster row is loaded. */ subagentRoster?: Parameters[2] /** User messages older than the loaded window, from the host's outline. */ @@ -121,9 +125,7 @@ export function NativeChatMessageList({ stopping?: boolean runtimeContext?: RuntimeFileOperationArgs | null }): React.JSX.Element { - const [navigationRequest, setNavigationRequest] = useState( - null - ) + const [navigationRequest, setNavigationRequest] = useState(null) const navigationSequence = useRef(0) const revealedDiff = navigationRequest?.kind === 'diff' ? navigationRequest.target : null const railJump = navigationRequest?.kind === 'rail' ? navigationRequest : null @@ -223,9 +225,10 @@ export function NativeChatMessageList({ allSlots, journalItems, stopping, - journalSubmissions + journalSubmissions, + { latestTurn, ...hostWork } ), - [allSlots, journalItems, stopping, journalSubmissions] + [allSlots, journalItems, stopping, journalSubmissions, latestTurn, hostWork] ) const transcriptWindow = useNativeChatTranscriptWindow({ scrollRef, @@ -267,11 +270,8 @@ export function NativeChatMessageList({ }) const requestRailJump = useCallback((item: NativeChatRailItem) => { navigationSequence.current += 1 - setNavigationRequest({ - kind: 'rail', - messageId: item.id, - requestId: navigationSequence.current - }) + const requestId = navigationSequence.current + setNavigationRequest({ kind: 'rail', messageId: item.id, requestId }) }, []) const railHistoryJump = useNativeChatRailHistoryJump({ items: rail.items, @@ -304,10 +304,8 @@ export function NativeChatMessageList({ openSubagentSections(target.subagentSections) } navigationSequence.current += 1 - setNavigationRequest({ - kind: 'diff', - target: { ...target, requestId: navigationSequence.current } - }) + const requestId = navigationSequence.current + setNavigationRequest({ kind: 'diff', target: { ...target, requestId } }) }, [beginNavigation, readerLeavesEnd, openSubagentSections] ) @@ -333,6 +331,7 @@ export function NativeChatMessageList({ const rowContext = useMemo( () => ({ agentName: structuredAgentLabel(session.agent), + actionablePromptIds: hostWork?.actionablePromptIds, expandSignal, revealedDiff, taskListPredecessors, @@ -348,6 +347,7 @@ export function NativeChatMessageList({ }), [ session.agent, + hostWork, allowFileUriLinks, expandSignal, expandedTurnIds, diff --git a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx index 72f1f62c21d3..00c9c4b94e05 100644 --- a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx @@ -218,6 +218,22 @@ describe('resolution receipts', () => { expect(screen.queryByText('1 grouped question from Claude')).toBeNull() }) + it('reports, without breathing, a question whose agent ended (the host says nobody waits)', () => { + const body: AgentJournalQuestionItem = { + kind: 'question', + question: 'Which branch?', + options: [], + resolution: { ...approval.resolution, state: 'pending', selectedOptionId: null } + } + + const { rerender } = render() + expect(screen.getByText('Awaiting user input:')).toHaveClass('animate-pulse') + rerender() + expect(screen.queryByText('Awaiting user input:')).toBeNull() + expect(screen.getByText('Asked:')).not.toHaveClass('animate-pulse') + expect(screen.getByText('Which branch?')).toBeInTheDocument() + }) + it('keeps an opened question open once it is answered', () => { const scrollWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'scrollWidth') Object.defineProperty(HTMLElement.prototype, 'scrollWidth', { diff --git a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx index f3e585ae5f9f..4d65f6400d1a 100644 --- a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx +++ b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx @@ -9,11 +9,15 @@ import { export function NativeChatResolutionReceipt({ body, - disclosureId + disclosureId, + awaiting = true }: { body: NativeChatResolvedPrompt /** Message this receipt stands in for; keys the question row's disclosure. */ disclosureId?: string + /** False when the host says nobody waits on this pending prompt: the agent that raised it ended, + * so its row reports what was asked instead of breathing. */ + awaiting?: boolean }): React.JSX.Element | null { const askDisclosureKey = disclosureId === undefined ? undefined : `ask:${disclosureId}` const subject: NativeChatAskRowSubject | null = @@ -35,7 +39,11 @@ export function NativeChatResolutionReceipt({ } if (body.resolution.state === 'pending') { return body.kind === 'question' ? ( - + ) : null } const { resolution } = body diff --git a/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx b/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx index b3efd9fa4c65..5d91e057be31 100644 --- a/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx +++ b/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx @@ -294,6 +294,7 @@ export function NativeChatStructuredSession( journalItems={controller.journalItems} journalSubmissions={controller.submissions} journalLatestTurn={controller.latestTurn} + journalHostWork={controller.hostWork} subagentRoster={controller.subagentRoster} railOutline={controller.railOutline} isVisible={props.isVisible} diff --git a/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx b/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx index b48227cc70c8..d41a7da47c4c 100644 --- a/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx +++ b/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx @@ -156,8 +156,8 @@ export function NativeChatSubagentEntries({ * reporting into this group long after a newer turn opened, so a turn boundary * is a fact about the turn and never evidence that contact with a child was * lost. Only a host can say that, and one does: `CodexSubagentRoster.settleSession` - * when the provider goes away, and `staleSubagentRosterRevisions` on the next - * journal open when the host itself died mid-flight. */ + * when the provider goes away, and the leftover settlement at the next ownership + * event when the host itself died mid-flight. */ export function NativeChatSubagentRun({ block, open: heldOpen, diff --git a/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx b/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx index 446deba8da5c..aac778310a4d 100644 --- a/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx @@ -14,11 +14,14 @@ import type { NativeChatTranscriptSlot } from './native-chat-transcript-slots' import type { NativeChatDiffReveal, NativeChatDiffTarget } from './native-chat-turn-diffs' import { NativeChatRewindContext } from './native-chat-rewind-context' import { nativeChatRowOffersRewind } from './native-chat-rewind-eligibility' +import { isActionableStructuredAgentSessionPrompt } from '../../../../shared/structured-agent-session-live-turn' /** Everything a row needs that is the same for every row. Held as one memoized * object so a row's props change only when that row's own slot does. */ export type NativeChatTranscriptRowContext = { agentName?: string + /** The prompts the host says still wait on the person; absent from an older host. */ + actionablePromptIds?: readonly string[] | undefined expandSignal: boolean revealedDiff: NativeChatDiffReveal | null taskListPredecessors: ReadonlyMap @@ -98,7 +101,14 @@ export const NativeChatTranscriptRow = memo(function NativeChatTranscriptRow({ {/* A turn with no user bubble carries its bar above its first row. */} {slot.statusAbove ? statusRow : null} {!slot.drawsMessage ? null : receipt ? ( - + ) : ( string> = @@ -148,18 +149,7 @@ const PIECES: Record - translate( - 'components.native-chat.failureWords.providerExitedRow', - COPY.providerExitedRow, - values - ), - providerExitedRejection: (values) => - translate( - 'components.native-chat.failureWords.providerExitedRejection', - COPY.providerExitedRejection, - values - ), + ...PROVIDER_EXIT_PIECES, providerRejected: () => translate('components.native-chat.failureWords.providerRejected', COPY.providerRejected), providerRejectedQuoted: (values) => diff --git a/src/renderer/src/components/native-chat/agent-session-provider-exit-words-text.ts b/src/renderer/src/components/native-chat/agent-session-provider-exit-words-text.ts new file mode 100644 index 000000000000..a9b3785888de --- /dev/null +++ b/src/renderer/src/components/native-chat/agent-session-provider-exit-words-text.ts @@ -0,0 +1,31 @@ +// Desktop words for an agent that stopped, kept beside the other failure pieces' translations. + +import { translate } from '@/i18n/i18n' +import { + AGENT_SESSION_FAILURE_COPY as COPY, + type AgentSessionFailureCopyId, + type AgentSessionFailureCopyValues +} from '../../../../shared/agent-session-failure-copy' + +export const PROVIDER_EXIT_PIECES = { + providerExitedRow: (values) => + translate( + 'components.native-chat.failureWords.providerExitedRow', + COPY.providerExitedRow, + values + ), + providerExitedRejection: (values) => + translate( + 'components.native-chat.failureWords.providerExitedRejection', + COPY.providerExitedRejection, + values + ), + providerExitedAnswer: (values) => + translate( + 'components.native-chat.failureWords.providerExitedAnswer', + COPY.providerExitedAnswer, + values + ) +} satisfies Partial< + Record string> +> diff --git a/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts b/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts index 3589b1084a0b..97248e1be30c 100644 --- a/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts +++ b/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts @@ -2,7 +2,10 @@ // shared English as its fallback so desktop and mobile never say it differently. import { translate } from '@/i18n/i18n' -import { agentSessionFailureSentence } from '../../../../shared/agent-session-failure-words' +import { + agentSessionFailureSentence, + type AgentSessionFailureWordsContext +} from '../../../../shared/agent-session-failure-words' import { agentSessionWriteNoticeParts } from '../../../../shared/agent-session-refusal-notice' import { AGENT_SESSION_WRITE_NOTICE_COPY as COPY, @@ -178,7 +181,8 @@ export function agentSessionWriteNoticeText(parts: readonly AgentSessionWriteNot export function agentSessionWriteFailureText( failure: AgentSessionWriteFailure, - write: AgentSessionWriteKind + write: AgentSessionWriteKind, + context: AgentSessionFailureWordsContext = {} ): string { - return agentSessionWriteNoticeText(agentSessionWriteNoticeParts(failure, write)) + return agentSessionWriteNoticeText(agentSessionWriteNoticeParts(failure, write, context)) } diff --git a/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts b/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts index 2b4a748a79de..14c1e0e0cbc5 100644 --- a/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts +++ b/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts @@ -322,13 +322,15 @@ export function splitNativeChatSlotsWaitingBehindLiveTurn( slots: readonly NativeChatTranscriptSlot[], journalItems: readonly AgentJournalRenderItem[] | undefined, stopping = false, - journalSubmissions?: readonly AgentJournalSubmission[] + journalSubmissions?: readonly AgentJournalSubmission[], + host: Parameters[4] = {} ): { slots: NativeChatTranscriptSlot[]; waitingSlots: NativeChatTranscriptSlot[] } { const waiting = nativeChatMessagesWaitingBehindLiveTurn( slots.flatMap((slot) => (slot.kind === 'message' ? [slot.message] : [])), journalItems, stopping, - journalSubmissions + journalSubmissions, + host ) const isWaiting = (slot: NativeChatTranscriptSlot): boolean => slot.kind === 'message' && diff --git a/src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts b/src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts new file mode 100644 index 000000000000..530ef37163e0 --- /dev/null +++ b/src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts @@ -0,0 +1,57 @@ +// The desktop chat answers only the prompts the host says still wait on the person: one an agent +// that has ended raised gets no card, and an older host, which names none, keeps today's rule. + +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' +import { EMPTY_STRUCTURED_AGENT_SESSION } from '../../../../shared/structured-agent-session-reducer' +import { pendingStructuredSessionPrompts } from './structured-agent-session-message-projection' +import { structuredAttentionReadObservation } from './structured-attention-read-observation' + +function approval(itemId: string, sequence: number): AgentJournalRenderItem { + return { + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } + } +} + +const ITEMS = [approval('live-prompt', 1), approval('dead-prompt', 2)] +const ids = (items: AgentJournalRenderItem[]) => items.map((item) => item.itemId) + +describe('the prompt cards a desktop chat offers', () => { + it('are the ones the host names', () => { + expect(ids(pendingStructuredSessionPrompts(ITEMS, ['live-prompt']))).toEqual(['live-prompt']) + expect(pendingStructuredSessionPrompts(ITEMS, [])).toEqual([]) + }) + + it('are every pending prompt when an older host names none', () => { + expect(ids(pendingStructuredSessionPrompts(ITEMS))).toEqual(['live-prompt', 'dead-prompt']) + }) +}) + +describe('what marks a chat read', () => { + it('changes when the host stops naming a prompt, with no row changing', () => { + const ask: AgentJournalRenderItem = { + itemId: 'ask', + revision: 1, + sequence: 0, + observedAt: 0, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'go' }] } + } + const state = { ...EMPTY_STRUCTURED_AGENT_SESSION, items: [ask, ...ITEMS] } + const named = structuredAttentionReadObservation({ + ...state, + actionablePromptIds: ['live-prompt'] + }) + const ended = structuredAttentionReadObservation({ ...state, actionablePromptIds: [] }) + expect(ended).not.toBe(named) + }) +}) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts b/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts index 25ff332ceb87..fa9297874932 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts @@ -6,6 +6,7 @@ import type { import type { StructuredAgentSessionOptimisticMessage } from '../../../../shared/structured-agent-session-message-projection' import { projectStructuredAgentSessionMessages as projectMessages } from '../../../../shared/structured-agent-session-message-projection' import { projectStructuredQuestionMessages } from './structured-agent-question-projection' +import { isActionableStructuredAgentSessionPrompt } from '../../../../shared/structured-agent-session-live-turn' /** The desktop's transcript: a message the host accepted and then rejected stays where it was * sent, as not sent, unless the queue holds it as a card. */ @@ -27,12 +28,16 @@ export type StructuredPromptItem = AgentJournalRenderItem & { body: Extract } +/** The prompt cards a person can answer: pending, and named by the host as waiting + * (`isActionableStructuredAgentSessionPrompt`) when the host names them. */ export function pendingStructuredSessionPrompts( - items: AgentJournalRenderItem[] + items: AgentJournalRenderItem[], + actionablePromptIds?: readonly string[] ): StructuredPromptItem[] { return agentSessionCurrentContextRows(items).items.filter( (item): item is StructuredPromptItem => (item.body.kind === 'approval' || item.body.kind === 'question') && - item.body.resolution.state === 'pending' + item.body.resolution.state === 'pending' && + isActionableStructuredAgentSessionPrompt(item.itemId, actionablePromptIds) ) } diff --git a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts index 5653e8f25b19..7b4b692dfe49 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts @@ -164,6 +164,13 @@ describe('queued message cards', () => { expect(cards.map((card) => queuedMessageCardSteers(card))).toEqual([true, true]) }) + it('a waiting card still offers Steer while nothing runs, as when its agent died and its cleanup waits', () => { + const cards = projectQueuedMessageCards([draft('waiting', 1)], [], { agentWorking: false }) + expect(cards.map((card) => [card.hold, queuedMessageCardSteers(card)])).toEqual([ + ['turn', true] + ]) + }) + it("the header names the queue's pause while it holds a card, and none over cards Resume would not send", () => { const stopped = { reason: 'stopped' } as const const project = (messages: AgentSessionQueuedMessage[], queuePaused = true) => diff --git a/src/renderer/src/components/native-chat/structured-attention-read-observation.ts b/src/renderer/src/components/native-chat/structured-attention-read-observation.ts index a614438deb57..8a365d4fc652 100644 --- a/src/renderer/src/components/native-chat/structured-attention-read-observation.ts +++ b/src/renderer/src/components/native-chat/structured-attention-read-observation.ts @@ -1,5 +1,6 @@ import type { StructuredAgentSessionState } from '../../../../shared/structured-agent-session-reducer' import { projectStructuredAgentSessionStatusState } from '../../../../shared/structured-agent-session-projection' +import { isActionableStructuredAgentSessionPrompt } from '../../../../shared/structured-agent-session-live-turn' const observations = new WeakMap() @@ -16,7 +17,10 @@ export function structuredAttentionReadObservation(state: StructuredAgentSession const request = projection.latestRequest const key = JSON.stringify([ state.cursor?.epoch, - projection.pendingPromptIds, + // The host's answer where it gives one: a prompt an agent that ended raised waits on no one. + projection.pendingPromptIds.filter((itemId) => + isActionableStructuredAgentSessionPrompt(itemId, state.actionablePromptIds) + ), request?.turnState !== 'running' ? [request?.kind, request?.id, request?.outcome] : null ]) observations.set(state, key) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts index bfccacad0933..55b02cf3ccc5 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts @@ -57,11 +57,16 @@ function namesWhatItStops(fingerprintMethod: string, fields: Record + fields: Record, + agentName: string | undefined ): { kind: 'not-done'; notice: string; failure: AgentSessionWriteFailure } { return { kind: 'not-done', - notice: agentSessionWriteFailureText(failure, writeKind(fingerprintMethod, fields)), + notice: agentSessionWriteFailureText( + failure, + writeKind(fingerprintMethod, fields), + agentName ? { agentName } : {} + ), failure } } @@ -73,11 +78,13 @@ export function useStructuredAgentSessionMutate(args: { /** Read at settle time, not at call time: the fence can move while a request * is in flight, and a result from the previous fence is not this session's. */ stateRef: { current: { fence: number | null } } + /** The chat's agent, which a refusal about it names. */ + agentName?: string }): { write: StructuredAgentSessionWrite mutate: StructuredAgentSessionMutate } { - const { enabled = true, sessionId, stateRef, target } = args + const { agentName, enabled = true, sessionId, stateRef, target } = args const inFlightStops = useRef( new Map>>() ) @@ -144,13 +151,19 @@ export function useStructuredAgentSessionMutate(args: { error instanceof RuntimeRpcCallError ? error.code : undefined ), fingerprintMethod, - fields + fields, + agentName ) : { kind: 'dropped' } } if (!result.ok) { return settlesHere() - ? notDone(agentSessionRefusalFailure(result.refusal), fingerprintMethod, fields) + ? notDone( + agentSessionRefusalFailure(result.refusal), + fingerprintMethod, + fields, + agentName + ) : { kind: 'dropped' } } if (!settlesHere()) { @@ -158,7 +171,7 @@ export function useStructuredAgentSessionMutate(args: { } return { kind: 'done', value: result.value } }, - [enabled, sessionId, stateRef, target] + [agentName, enabled, sessionId, stateRef, target] ) const write = useCallback( diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts index ef730a1781df..84f1eec124a0 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts @@ -1,5 +1,8 @@ import { useMemo } from 'react' -import { runningStructuredAgentSessionTurnId } from '../../../../shared/structured-agent-session-live-turn' +import { + runningStructuredAgentSessionTurnId, + structuredAgentSessionHostSaysWorking +} from '../../../../shared/structured-agent-session-live-turn' import { isStructuredAgentSessionMainAgentWorking } from '../../../../shared/structured-agent-session-main-agent-working' import type { StructuredAgentSessionState } from '../../../../shared/structured-agent-session-reducer' import type { StructuredAgentSubagentRoster } from '../../../../shared/structured-agent-session-subagent-roster' @@ -13,6 +16,13 @@ const NO_JOURNAL_ITEMS: StructuredAgentSessionState['items'] = [] const NO_SUBMISSIONS: StructuredAgentSessionState['submissions'] = [] const NO_SUBAGENT_ROSTER: StructuredAgentSubagentRoster = new Map() +/** The host's prompts still waiting on the person and its working answer; empty from an older + * host, whose readers derive both from the rows. */ +export type StructuredAgentSessionHostWork = { + actionablePromptIds?: readonly string[] + working?: boolean +} + export function useStructuredAgentSessionTransportState( state: StructuredAgentSessionState, enabled: boolean @@ -22,14 +32,20 @@ export function useStructuredAgentSessionTransportState( const subagentRoster = (enabled ? state.subagentRoster : undefined) ?? NO_SUBAGENT_ROSTER const fence = enabled ? state.fence : null const latestTurn = enabled ? state.latestTurn : undefined + // The host's answer to which pending prompts still wait on the person; absent from an older host. + const actionablePromptIds = enabled ? state.actionablePromptIds : undefined const current = useMemo( () => agentSessionCurrentContextRows(journalItems, submissions), [journalItems, submissions] ) // The host's whole-journal turn, never the loaded rows': a long turn's record is off the page. const turnId = runningStructuredAgentSessionTurnId({ items: journalItems, latestTurn }) - // The rule the host projects every session list's Working from, so this chat cannot disagree. - const isWorking = isStructuredAgentSessionMainAgentWorking(turnId, current.submissions, fence) + // The host's own answer, which counts only the agent running now; an older host gives none, and + // the rule it projects every session list's Working from stands in, so this chat cannot disagree. + const hostWorking = enabled ? state.working : undefined + const isWorking = structuredAgentSessionHostSaysWorking(hostWorking, () => + isStructuredAgentSessionMainAgentWorking(turnId, current.submissions, fence) + ) const nextQueuedMessageId = (enabled ? state.nextQueuedMessageId : null) ?? null // The host names the card its queue sends next. That send lands in a later update than a turn's // end or a Resume, so until then the chat still reads as working and nothing flips in between. @@ -55,9 +71,18 @@ export function useStructuredAgentSessionTransportState( structuredSessionForegroundCommands(roster, { items: journalItems, latestTurn }) ) }, [enabled, journalItems, latestTurn, state.backgroundTasks, turnId]) + const hostWork = useMemo( + () => ({ + ...(actionablePromptIds ? { actionablePromptIds } : {}), + ...(hostWorking !== undefined ? { working: hostWorking } : {}) + }), + [actionablePromptIds, hostWorking] + ) return { journalItems, latestTurn, + actionablePromptIds, + hostWork, subagentRoster, submissions, fence, diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx new file mode 100644 index 000000000000..4edac4098a2e --- /dev/null +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx @@ -0,0 +1,153 @@ +// @vitest-environment happy-dom + +// Whether the chat's own agent is working is the host's answer when it gives one: it counts only +// the agent running now, so a turn an ended agent left running reads as stopped and Resume shows. +// An older host gives none, and the chat derives it from its rows as before. + +import { cleanup, renderHook } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' +import type { AgentSessionHistoryPage } from '../../../../shared/agent-session-wire' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession, + type StructuredAgentSessionState +} from '../../../../shared/structured-agent-session-reducer' + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: vi.fn() +})) + +import { useStructuredAgentSessionTransportState } from './use-structured-agent-session-transport-state' +import { useStructuredAgentSessionQueuedMessages } from './use-structured-agent-session-queued-messages' +import { useStructuredAgentSessionMutate } from './use-structured-agent-session-mutate' + +afterEach(cleanup) + +/** A turn row an agent that has since ended left running. */ +const LEFT_RUNNING: AgentJournalRenderItem = { + itemId: 'turn-1', + sequence: 1, + revision: 1, + observedAt: 1, + body: { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-1', state: 'running' } + } +} + +function page(fields: Partial = {}): AgentSessionHistoryPage { + const cursor = { epoch: 'epoch-a', sequence: 1 } + return { + sessionId: 'session-a', + epoch: 'epoch-a', + direction: 'tail', + items: [LEFT_RUNNING], + removedItemIds: [], + submissions: [], + window: { oldest: cursor, newest: cursor, nextCursor: cursor }, + liveCursor: cursor, + hasOlder: false, + hasNewer: false, + ...fields + } +} + +function loaded(fields: Partial): StructuredAgentSessionState { + return reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'history-page', + page: page(fields) + }) +} + +function isWorking(state: StructuredAgentSessionState): boolean { + return renderHook(() => useStructuredAgentSessionTransportState(state, true)).result.current + .isWorking +} + +describe('the chat’s Working', () => { + it('is the host’s answer: a turn an ended agent left running reads as stopped', () => { + expect(isWorking(loaded({ working: false }))).toBe(false) + expect(isWorking(loaded({ items: [], working: true }))).toBe(true) + }) + + it('falls back to the rows when an older host gives no answer', () => { + expect(loaded({}).working).toBeUndefined() + expect(isWorking(loaded({}))).toBe(true) + }) + + it('follows a frame that restates it with no row, and keeps it across a frame that names none', () => { + const stopped = reduceStructuredAgentSession(loaded({ working: true }), { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + items: [], + removedItemIds: [], + submissions: [], + cursor: { epoch: 'epoch-a', sequence: 1 } + }, + working: false + } + }) + expect(stopped.working).toBe(false) + expect(isWorking(stopped)).toBe(false) + const quiet = reduceStructuredAgentSession(stopped, { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + items: [], + removedItemIds: [], + submissions: [], + cursor: { epoch: 'epoch-a', sequence: 1 } + } + } + }) + expect(quiet.working).toBe(false) + }) + + it('offers Resume over held cards once the host says the agent stopped; an older host’s rows hide it', () => { + const held = { + queuePause: { reason: 'stopped' as const }, + queuedMessages: [ + { + messageId: 'held', + position: 1, + body: { kind: 'message' as const, role: 'user' as const, blocks: [] }, + state: 'waiting' as const + } + ] + } + const resume = (state: StructuredAgentSessionState) => + renderHook(() => { + const transport = useStructuredAgentSessionTransportState(state, true) + const { mutate } = useStructuredAgentSessionMutate({ + sessionId: 'session-a', + target: { kind: 'local' }, + stateRef: { current: { fence: 1 } } + }) + return useStructuredAgentSessionQueuedMessages({ + enabled: true, + queuedMessages: transport.queuedMessages ?? [], + queuePause: transport.queuePause, + submissions: [], + hasPendingPrompt: false, + isWorking: transport.isWorking, + composerScopeKey: undefined, + mutate, + editTransport: { + target: { kind: 'local' }, + sessionId: 'session-a', + capable: false, + write: async () => ({ kind: 'dropped' }) + } + }) + }).result.current.queueResume + expect(resume(loaded({ ...held, working: false }))).toBeDefined() + expect(resume(loaded(held))).toBeUndefined() + }) +}) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts index 47ca68a5db76..0eb220affd2e 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts @@ -9,8 +9,10 @@ export function useStructuredAgentSessionTransport(args: { target: RuntimeClientTarget isVisible: boolean enabled: boolean + /** The chat's agent, which a refused write about it names. */ + agentName?: string }) { - const { enabled, isVisible, sessionId, target } = args + const { agentName, enabled, isVisible, sessionId, target } = args const providerVisible = isVisible && enabled useStructuredAgentSessionHold({ sessionId, @@ -25,7 +27,8 @@ export function useStructuredAgentSessionTransport(args: { sessionId, target, stateRef, - enabled + enabled, + ...(agentName ? { agentName } : {}) }) useEffect(() => { stateRef.current = read.state diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session.ts b/src/renderer/src/components/native-chat/use-structured-agent-session.ts index 608bf683791a..c22e712eb803 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session.ts @@ -91,7 +91,8 @@ export function useStructuredAgentSession(args: { sessionId, target, isVisible, - enabled: transportEnabled + enabled: transportEnabled, + agentName: structuredAgentLabel(agent) }) const commandPending = useRef(false) const legacyLaunchStop = useStructuredLegacyLaunchStop(sessionId) @@ -134,7 +135,10 @@ export function useStructuredAgentSession(args: { hostStopping, mutate }) - const prompts = pendingStructuredSessionPrompts(transportState.journalItems) + const prompts = pendingStructuredSessionPrompts( + transportState.journalItems, + transportState.actionablePromptIds + ) const promptsUnanswerableHere = pendingPromptsAllUnanswerableHere(prompts) // A send after a command the queue will run goes behind it, even with follow-ups off. // A host's queue waits on any pending prompt, and nothing here can settle one this build cannot @@ -250,6 +254,8 @@ export function useStructuredAgentSession(args: { journalItems: transcriptItems, /** The host's newest turn record, which places a live turn whose record is not loaded. */ latestTurn: transportState.latestTurn, + /** The host's prompts still waiting on the person and its working answer. */ + hostWork: transportState.hostWork, subagentRoster: transportState.subagentRoster, messages, status: transportEnabled ? state.status : 'ready', diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index 55263d5ee7fc..40ab6edccb6a 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2892,6 +2892,7 @@ "notSignedIn": "{{agent}} is not signed in.", "previousExitUnverifiable": "Couldn't stop {{agent}} from before.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", + "providerExitedAnswer": "{{agent}} has stopped, so your answer was not sent. Send a message to continue.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerRateLimited": "{{agent}} is rate-limited and retrying.", "providerRejected": "The provider did not accept this message.", diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index 2f8812801cee..7725d93308e1 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -18062,6 +18062,7 @@ "chooseClaudeAccountThenSend": "Choose or add one in Claude Accounts settings, then send your message again.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", + "providerExitedAnswer": "{{agent}} has stopped, so your answer was not sent. Send a message to continue.", "providerRejected": "The provider did not accept this message.", "providerRejectedQuoted": "The provider did not accept this message: {{detail}}.", "attachmentEmpty": "An image on this message is empty, so the message was not sent.", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index 2c7e4f05eda9..369c746c8396 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -17762,6 +17762,7 @@ "chooseClaudeAccountThenSend": "Elija o agregue una en la configuración de Cuentas de Claude y vuelva a enviar su mensaje.", "providerExitedRow": "{{agent}} se detuvo mientras esta respuesta estaba en curso. Puede continuar en esta conversación.", "providerExitedRejection": "{{agent}} se detuvo antes de que se enviara este mensaje.", + "providerExitedAnswer": "{{agent}} se detuvo, así que su respuesta no se envió. Envíe un mensaje para continuar.", "providerRejected": "El proveedor no aceptó este mensaje.", "providerRejectedQuoted": "El proveedor no aceptó este mensaje: {{detail}}.", "attachmentEmpty": "Una imagen de este mensaje está vacía, así que el mensaje no se envió.", diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 04f622973aba..44aa3ff9b9ac 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17864,6 +17864,7 @@ "chooseClaudeAccountThenSend": "Choisissez-en un ou ajoutez-en un dans les paramètres Comptes Claude, puis renvoyez votre message.", "providerExitedRow": "{{agent}} s'est arrêté pendant que cette réponse était en cours. Vous pouvez continuer dans cette conversation.", "providerExitedRejection": "{{agent}} s'est arrêté avant l'envoi de ce message.", + "providerExitedAnswer": "{{agent}} s'est arrêté, donc votre réponse n'a pas été envoyée. Envoyez un message pour continuer.", "providerRejected": "Le fournisseur n'a pas accepté ce message.", "providerRejectedQuoted": "Le fournisseur n'a pas accepté ce message : {{detail}}.", "attachmentEmpty": "Une image de ce message est vide, le message n'a donc pas été envoyé.", diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index a632ba19b243..cd2fdd042efd 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17797,6 +17797,7 @@ "chooseClaudeAccountThenSend": "Claude アカウントの設定で選択または追加してから、メッセージをもう一度送信してください。", "providerExitedRow": "この応答の途中で {{agent}} が停止しました。この会話で続けることができます。", "providerExitedRejection": "このメッセージが送信される前に {{agent}} が停止しました。", + "providerExitedAnswer": "{{agent}} が停止したため、回答は送信されませんでした。続けるにはメッセージを送信してください。", "providerRejected": "プロバイダーはこのメッセージを受け付けませんでした。", "providerRejectedQuoted": "プロバイダーはこのメッセージを受け付けませんでした: {{detail}}。", "attachmentEmpty": "このメッセージの画像が空のため、メッセージは送信されませんでした。", diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 4e9aac6cd485..ebd15870baa9 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17797,6 +17797,7 @@ "chooseClaudeAccountThenSend": "Claude 계정 설정에서 선택하거나 추가한 다음 메시지를 다시 보내세요.", "providerExitedRow": "이 응답이 진행되는 동안 {{agent}}이(가) 중지되었습니다. 이 대화에서 계속할 수 있습니다.", "providerExitedRejection": "이 메시지가 전송되기 전에 {{agent}}이(가) 중지되었습니다.", + "providerExitedAnswer": "{{agent}}이(가) 중지되어 답변이 전송되지 않았습니다. 계속하려면 메시지를 보내세요.", "providerRejected": "공급자가 이 메시지를 받아들이지 않았습니다.", "providerRejectedQuoted": "공급자가 이 메시지를 받아들이지 않았습니다: {{detail}}.", "attachmentEmpty": "이 메시지의 이미지가 비어 있어 메시지가 전송되지 않았습니다.", diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 0592f647928e..151a8f16647c 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17776,6 +17776,7 @@ "chooseClaudeAccountThenSend": "请在 Claude 账户设置中选择或添加一个，然后重新发送您的消息。", "providerExitedRow": "{{agent}} 在生成此回复时停止了。您可以在此对话中继续。", "providerExitedRejection": "{{agent}} 在发送此消息前已停止。", + "providerExitedAnswer": "{{agent}} 已停止，因此您的回答未发送。发送消息以继续。", "providerRejected": "提供商未接受此消息。", "providerRejectedQuoted": "提供商未接受此消息：{{detail}}。", "attachmentEmpty": "此消息中的一张图片为空，因此消息未发送。", diff --git a/src/shared/agent-session-failure-copy.ts b/src/shared/agent-session-failure-copy.ts index d498565e3847..045d62135118 100644 --- a/src/shared/agent-session-failure-copy.ts +++ b/src/shared/agent-session-failure-copy.ts @@ -76,6 +76,8 @@ export const AGENT_SESSION_FAILURE_COPY = { providerExitedRow: '{{agent}} stopped while this response was in progress. You can continue in this conversation.', providerExitedRejection: '{{agent}} stopped before this message was sent.', + providerExitedAnswer: + '{{agent}} has stopped, so your answer was not sent. Send a message to continue.', providerRejected: 'The provider did not accept this message.', providerRejectedQuoted: 'The provider did not accept this message: {{detail}}.', attachmentEmpty: 'An image on this message is empty, so the message was not sent.', diff --git a/src/shared/agent-session-failure-words.test.ts b/src/shared/agent-session-failure-words.test.ts index bfdc644b8bee..f0eff4cdbffa 100644 --- a/src/shared/agent-session-failure-words.test.ts +++ b/src/shared/agent-session-failure-words.test.ts @@ -24,7 +24,7 @@ import { /** What Orca's own text looks like: a code or marker, a uuid, a path, an exception. */ const ORCA_INTERNAL = /\b[a-z]+_[a-z_]+\b|[0-9a-f]{8}-[0-9a-f]{4}-|[/\\][\w.-]+[/\\]|Error:|ENOENT/ -const SURFACES: readonly AgentSessionFailureSurface[] = ['row', 'rejection'] +const SURFACES: readonly AgentSessionFailureSurface[] = ['row', 'rejection', 'answer'] const LEGACY_MARKER_KINDS: ReadonlySet = new Set([ 'cancelled', 'writeFailed', @@ -76,7 +76,11 @@ describe('the words written beside a failure fact', () => { expect(sentence).not.toMatch(ORCA_INTERNAL) if (surface === 'row') { expect(agentSessionFailureWords(fact, { ...context, surface }).text).toBe(sentence) - } else if (isSubmissionRejectionFact(fact) && !LEGACY_MARKER_KINDS.has(fact.kind)) { + } else if ( + surface === 'rejection' && + isSubmissionRejectionFact(fact) && + !LEGACY_MARKER_KINDS.has(fact.kind) + ) { expect(agentSessionFailureWords(fact, { ...context, surface }).reason).toBe(sentence) } } diff --git a/src/shared/agent-session-failure-words.ts b/src/shared/agent-session-failure-words.ts index cccb145c8af3..64764c2a4035 100644 --- a/src/shared/agent-session-failure-words.ts +++ b/src/shared/agent-session-failure-words.ts @@ -54,8 +54,9 @@ export type AgentJournalDispatchRejection = { rejection: SubmissionRejectionFact } -/** `row`: a status row, about the chat. `rejection`: a rejected message's reason, about it. */ -export type AgentSessionFailureSurface = 'row' | 'rejection' +/** `row`: a status row, about the chat. `rejection`: a rejected message's reason, about it. + * `answer`: a refused answer to an approval or question, about it. */ +export type AgentSessionFailureSurface = 'row' | 'rejection' | 'answer' export type AgentSessionFailureWordsContext = { /** The chat's agent, when the writer knows it. */ @@ -235,7 +236,14 @@ const FAILURE_SENTENCES = { : say('chooseClaudeAccountThenSend') ]), providerExited: (context, _fact, surface, say) => - say(surface === 'row' ? 'providerExitedRow' : 'providerExitedRejection', agent(say, context)), + say( + surface === 'row' + ? 'providerExitedRow' + : surface === 'answer' + ? 'providerExitedAnswer' + : 'providerExitedRejection', + agent(say, context) + ), restartFailed: couldNot('couldNotRestart'), providerRejected: (_context, fact, _surface, say) => quotingPersonDetail(say, 'providerRejected', 'providerRejectedQuoted', fact.detail), diff --git a/src/shared/agent-session-journal-schemas.ts b/src/shared/agent-session-journal-schemas.ts index 9e5262693514..6c63c6e1d397 100644 --- a/src/shared/agent-session-journal-schemas.ts +++ b/src/shared/agent-session-journal-schemas.ts @@ -70,7 +70,8 @@ const SubagentEntry = z.object({ state: z.string().min(1), tokens: z.number().optional(), startedAt: z.number().optional(), - settledAt: z.number().optional() + settledAt: z.number().optional(), + ownerFence: z.number().int().nonnegative().optional() }) /** Renderers select blocks by `type` equality and skip what they cannot draw, @@ -125,7 +126,8 @@ const Block = openDiscriminatedUnion( outputFile: z.string().optional(), tokens: z.number().optional(), startedAt: z.number().optional(), - settledAt: z.number().optional() + settledAt: z.number().optional(), + ownerFence: z.number().int().nonnegative().optional() }) ]) ) diff --git a/src/shared/agent-session-lease-adjudication.ts b/src/shared/agent-session-lease-adjudication.ts index 75de4a74c40d..3365159b7a0b 100644 --- a/src/shared/agent-session-lease-adjudication.ts +++ b/src/shared/agent-session-lease-adjudication.ts @@ -83,7 +83,8 @@ export function isProvenAliveProbe(probe: AgentSessionOwnerProbe): boolean { return probe.outcome === 'identity-matched' && probe.matchedOn.length > 0 } -function deathEvidenceFor( +/** The proof a probe gives that the lease's owner is gone, or null when it proves nothing. */ +export function agentSessionDeathEvidenceFromProbe( probe: AgentSessionOwnerProbe, observedAt: number, lease: AgentSessionLease @@ -270,7 +271,7 @@ export function adjudicateAgentSessionRestart(args: { reason: 'owner outlived the runtime that held its transport' } } - const evidence = deathEvidenceFor(probe, observedAt, lease) + const evidence = agentSessionDeathEvidenceFromProbe(probe, observedAt, lease) if (evidence) { return { disposition: 'evicted', nextFence: nextAgentSessionFence(lease), evidence } } diff --git a/src/shared/agent-session-refusal-details.ts b/src/shared/agent-session-refusal-details.ts index ae52282631a8..5046cc529652 100644 --- a/src/shared/agent-session-refusal-details.ts +++ b/src/shared/agent-session-refusal-details.ts @@ -47,6 +47,8 @@ export const AGENT_SESSION_REFUSAL_REASONS = { 'rewindUnconfirmed', // A prompt card or an option 'promptGone', + /** The agent that raised the prompt has ended, so nothing can take its answer. */ + 'promptOwnerEnded', 'optionRejected', // The provider 'providerStarting', diff --git a/src/shared/agent-session-refusal-notice.test.ts b/src/shared/agent-session-refusal-notice.test.ts index aadf9e26fa49..c784fecbee3f 100644 --- a/src/shared/agent-session-refusal-notice.test.ts +++ b/src/shared/agent-session-refusal-notice.test.ts @@ -451,6 +451,12 @@ describe('the notice for every reason a host names', () => { } const notDone = parts.filter((part) => typeof part === 'string' && part.startsWith('notDone')) const answeredAway = write === 'answer' && parts.includes('questionChanged') + // "...so your answer was not sent" is that sentence's own. + const ownerStopped = + write === 'answer' && + parts.some( + (part) => typeof part === 'object' && 'surface' in part && part.surface === 'answer' + ) const unsupported = failure.code === 'structured_agent_session_unsupported' && failure.details?.reason !== 'hostUnsupported' && @@ -467,7 +473,9 @@ describe('the notice for every reason a host names', () => { ] as const ).some((sentence) => parts.includes(sentence)) expect(notDone, cell).toEqual( - answeredAway || unsupported || saysNotDone || clearWhileWorking ? [] : [NOT_DONE[write]] + answeredAway || ownerStopped || unsupported || saysNotDone || clearWhileWorking + ? [] + : [NOT_DONE[write]] ) } }) @@ -526,6 +534,12 @@ describe('the notice for every reason a host names', () => { 'answer', 'This question was already answered or has changed.' ], + [ + 'agent_session_operation_invalid', + 'promptOwnerEnded', + 'answer', + 'The agent has stopped, so your answer was not sent. Send a message to continue.' + ], [ 'agent_session_checkpoint_stale', 'fenceStale', @@ -539,6 +553,33 @@ describe('the notice for every reason a host names', () => { }) }) +describe('an answer to a prompt whose agent stopped', () => { + it('names the agent the client knows, on the phone and on desktop alike', () => { + const refusal = { + code: 'agent_session_operation_invalid' as const, + message: HOST_TEXT, + details: { reason: 'promptOwnerEnded' as const } + } + expect(agentSessionRefusalNotice(refusal, 'answer', { agentName: 'Codex' })).toBe( + 'Codex has stopped, so your answer was not sent. Send a message to continue.' + ) + }) + + it.each(['providerStartFailed', 'notSignedIn', 'cliMissing', 'historyTooLarge'] as const)( + 'keeps that sentence its own: %s still reads "Your answer was not sent."', + (reason) => { + const refusal = { + code: 'agent_session_operation_invalid' as const, + message: HOST_TEXT, + details: { reason } + } + expect(agentSessionRefusalNotice(refusal, 'answer', { agentName: 'Codex' })).toBe( + 'Your answer was not sent.' + ) + } + ) +}) + describe('a refusal from a host that names no reason this build knows', () => { it.each([ ['an older host', undefined], diff --git a/src/shared/agent-session-refusal-notice.ts b/src/shared/agent-session-refusal-notice.ts index f81e1b51a901..dfaaca943741 100644 --- a/src/shared/agent-session-refusal-notice.ts +++ b/src/shared/agent-session-refusal-notice.ts @@ -125,6 +125,11 @@ function reasonParts( return undefined } if ('fact' in words) { + if (write === 'answer' && words.fact === 'providerExited') { + // An approval or question was waiting on the person when its agent stopped: one sentence + // that says the answer was not sent, names the agent, and says how to go on. + return [{ failure: { kind: words.fact }, surface: 'answer', context }] + } return write === 'send' || write === 'composer-send' ? [ NOT_DONE[write], @@ -252,10 +257,11 @@ export function agentSessionWriteNoticeEnglish( * message is not read. */ export function agentSessionRefusalNotice( refusal: Pick, - write: AgentSessionWriteKind + write: AgentSessionWriteKind, + context: AgentSessionFailureWordsContext = {} ): string { return agentSessionWriteNoticeEnglish( - agentSessionWriteNoticeParts(agentSessionRefusalFailure(refusal), write) + agentSessionWriteNoticeParts(agentSessionRefusalFailure(refusal), write, context) ) } diff --git a/src/shared/agent-session-refusal-reason-words.ts b/src/shared/agent-session-refusal-reason-words.ts index 03c5c1c04dd2..76b9d4ff7303 100644 --- a/src/shared/agent-session-refusal-reason-words.ts +++ b/src/shared/agent-session-refusal-reason-words.ts @@ -88,6 +88,9 @@ const REASON_WORDS = { rewindRefused: codeWords('hostFinding'), rewindUnconfirmed: codeWords('hostFinding'), promptGone: causeWords('questionChanged', 'nothingLeft'), + // An approval or question waited on the person when its agent stopped: said naming the agent, + // with sending a message as the step on. + promptOwnerEnded: { fact: 'providerExited', action: 'nothingLeft' }, optionRejected: causeWords('optionRejected', 'retry'), providerStarting: AGENT_STARTING, goalsUnsupported: causeWords('goalsUnsupported', 'hostFinding'), diff --git a/src/shared/agent-session-rewind.ts b/src/shared/agent-session-rewind.ts index e7d864390cca..36e30f9b79a4 100644 --- a/src/shared/agent-session-rewind.ts +++ b/src/shared/agent-session-rewind.ts @@ -55,6 +55,9 @@ export const AgentSessionRewindRecordSchema = z.object({ observedAt: z.number().finite(), // Absent on records written before rows stated them: the rebuild then places the row by position. turnScope: AgentJournalTurnScopeSchema.optional(), + // The row's execution provenance, carried into the rebuilt epoch. Absent on records written + // before rows carried it. + ownerFence: z.number().int().nonnegative().optional(), ...AgentJournalProducerLinkageFields }) ) diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index b8ff6a140556..a90303343850 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -148,6 +148,16 @@ export type AgentSessionHistoryPage = { subagentRoster?: AgentSessionSubagentRosterEntry[] /** As of the page's read; a client applies it only from a page that replaces its state. */ latestTurn?: AgentSessionLatestTurn | null + /** Rides with `latestTurn`: the approvals and questions still waiting on the person that the + * agent running now raised, over the whole journal. One an agent that has ended raised is not + * listed, since nothing can take its answer. Absent from an older host, whose clients read every + * pending prompt they hold as waiting. */ + actionablePromptIds?: string[] + /** Rides with `latestTurn`: whether the agent running now is working, by the host's projection of + * current work — its running turn, or a send it owes, eligible or handed over and unanswered. + * What an agent that has ended left counts for nothing. Absent from an older host, whose + * clients derive it from the rows they hold. */ + working?: boolean } export type AgentSessionHistoryResult = @@ -207,6 +217,10 @@ export type AgentSessionSubscribeEvent = /** Rides every batch that carries rows, removals or submissions, so absent there means an * older host; absent on one that carries none, which changes no turn. */ latestTurn?: AgentSessionLatestTurn | null + /** Rides with `latestTurn` (`AgentSessionHistoryPage.actionablePromptIds`). */ + actionablePromptIds?: string[] + /** Rides with `latestTurn` (`AgentSessionHistoryPage.working`). */ + working?: boolean } & AgentSessionFrameFields) | ({ type: 'reset' @@ -459,9 +473,7 @@ export type AgentSessionSlashCommand = { /** The provider's own command surface, read per session. Additive read-only * surface: a host that predates it answers `method_not_found`, and the client * keeps rendering its curated catalog. */ -export type AgentSessionCommandsResult = { - commands?: AgentSessionSlashCommand[] -} +export type AgentSessionCommandsResult = { commands?: AgentSessionSlashCommand[] } /** Longest objective a client may send; matches the provider's own limit. */ export const AGENT_SESSION_THREAD_GOAL_OBJECTIVE_MAX_LENGTH = 4000 @@ -473,9 +485,7 @@ export type AgentSessionThreadGoalChange = | { kind: 'status'; status: 'active' | 'paused' } | { kind: 'clear' } -export type AgentSessionThreadGoalResult = { - change: AgentSessionThreadGoalChange['kind'] -} +export type AgentSessionThreadGoalResult = { change: AgentSessionThreadGoalChange['kind'] } /** Provider-reported choices and effective next-turn values. Additive read-only * surface so older hosts can reject it without changing structured v1 writes. */ diff --git a/src/shared/native-chat-messages-waiting-behind-live-turn.test.ts b/src/shared/native-chat-messages-waiting-behind-live-turn.test.ts new file mode 100644 index 000000000000..91fecb6b5195 --- /dev/null +++ b/src/shared/native-chat-messages-waiting-behind-live-turn.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from './agent-session-journal-types' +import { nativeChatMessagesWaitingBehindLiveTurn } from './native-chat-messages-waiting-behind-live-turn' + +/** A /compact whose turn record still reads running. */ +const compactRunning: AgentJournalRenderItem[] = [ + { + itemId: 'compact-entry', + revision: 0, + sequence: 1, + observedAt: 1, + body: { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: '/compact' }], + command: { name: 'compact' } + } + }, + { + itemId: 'compact-turn', + revision: 0, + sequence: 2, + observedAt: 2, + body: { kind: 'turn', turnId: 'compact-turn', userItemId: 'compact-entry', state: 'running' } + } +] +const queued = [{ id: 'card-1', role: 'user' as const, queued: true as const }] + +describe('a queued card behind a running /compact', () => { + it('waits behind it while the host says it runs, or an older host says nothing', () => { + expect(nativeChatMessagesWaitingBehindLiveTurn(queued, compactRunning)).toEqual( + new Set(['card-1']) + ) + const latestTurn = { + itemId: 'compact-turn', + observedAt: 2, + turn: { turnId: 'compact-turn', state: 'running' as const } + } + expect( + nativeChatMessagesWaitingBehindLiveTurn(queued, compactRunning, false, [], { latestTurn }) + ).toEqual(new Set(['card-1'])) + }) + + it('waits behind nothing once the host says no turn runs: the agent that ran it ended', () => { + expect( + nativeChatMessagesWaitingBehindLiveTurn(queued, compactRunning, false, [], { + latestTurn: null, + working: false + }) + ).toEqual(new Set()) + }) +}) diff --git a/src/shared/native-chat-messages-waiting-behind-live-turn.ts b/src/shared/native-chat-messages-waiting-behind-live-turn.ts index 767bb1e8c826..6c42a8029dbc 100644 --- a/src/shared/native-chat-messages-waiting-behind-live-turn.ts +++ b/src/shared/native-chat-messages-waiting-behind-live-turn.ts @@ -1,10 +1,11 @@ // Which rows wait at the tail, after the live turn's activity line, rather than in the transcript. import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' +import type { AgentSessionLatestTurn } from './agent-session-wire' import { readAgentJournalTurn } from './agent-session-turn-record' import type { NativeChatRole } from './native-chat-types' import { isStructuredAgentSessionCommandTurn } from './structured-agent-session-command-entry' -import { liveStructuredAgentSessionTurnScope } from './structured-agent-session-live-turn' +import { runningStructuredAgentSessionTurnScope } from './structured-agent-session-live-turn' import { structuredAgentSessionOpeningSendIn } from './structured-agent-session-opening-send' /** @@ -30,7 +31,10 @@ export function nativeChatMessagesWaitingBehindLiveTurn( }[], items: readonly AgentJournalRenderItem[] | null | undefined, stopping = false, - submissions?: readonly AgentJournalSubmission[] + submissions?: readonly AgentJournalSubmission[], + /** The host's view of current work (`latestTurn`, `working`): an ended generation's running turn + * or unanswered send holds nothing. Absent from an older host, whose rows are read instead. */ + host: { latestTurn?: AgentSessionLatestTurn | null; working?: boolean } = {} ): ReadonlySet { const waits = (message: (typeof messages)[number]): boolean => message.queued === true || (stopping && message.sentWhileStopping === true) @@ -45,9 +49,11 @@ export function nativeChatMessagesWaitingBehindLiveTurn( if (candidates.length === 0 || !items) { return new Set() } - const behindHold = stopping || commandTurnRunning(items) + const behindHold = stopping || commandTurnRunning(items, host.latestTurn) const behindOpening = - submissions !== undefined && structuredAgentSessionOpeningSendIn(items, submissions) !== null + host.working !== false && + submissions !== undefined && + structuredAgentSessionOpeningSendIn(items, submissions) !== null return new Set( candidates .filter( @@ -59,8 +65,14 @@ export function nativeChatMessagesWaitingBehindLiveTurn( ) } -function commandTurnRunning(items: readonly AgentJournalRenderItem[]): boolean { - const running = liveStructuredAgentSessionTurnScope(items) +function commandTurnRunning( + items: readonly AgentJournalRenderItem[], + latestTurn: AgentSessionLatestTurn | null | undefined +): boolean { + const running = runningStructuredAgentSessionTurnScope({ + items, + ...(latestTurn !== undefined ? { latestTurn } : {}) + }) const bodyOf = (itemId: string) => items.find((item) => item.itemId === itemId)?.body return ( running.kind === 'turn' && diff --git a/src/shared/native-chat-turn-membership.ts b/src/shared/native-chat-turn-membership.ts index 40dbf096e2ef..43c28719dda2 100644 --- a/src/shared/native-chat-turn-membership.ts +++ b/src/shared/native-chat-turn-membership.ts @@ -46,6 +46,8 @@ export type NativeChatTurnJournal = { * record is not loaded, so the turn's loaded rows still draw under its bar. Absent from older * hosts. */ latestTurn?: AgentSessionLatestTurn | null + /** The host's answer to whether the session's own agent is working; absent from older hosts. */ + working?: boolean } /** diff --git a/src/shared/native-chat-types.ts b/src/shared/native-chat-types.ts index 28e9f86f9f3d..c43191e3d171 100644 --- a/src/shared/native-chat-types.ts +++ b/src/shared/native-chat-types.ts @@ -151,6 +151,9 @@ export type NativeChatSubagentEntry = { startedAt?: number /** Epoch ms the entry latched terminal. */ settledAt?: number + /** The agent generation (lease fence) whose execution this entry last observed: stamped by the + * journal on a fresh observation, kept when carried unchanged. Absent: the row's own. */ + ownerFence?: number } /** One spawn group's roster, revised in place as its children report activity. @@ -195,6 +198,8 @@ export type NativeChatBackgroundTaskBlock = { startedAt?: number /** Epoch ms the row latched terminal. */ settledAt?: number + /** As `NativeChatSubagentEntry.ownerFence`. */ + ownerFence?: number } export type NativeChatBlock = diff --git a/src/shared/structured-agent-session-coalescer.ts b/src/shared/structured-agent-session-coalescer.ts index f80658838dc1..fc418c8bf5e6 100644 --- a/src/shared/structured-agent-session-coalescer.ts +++ b/src/shared/structured-agent-session-coalescer.ts @@ -1,5 +1,9 @@ import type { AgentSessionSubscribeEvent } from './agent-session-wire' -import { latestTurnAfterStructuredAgentSessionBatch } from './structured-agent-session-live-turn' +import { + actionablePromptIdsAfterStructuredAgentSessionBatch, + hostWorkingAfterStructuredAgentSessionBatch, + latestTurnAfterStructuredAgentSessionBatch +} from './structured-agent-session-live-turn' export const STRUCTURED_AGENT_SESSION_CLIENT_COALESCE_MS = 48 @@ -29,6 +33,11 @@ function mergeBatch( } // As applying both in turn would leave it, so an older host's rows still drop a stale claim. const latestTurn = latestTurnAfterStructuredAgentSessionBatch(left.latestTurn, right) + const actionablePromptIds = actionablePromptIdsAfterStructuredAgentSessionBatch( + left.actionablePromptIds, + right + ) + const working = hostWorkingAfterStructuredAgentSessionBatch(left.working, right) return { type: 'batch', ...(right.commands !== undefined || left.commands !== undefined @@ -55,7 +64,9 @@ function mergeBatch( ...(right.activity !== undefined || left.activity !== undefined ? { activity: right.activity !== undefined ? right.activity : (left.activity ?? null) } : {}), - ...(latestTurn !== undefined ? { latestTurn } : {}) + ...(latestTurn !== undefined ? { latestTurn } : {}), + ...(actionablePromptIds !== undefined ? { actionablePromptIds } : {}), + ...(working !== undefined ? { working } : {}) } } diff --git a/src/shared/structured-agent-session-live-turn.ts b/src/shared/structured-agent-session-live-turn.ts index a716b8f4a131..0316defad8f2 100644 --- a/src/shared/structured-agent-session-live-turn.ts +++ b/src/shared/structured-agent-session-live-turn.ts @@ -35,16 +35,20 @@ import { type StructuredAgentSessionToolAction } from './structured-agent-session-tool-call-block' +/** `current`: a host's answer to whether the running turn's item is current work + * (`StructuredAgentSessionWorkScope`); one an ended generation opened is none. */ export function activeStructuredAgentSessionTurnId( - items: readonly AgentJournalRenderItem[] + items: readonly AgentJournalRenderItem[], + current?: (item: AgentJournalRenderItem) => boolean ): string | null { for (let index = items.length - 1; index >= 0; index -= 1) { - if (isAgentSessionContextClear(items[index]?.body)) { + const item = items[index] + if (isAgentSessionContextClear(item?.body)) { return null } - const turn = readAgentJournalTurn(items[index]?.body) + const turn = readAgentJournalTurn(item?.body) if (turn) { - return turn.state === 'running' ? turn.turnId : null + return turn.state === 'running' && (!current || (item && current(item))) ? turn.turnId : null } } return null @@ -81,6 +85,14 @@ export function newestStructuredAgentSessionTurnBySequence( export function liveStructuredAgentSessionTurnScope( items: Iterable ): AgentJournalTurnScope { + const running = runningStructuredAgentSessionTurnItemBySequence(items) + return running ? { kind: 'turn', turnItemId: running.item.itemId } : AGENT_JOURNAL_THREAD_SCOPE +} + +/** The newest turn record's item when that turn is running, by sequence. */ +export function runningStructuredAgentSessionTurnItemBySequence( + items: Iterable +): { item: AgentJournalRenderItem; turnId: string } | null { let newest: AgentJournalRenderItem | null = null for (const item of items) { if ( @@ -90,17 +102,8 @@ export function liveStructuredAgentSessionTurnScope( newest = item } } - return newest && readAgentJournalTurn(newest.body)?.state === 'running' - ? { kind: 'turn', turnItemId: newest.itemId } - : AGENT_JOURNAL_THREAD_SCOPE -} - -/** Whether that newest turn is still running, which is all most callers want. */ -export function activeStructuredAgentSessionTurnIdBySequence( - items: Iterable -): string | null { - const newest = newestStructuredAgentSessionTurnBySequence(items) - return newest?.state === 'running' ? newest.turnId : null + const turn = newest ? readAgentJournalTurn(newest.body) : null + return newest && turn?.state === 'running' ? { item: newest, turnId: turn.turnId } : null } /** The newest turn record whatever state it ended in, STATE INCLUDED. Restart resume compares both @@ -172,6 +175,70 @@ export function latestTurnAfterStructuredAgentSessionBatch( return carriesRows || event.latestTurn !== undefined ? event.latestTurn : previous } +/** The host's actionable prompts once `event` applies, under `latestTurn`'s rule: a batch carrying + * rows restates them, so one without them came from an older host and drops the claim. */ +export function actionablePromptIdsAfterStructuredAgentSessionBatch( + previous: string[] | undefined, + event: Extract +): string[] | undefined { + const { items, removedItemIds, submissions } = event.batch + const carriesRows = items.length > 0 || removedItemIds.length > 0 || submissions.length > 0 + return carriesRows || event.actionablePromptIds !== undefined + ? event.actionablePromptIds + : previous +} + +/** The host's working answer once `event` applies, under `latestTurn`'s rule. */ +export function hostWorkingAfterStructuredAgentSessionBatch( + previous: boolean | undefined, + event: Extract +): boolean | undefined { + const { items, removedItemIds, submissions } = event.batch + const carriesRows = items.length > 0 || removedItemIds.length > 0 || submissions.length > 0 + return carriesRows || event.working !== undefined ? event.working : previous +} + +/** Whether a frame's turn, prompts and working answer, when it names them, are what the state + * already holds: a generation's end can change them with no row, so a frame that does is never a + * no-op. */ +export function restatesStructuredAgentSessionHostWork( + event: Pick< + Extract, + 'latestTurn' | 'actionablePromptIds' | 'working' + >, + state: { + latestTurn?: AgentSessionLatestTurn | null + actionablePromptIds?: string[] + working?: boolean + } +): boolean { + return ( + (event.latestTurn === undefined || + JSON.stringify(event.latestTurn) === JSON.stringify(state.latestTurn)) && + (event.actionablePromptIds === undefined || + event.actionablePromptIds.join('\n') === state.actionablePromptIds?.join('\n')) && + (event.working === undefined || event.working === state.working) + ) +} + +/** Whether the session's own agent is working: the host's answer when it gave one, else what the + * client derives from its rows (an older host). */ +export function structuredAgentSessionHostSaysWorking( + hostWorking: boolean | undefined, + derived: () => boolean +): boolean { + return hostWorking ?? derived() +} + +/** Whether a pending prompt waits on the person: the host names the ones that do, and a prompt an + * agent that has ended raised is not among them. An older host names none: every pending one. */ +export function isActionableStructuredAgentSessionPrompt( + itemId: string, + actionablePromptIds: readonly string[] | undefined +): boolean { + return actionablePromptIds === undefined || actionablePromptIds.includes(itemId) +} + /** * Whether the newest thing the active turn produced is the model's own reasoning. * diff --git a/src/shared/structured-agent-session-main-agent-working.ts b/src/shared/structured-agent-session-main-agent-working.ts index 05e3b775aaa9..ea8af7771282 100644 --- a/src/shared/structured-agent-session-main-agent-working.ts +++ b/src/shared/structured-agent-session-main-agent-working.ts @@ -1,14 +1,30 @@ import type { AgentJournalSubmission } from './agent-session-journal-types' import { hasUnansweredStructuredAgentSessionDispatch } from './structured-agent-session-unanswered-dispatch' +/** The host's answer to which work is current, built only by its operational projection + * (`structuredAgentSessionCurrentWork`). A client has none, so every running turn, pending prompt + * and unanswered send it holds reads as current, as it always has. */ +export type StructuredAgentSessionWorkScope = { + /** Whether the live generation's execution produced this item. */ + isCurrentItem: (itemId: string) => boolean + /** Whether this send is still owed: handed to the live generation and unanswered, or accepted by + * this host and not yet handed over. */ + owesSend: (submission: AgentJournalSubmission) => boolean +} + /** Whether the session's own agent is working: a running turn, or a send it has not answered. - * The one rule behind every session list's Working and the chat's own Stop. */ + * The one rule behind every session list's Working and the chat's own Stop. A host passes the turn + * its projection scoped, and that projection's `scope`. */ export function isStructuredAgentSessionMainAgentWorking( activeTurnId: string | null, submissions: readonly AgentJournalSubmission[], - currentFence?: number | null + currentFence?: number | null, + scope?: StructuredAgentSessionWorkScope ): boolean { return ( - activeTurnId !== null || hasUnansweredStructuredAgentSessionDispatch(submissions, currentFence) + activeTurnId !== null || + (scope + ? submissions.some(scope.owesSend) + : hasUnansweredStructuredAgentSessionDispatch(submissions, currentFence)) ) } diff --git a/src/shared/structured-agent-session-owed-work.ts b/src/shared/structured-agent-session-owed-work.ts index fe7c9695ae73..259bbfe6877c 100644 --- a/src/shared/structured-agent-session-owed-work.ts +++ b/src/shared/structured-agent-session-owed-work.ts @@ -1,6 +1,9 @@ import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' import { activeStructuredAgentSessionTurnId } from './structured-agent-session-live-turn' -import { isStructuredAgentSessionMainAgentWorking } from './structured-agent-session-main-agent-working' +import { + isStructuredAgentSessionMainAgentWorking, + type StructuredAgentSessionWorkScope +} from './structured-agent-session-main-agent-working' import { agentSessionCurrentContextRows } from './agent-session-context-clear' /** A running turn or an unanswered send: what a `working` status means, and what an `attention` @@ -8,12 +11,17 @@ import { agentSessionCurrentContextRows } from './agent-session-context-clear' export function owesStructuredAgentSessionWork( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[], - currentFence?: number | null + currentFence?: number | null, + scope?: StructuredAgentSessionWorkScope ): boolean { const current = agentSessionCurrentContextRows(items, submissions) return isStructuredAgentSessionMainAgentWorking( - activeStructuredAgentSessionTurnId(current.items), + activeStructuredAgentSessionTurnId( + current.items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ), current.submissions, - currentFence + currentFence, + scope ) } diff --git a/src/shared/structured-agent-session-projection.ts b/src/shared/structured-agent-session-projection.ts index 610b08e91670..6b66904c2759 100644 --- a/src/shared/structured-agent-session-projection.ts +++ b/src/shared/structured-agent-session-projection.ts @@ -39,6 +39,7 @@ import { sha256 } from './sha256' import { readAgentMessageSource } from './agent-session-message-source' import { structuredAgentSessionStatusStartedAt } from './structured-agent-session-status-started-at' import { owesStructuredAgentSessionWork } from './structured-agent-session-owed-work' +import type { StructuredAgentSessionWorkScope } from './structured-agent-session-main-agent-working' import { agentSessionCurrentContextRows } from './agent-session-context-clear' // Re-exported so the live-turn readers' and the unanswered-send rule's existing consumers keep one @@ -214,23 +215,32 @@ export function structuredAgentSessionTabId(sessionId: string): string { return `structured-agent-session-${sessionId}` } -function isPendingStructuredAgentSessionPrompt(item: AgentJournalRenderItem): boolean { - return ( +/** A prompt waiting on the user, unless the host's scope says an ended generation raised it: + * nothing can take its answer. */ +function pendingStructuredAgentSessionPrompt( + scope?: StructuredAgentSessionWorkScope +): (item: AgentJournalRenderItem) => boolean { + return (item) => (item.body.kind === 'approval' || item.body.kind === 'question') && - item.body.resolution.state === 'pending' - ) + item.body.resolution.state === 'pending' && + (!scope || scope.isCurrentItem(item.itemId)) } +/** `scope`: a host's (`StructuredAgentSessionWorkScope`), so work an ended generation left neither + * holds nor reads as Working. */ export function projectStructuredAgentSessionStatus( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[] = [], - currentFence?: number | null + currentFence?: number | null, + scope?: StructuredAgentSessionWorkScope ): StructuredAgentSessionProjectedStatus { ;({ items, submissions } = agentSessionCurrentContextRows(items, submissions)) - if (items.some(isPendingStructuredAgentSessionPrompt)) { + if (items.some(pendingStructuredAgentSessionPrompt(scope))) { return 'attention' } - return owesStructuredAgentSessionWork(items, submissions, currentFence) ? 'working' : 'idle' + return owesStructuredAgentSessionWork(items, submissions, currentFence, scope) + ? 'working' + : 'idle' } /** The activity fields a sidebar row shows beside the prompt, named as the agent-status @@ -269,7 +279,8 @@ export function projectStructuredAgentSessionStatusSummary( export function projectStructuredAgentSessionStatusState( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[] = [], - currentFence?: number | null + currentFence?: number | null, + scope?: StructuredAgentSessionWorkScope ): { summary: StructuredAgentSessionStatusProjection latestRequest: StructuredAgentSessionLatestRequest | null @@ -288,7 +299,7 @@ export function projectStructuredAgentSessionStatusState( } } ;({ items, submissions } = agentSessionCurrentContextRows(items, submissions)) - const status = projectStructuredAgentSessionStatus(items, submissions, currentFence) + const status = projectStructuredAgentSessionStatus(items, submissions, currentFence, scope) const statusToolCall = status === 'working' ? statusStructuredAgentSessionToolCall(items) : null const toolName = statusToolCall ? normalizeOptionalField(statusToolCall.name, AGENT_STATUS_TOOL_NAME_MAX_LENGTH) @@ -318,10 +329,11 @@ export function projectStructuredAgentSessionStatusState( ) return { latestRequest, - owesWork: status !== 'idle' && owesStructuredAgentSessionWork(items, submissions, currentFence), + owesWork: + status !== 'idle' && owesStructuredAgentSessionWork(items, submissions, currentFence, scope), pendingPromptIds: status === 'attention' - ? items.filter(isPendingStructuredAgentSessionPrompt).map((item) => item.itemId) + ? items.filter(pendingStructuredAgentSessionPrompt(scope)).map((item) => item.itemId) : [], summary: { status, diff --git a/src/shared/structured-agent-session-reducer.test.ts b/src/shared/structured-agent-session-reducer.test.ts index 1592e949b1b2..1838740ae486 100644 --- a/src/shared/structured-agent-session-reducer.test.ts +++ b/src/shared/structured-agent-session-reducer.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from 'vitest' import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' import type { AgentSessionHistoryPage, AgentSessionLatestTurn } from './agent-session-wire' -import { runningStructuredAgentSessionTurnId } from './structured-agent-session-live-turn' +import { + isActionableStructuredAgentSessionPrompt, + runningStructuredAgentSessionTurnId +} from './structured-agent-session-live-turn' import { EMPTY_STRUCTURED_AGENT_SESSION, reduceStructuredAgentSession @@ -595,6 +598,75 @@ describe('structured agent session reducer', () => { }) }) + describe("the host's actionable prompts", () => { + const prompt = (itemId: string, sequence: number): AgentJournalRenderItem => ({ + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } + }) + const PROMPTS = [prompt('live-prompt', 1), prompt('dead-prompt', 2)] + const opened = (page: AgentSessionHistoryPage) => + reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'event', + event: { type: 'snapshot', sessionId: 'session-a', fence: 1, page } + }) + const batch = ( + state: ReturnType, + items: AgentJournalRenderItem[], + actionablePromptIds?: string[] + ) => + reduceStructuredAgentSession(state, { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + cursor: { epoch: 'epoch-a', sequence: state.cursor!.sequence + items.length }, + items, + removedItemIds: [], + submissions: [] + }, + ...(actionablePromptIds !== undefined ? { actionablePromptIds } : {}) + } + }) + const actionable = (state: ReturnType) => + state.items + .filter((entry) => + isActionableStructuredAgentSessionPrompt(entry.itemId, state.actionablePromptIds) + ) + .map((entry) => entry.itemId) + + it('reads them from the snapshot, keeps them across a frame without rows, and takes the next', () => { + const state = opened({ ...hydrationPage(PROMPTS), actionablePromptIds: ['live-prompt'] }) + expect(actionable(state)).toEqual(['live-prompt']) + const quiet = batch(state, []) + expect(quiet.actionablePromptIds).toEqual(['live-prompt']) + // The generation ended with no row written: the host's frame names none. + const ended = batch(quiet, [], []) + expect(actionable(ended)).toEqual([]) + }) + + it('an older host names none: its page reduces, and every pending prompt stays answerable', () => { + // As an older host serialized it: no such field at all. + const older: AgentSessionHistoryPage = JSON.parse(JSON.stringify(hydrationPage(PROMPTS))) + const state = opened(older) + expect(state.actionablePromptIds).toBeUndefined() + expect(actionable(state)).toEqual(['live-prompt', 'dead-prompt']) + // Rows from a downgraded host restate nothing: back to every pending prompt. + const named = opened({ ...hydrationPage(PROMPTS), actionablePromptIds: ['live-prompt'] }) + const downgraded = batch(named, [prompt('another', 3)]) + expect(downgraded.actionablePromptIds).toBeUndefined() + }) + }) + it('records the host clock from frames that carry it and keeps it otherwise', () => { const snapshot = reduceStructuredAgentSession( EMPTY_STRUCTURED_AGENT_SESSION, diff --git a/src/shared/structured-agent-session-reducer.ts b/src/shared/structured-agent-session-reducer.ts index 6ff6124feb38..f20c673e7969 100644 --- a/src/shared/structured-agent-session-reducer.ts +++ b/src/shared/structured-agent-session-reducer.ts @@ -26,7 +26,12 @@ import { import { compareAgentJournalItems } from './agent-session-journal-position' import { readAgentJournalTurn } from './agent-session-turn-record' import { queuePublicationField } from './structured-agent-session-queue-publication-field' -import { latestTurnAfterStructuredAgentSessionBatch } from './structured-agent-session-live-turn' +import { + actionablePromptIdsAfterStructuredAgentSessionBatch, + hostWorkingAfterStructuredAgentSessionBatch, + latestTurnAfterStructuredAgentSessionBatch, + restatesStructuredAgentSessionHostWork +} from './structured-agent-session-live-turn' import { foldStructuredAgentSubagentRoster, foldStructuredAgentSubagentRosterPage, @@ -75,6 +80,12 @@ export type StructuredAgentSessionState = { /** The host's newest turn record over the whole journal, which says whether a turn runs; absent * from an older host, whose answer is read off `items` instead. */ latestTurn?: AgentSessionLatestTurn | null + /** The prompts the host says still wait on the person (`AgentSessionHistoryPage`); absent from + * an older host, whose every pending prompt in `items` is read as waiting. */ + actionablePromptIds?: string[] + /** Whether the host says the session's own agent is working (`AgentSessionHistoryPage.working`); + * absent from an older host, whose answer is derived from `items` and `submissions`. */ + working?: boolean /** Bumped per live batch that leaves a turn row's newest revision outside the window * (dropped or trimmed), so a whole-journal answer derived from turn rows is asked for again. */ unloadedTurnRevisions?: number @@ -130,6 +141,10 @@ function replacePage( subagentRoster: foldStructuredAgentSubagentRosterPage(undefined, page), activity: activity ?? null, ...(page.latestTurn !== undefined ? { latestTurn: page.latestTurn } : {}), + ...(page.actionablePromptIds !== undefined + ? { actionablePromptIds: page.actionablePromptIds } + : {}), + ...(page.working !== undefined ? { working: page.working } : {}), ...(backgroundTasks !== undefined ? { backgroundTasks } : page.backgroundTasks !== undefined @@ -275,6 +290,7 @@ export function reduceStructuredAgentSession( (event.nextQueuedMessageId === undefined || event.nextQueuedMessageId === state.nextQueuedMessageId) && backgroundTaskStatesEqual(backgroundTasks, state.backgroundTasks) && + restatesStructuredAgentSessionHostWork(event, state) && activity?.turnId === state.activity?.turnId && activity?.text === state.activity?.text && state.status === 'ready' && @@ -310,6 +326,11 @@ export function reduceStructuredAgentSession( readRefusal: undefined, commands: event.commands !== undefined ? event.commands : state.commands, latestTurn: latestTurnAfterStructuredAgentSessionBatch(state.latestTurn, event), + actionablePromptIds: actionablePromptIdsAfterStructuredAgentSessionBatch( + state.actionablePromptIds, + event + ), + working: hostWorkingAfterStructuredAgentSessionBatch(state.working, event), ...queuePublicationField(event, state), backgroundTasks, ...(activity !== undefined ? { activity } : {}),