From 20eb8df78f0f6b812152dd9a14ac6dc05c8eb5c5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:38:16 -0700 Subject: [PATCH 01/22] refactor(native-chat): settle a dead agent's leftover work at its ownership events, and open chats read-only Opening a chat no longer writes: the leftover-work settlement, the subagent roster correction, the pending-send marks and the queued-message repair move to the moments that prove an owner gone (its observed exit, startup, a new owner). One settlement plans every category in one transaction. --- .../scripts/vitest-sqlite-runtime-files.mjs | 2 + ...tructured-journal-exit-convergence.test.ts | 17 +- .../journal-epoch-controller.ts | 13 +- .../journal-epoch-founding.ts | 55 +++ .../journal-host-database-test-support.ts | 27 +- .../journal-lifecycle-batch-appender.ts | 31 +- .../journal-row-writer.ts | 31 +- .../journal-store-collaborators.ts | 17 +- .../journal-store-contracts.ts | 15 + .../journal-store-open.ts | 35 +- .../journal-store-restore.ts | 25 +- .../agent-session-journal/journal-store.ts | 16 +- .../journal-subagent-liveness.test.ts | 58 +-- .../journal-subagent-liveness.ts | 41 +- ...queued-message-bookkeeping-failure.test.ts | 5 +- .../queued-message-store.test.ts | 51 ++- .../queued-message-table.ts | 8 + .../agent-session-record-test-fixture.ts | 3 +- ...-agent-session-accept-then-deliver.test.ts | 10 +- ...ed-agent-session-acquisition-settlement.ts | 72 ++++ ...structured-agent-session-attach-context.ts | 6 +- ...ured-agent-session-attach-orchestration.ts | 30 +- ...gent-session-attach-reconciliation.test.ts | 15 +- ...-agent-session-attach-test-conversation.ts | 12 +- .../structured-agent-session-chat-stop.ts | 2 + .../structured-agent-session-child-exit.ts | 44 ++- ...uctured-agent-session-conversation-open.ts | 115 +----- ...tured-agent-session-crash-turn-end.test.ts | 101 +++-- ...session-dead-generation-settlement.test.ts | 8 +- ...gent-session-dead-generation-settlement.ts | 264 +++++-------- ...ctured-agent-session-forget-status.test.ts | 3 +- .../structured-agent-session-host-delivery.ts | 72 +--- .../structured-agent-session-host-lifetime.ts | 1 + ...d-agent-session-host-runtime-state.test.ts | 3 +- ...tructured-agent-session-host-stop-event.ts | 16 +- .../structured-agent-session-host.ts | 5 +- ...-session-leftover-settlement-scope.test.ts | 176 +++++++++ ...ssion-leftover-settlement-triggers.test.ts | 369 ++++++++++++++++++ ...ession-leftover-settlement.test-fixture.ts | 278 +++++++++++++ ...tured-agent-session-leftover-settlement.ts | 125 ++++++ .../structured-agent-session-orca-stop-row.ts | 5 +- ...gent-session-provider-child-record.test.ts | 7 +- ...ed-agent-session-queued-card-holds.test.ts | 4 +- ...tured-agent-session-queued-compact.test.ts | 4 +- ...session-queued-message-rig.test-fixture.ts | 64 ++- ...tructured-agent-session-queued-messages.ts | 30 +- ...ed-agent-session-queued-newer-orca.test.ts | 14 +- ...ed-agent-session-queued-pause-lift.test.ts | 4 +- .../structured-agent-session-queued-pause.ts | 4 +- ...ured-agent-session-queued-stop-row.test.ts | 15 +- .../structured-agent-session-queued-stop.ts | 2 +- ...ssion-restart-interruption-test-harness.ts | 3 + ...t-session-restart-offer-newer-orca.test.ts | 7 +- ...tructured-agent-session-restart-restore.ts | 21 +- .../structured-agent-session-reveal.ts | 34 +- ...agent-session-send-open-stale-turn.test.ts | 22 +- ...-session-settlement-double.test-fixture.ts | 65 +++ .../structured-agent-session-shown-work.ts | 12 +- ...ed-agent-session-stale-state-settlement.ts | 171 ++++++++ ...ured-agent-session-stale-turn-open.test.ts | 36 +- ...d-agent-session-stale-turn-verdict.test.ts | 6 +- ...agent-session-startup-failure-exit.test.ts | 51 +-- ...ctured-agent-session-startup-settlement.ts | 177 +++++++++ ...ructured-agent-session-status-feed.test.ts | 8 +- ...agent-session-status-journal-projection.ts | 4 +- ...d-agent-session-stop-event-binding.test.ts | 2 +- ...d-agent-session-stop-event-restart.test.ts | 2 +- ...-agent-session-stop-settle-binding.test.ts | 6 +- ...structured-agent-session-stop-wind-down.ts | 4 +- .../structured-agent-session-stopping.test.ts | 49 +-- .../structured-agent-session-stopping.ts | 5 +- .../structured-agent-session-turns-cancel.ts | 15 +- .../structured-agent-session-turns-prompt.ts | 5 + ...ured-agent-session-unexpected-exit.test.ts | 63 +-- ...tructured-agent-session-unfinished-work.ts | 2 +- ...-agent-session-unopened-send-withdrawal.ts | 2 +- ...ured-agent-session-unrun-send-hold.test.ts | 2 +- ...red-agent-session-unsent-send-hold.test.ts | 35 +- ...t-session-wedged-profile-migration.test.ts | 7 +- ...tured-agent-session-working-at-teardown.ts | 3 +- .../agent-session-conversation-receipts.ts | 11 +- .../agent-session-lease-transitions.ts | 22 +- .../runtime/agent-session-record-founding.ts | 4 +- .../agent-session-reservation-admission.ts | 4 +- ...-agent-session-startup-tab-restore.test.ts | 2 + .../structured-worker-identity.test.ts | 3 +- .../native-chat/NativeChatSubagentRun.tsx | 4 +- .../agent-session-lease-adjudication.test.ts | 1 + .../agent-session-legacy-handoff-lease.ts | 11 +- .../agent-session-record.test-fixture.ts | 1 + src/shared/agent-session-record.ts | 6 +- .../structured-agent-session-live-turn.ts | 45 ++- ...ctured-agent-session-main-agent-working.ts | 17 +- .../structured-agent-session-owed-work.ts | 16 +- .../structured-agent-session-projection.ts | 41 +- 95 files changed, 2481 insertions(+), 856 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-epoch-founding.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 85a6f048eb5b..2a237b0f13ea 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -108,6 +108,8 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-lease-renewer.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-main-agent-working-agreement.test.ts', diff --git a/src/main/codex/codex-structured-journal-exit-convergence.test.ts b/src/main/codex/codex-structured-journal-exit-convergence.test.ts index c9f56af83048..60c0eacd75ad 100644 --- a/src/main/codex/codex-structured-journal-exit-convergence.test.ts +++ b/src/main/codex/codex-structured-journal-exit-convergence.test.ts @@ -24,6 +24,7 @@ import { } from '../native-chat/agent-session-wire/structured-agent-session-child-exit' import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import { pendingPromptExists } from '../native-chat/agent-session-wire/structured-agent-session-queued-messages' +import { NO_STORE_RECEIPTS } from '../native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture' import { createCodexJournalTranslator } from './codex-structured-journal-translation' import { handleCodexSessionExit } from './codex-structured-session-close' import type { CodexSession, CodexStructuredSessionEvent } from './codex-structured-session-state' @@ -46,15 +47,16 @@ afterEach(async () => { } }) -function userVisible(journal: AgentSessionJournal) { +/** What the chat shows at the lease's `fence`: 7 while the child runs, 8 once its exit released it. */ +function userVisible(journal: AgentSessionJournal, fence: number) { const items = journal.snapshot().items return { working: isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), + journal.activeTurnId(fence), journal.submissions(), - 8 + fence ), - prompt: pendingPromptExists(journal), + prompt: pendingPromptExists(journal, fence), turnStates: items.flatMap((item) => readAgentJournalTurn(item.body)?.state ?? []), runningCalls: items.filter( (item) => item.body.kind === 'tool-call' && item.body.state === 'running' @@ -149,7 +151,7 @@ it('settles a refused Codex exit through the host in one commit, then permits a ).toEqual({ accepted: true }) await expect(deferred.drained()).resolves.toEqual({ ok: true }) expect(deferred.state().queuedBytes).toBe(0) - expect(userVisible(journal)).toEqual({ + expect(userVisible(journal, 7)).toEqual({ working: true, prompt: true, turnStates: ['running'], @@ -230,7 +232,8 @@ it('settles a refused Codex exit through the host in one commit, then permits a transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)) + ) => (record = transition(record)), + conversationReceipts: NO_STORE_RECEIPTS }, sessions: new Map([[SESSION, hostSession]]), flushLifecycle: () => deferred.lifecycleBarrier(), @@ -240,7 +243,7 @@ it('settles a refused Codex exit through the host in one commit, then permits a }, event ) - expect(userVisible(journal)).toEqual({ + expect(userVisible(journal, 8)).toEqual({ working: false, prompt: false, turnStates: ['interrupted'], diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index 3a79762ee2dd..ba70b0a06721 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -10,6 +10,7 @@ import type { JournalLoad } from './journal-open' import type { AgentJournalEpochReason } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' import type { JournalWriteBody } from './journal-write-queue' +import type { JournalEpochFounding } from './journal-epoch-founding' export class JournalEpochController { constructor( @@ -25,6 +26,8 @@ export class JournalEpochController { queuePauseRestatement: () => JournalQueuePauseRestatement cursor: () => AgentJournalCursor adopt: (loaded: JournalLoad) => void + /** A published epoch replaces one still held unwritten. */ + founding: Pick } ) {} @@ -36,7 +39,10 @@ export class JournalEpochController { reason, fence, now: this.deps.now(), - onPublished: this.deps.adopt + onPublished: (loaded) => { + this.deps.founding.settled() + this.deps.adopt(loaded) + } }) } @@ -72,7 +78,10 @@ export class JournalEpochController { queuePause: this.deps.queuePauseRestatement(), now: this.deps.now, mintEpoch: this.deps.mintEpoch, - onPublished: this.deps.adopt + onPublished: (loaded) => { + this.deps.founding.settled() + this.deps.adopt(loaded) + } }) return this.deps.cursor() }) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts b/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts new file mode 100644 index 000000000000..a46d512f726d --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts @@ -0,0 +1,55 @@ +// A chat opened with no journal yet. Opening it writes nothing: its first epoch row is held here, +// in memory, and lands in the transaction of the first row the chat writes. A crash before then +// leaves no journal, exactly as before the open, and the next open mints a new epoch. + +import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' +import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import type Database from '../../sqlite/sync-database' +import type { JournalLoad } from './journal-open' +import { applyJournalRow, createJournalReducerState } from './journal-reducer' +import { insertJournalRow, publishJournalSessionEpoch } from './journal-row-table' +import type { JournalRow } from './journal-row-schema' + +export class JournalEpochFounding { + private pending: JournalRow | null = null + + constructor(private readonly identity: AgentSessionJournalIdentity) {} + + /** The empty journal a reader sees until the first write founds it. */ + hold(epoch: string, now: number): JournalLoad { + const row: JournalRow = { + kind: 'epoch', + reason: 'session_created', + providerHandle: agentSessionJournalProviderHandle(this.identity), + v: journalRowSchemaVersion([]), + epoch, + seq: 1, + fence: 0, + ts: now + } + const state = createJournalReducerState(this.identity.sessionId, epoch) + applyJournalRow(state, row) + state.oldestSequence = 1 + this.pending = row + return { state, newer: null, damage: null } + } + + /** Whether the epoch still waits for its first write. */ + held(): boolean { + return this.pending !== null + } + + /** First inside a transaction that writes the chat's rows: a rollback takes the epoch with it. */ + inTransaction(db: Database.Database): void { + if (this.pending) { + insertJournalRow(db, this.identity.sessionId, this.pending) + publishJournalSessionEpoch(db, this.identity, this.pending.epoch) + } + } + + /** The transaction that carried the epoch committed, or another epoch replaced it. */ + settled(): void { + this.pending = null + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts index 32b0c846571f..15ee5c71d815 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database-test-support.ts @@ -6,7 +6,10 @@ import { resolve } from 'node:path' import { JournalHostDatabase } from './journal-host-database' import { replayJournal, type JournalLoad } from './journal-open' import { serializeJournalRow, type JournalRow } from './journal-row-schema' +import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' +import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { + insertJournalRow, iterateJournalEpochRows, publishJournalSessionEpoch, readJournalSessionEpoch, @@ -137,7 +140,9 @@ export function insertTestJournalRow( insertTestJournalRowJson(db, sessionId, row.seq, serializeJournalRow(row), row.ts) } -/** A raw `row_json` at `seq`, the way a newer build or a bad write would leave it. */ +/** A raw `row_json` at `seq`, the way a newer build or a bad write would leave it. A chat whose + * journal was opened and never written holds its epoch in memory only; the row's own epoch is + * founded first, as that journal's first write would have. */ export function insertTestJournalRowJson( db: Database.Database, sessionId: string, @@ -145,11 +150,31 @@ export function insertTestJournalRowJson( rowJson: string, ts = 1 ): void { + if (readJournalSessionEpoch(db, sessionId) === null) { + const parsed: unknown = JSON.parse(rowJson) + const epoch = + typeof parsed === 'object' && parsed !== null && 'epoch' in parsed ? parsed.epoch : '' + foundTestJournalEpoch(db, sessionId, String(epoch)) + } db.prepare( 'INSERT INTO journal_rows (session_id, epoch, seq, ts, row_json) VALUES (?, ?, ?, ?, ?)' ).run(sessionId, liveEpoch(db, sessionId), seq, ts, rowJson) } +function foundTestJournalEpoch(db: Database.Database, sessionId: string, epoch: string): void { + insertJournalRow(db, sessionId, { + kind: 'epoch', + reason: 'session_created', + providerHandle: agentSessionJournalProviderHandle({ agent: 'claude', providerHandle: null }), + v: journalRowSchemaVersion([]), + epoch, + seq: 1, + fence: 0, + ts: 1 + }) + publishTestJournalEpoch(db, sessionId, epoch) +} + export function updateTestJournalRowJson( db: Database.Database, sessionId: string, diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index f942f0185d05..c49d7fe4f67e 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -16,9 +16,11 @@ import { } from './journal-row-builders' import type { JournalLifecycleBatchInput, + JournalPlannedLifecycleBatchInput, JournalResolvedLifecycleBatchInput } from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' +import type { JournalOperationReceipt } from './journal-row-writer' import { journalQueuedRejectionRowBuilders } from './journal-pending-submission-recovery' export class JournalLifecycleBatchAppender { @@ -27,7 +29,8 @@ export class JournalLifecycleBatchAppender { state: () => JournalReducerState cursor: () => AgentJournalCursor enqueueRows: ( - plan: () => readonly ((seq: number, ts: number) => JournalRow)[] + plan: () => readonly ((seq: number, ts: number) => JournalRow)[], + receipt?: JournalOperationReceipt ) => Promise } ) {} @@ -58,10 +61,28 @@ export class JournalLifecycleBatchAppender { ...this.planMutations(input, input.mutations) ] }) - .then((rows) => { - const last = rows.at(-1) - return last ? { epoch: last.epoch, sequence: last.seq } : this.deps.cursor() - }) + .then((rows) => this.cursorAfter(rows)) + } + + /** `append`, with every row chosen at the batch's own turn in the queue. */ + appendPlanned(input: JournalPlannedLifecycleBatchInput): Promise { + return this.deps + .enqueueRows(() => { + if (this.wasApplied(input.settlementId)) { + return [] + } + const { mutations, dispatches } = input.plan() + return [ + ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), + ...this.planMutations(input, mutations) + ] + }, input.receipt) + .then((rows) => this.cursorAfter(rows)) + } + + private cursorAfter(rows: readonly JournalRow[]): AgentJournalCursor { + const last = rows.at(-1) + return last ? { epoch: last.epoch, sequence: last.seq } : this.deps.cursor() } /** The rows a resolved settlement writes, planned at its own turn in the queue: its mutations, diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 3bbd9ef09ae0..2deac0e645cc 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -5,6 +5,7 @@ import type { AgentJournalCursor } from '../../../shared/agent-session-journal-t import type { JournalRow } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' import type { JournalWriteBody } from './journal-write-queue' +import type { JournalEpochFounding } from './journal-epoch-founding' /** Runs between BEGIN IMMEDIATE and COMMIT, on the SAME connection as the row * insert; a throw rolls the whole append back. Synchronous by construction so @@ -34,6 +35,8 @@ export type JournalRowWriterDeps = { inTransaction?: JournalRowTransactionHook /** After any rollback, so a cache filled inside the transaction cannot outlive it. */ rolledBack?: () => void + /** The chat's held first epoch, written with its first row (`journal-epoch-founding.ts`). */ + founding?: Pick } const BOOKKEEPING_SAVEPOINT = 'journal_row_bookkeeping' @@ -53,6 +56,7 @@ export class JournalRowWriter { try { // One INSERT: the chat's epoch pointer moves only when the epoch does. this.deps.database().transaction((db) => { + this.deps.founding?.inTransaction(db) insertJournalRow(db, this.deps.sessionId, row) hook?.(db, row) receipt?.write(db) @@ -65,6 +69,7 @@ export class JournalRowWriter { // COMMIT landed, so the row is durable: adopt it before anything that can // fail. Rejecting here instead would leave the next append reusing a // sequence the table already holds. The ledger first: it cannot throw, the fold can. + this.deps.founding?.settled() receipt?.committed() this.deps.commit([row]) return row @@ -80,7 +85,8 @@ export class JournalRowWriter { return this.deps.serialize(() => this.writeRows(plan, receipt)) } - /** `enqueueRows`' write, for a caller already running at its own turn in the queue. */ + /** `enqueueRows`' write, for a caller already running at its own turn in the queue. A receipt + * commits even when the plan is empty: what it records is already true of the rows. */ writeRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[], receipt?: JournalOperationReceipt @@ -89,7 +95,7 @@ export class JournalRowWriter { const first = this.deps.nextSequence() const ts = this.deps.now() const rows = plan().map((build, index) => build(first + index, ts)) - if (rows.length === 0) { + if (rows.length === 0 && !receipt) { return rows } for (const row of rows) { @@ -97,6 +103,9 @@ export class JournalRowWriter { } try { this.deps.database().transaction((db) => { + if (rows.length > 0) { + this.deps.founding?.inTransaction(db) + } for (const row of rows) { insertJournalRow(db, this.deps.sessionId, row) this.runBookkeeping(db, row) @@ -108,10 +117,26 @@ export class JournalRowWriter { throw error } receipt?.committed() - this.deps.commit(rows) + if (rows.length > 0) { + this.deps.founding?.settled() + this.deps.commit(rows) + } return rows } + /** Writes the chat's held first epoch on its own, for an acquisition: a chat whose agent started + * is kept even if nothing else is written. Nothing when it is founded already. */ + found(): Promise { + return this.deps.serialize(() => { + if (!this.deps.founding?.held()) { + return + } + assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) + this.deps.database().transaction((db) => this.deps.founding?.inTransaction(db)) + this.deps.founding.settled() + }) + } + /** Assign the next sequence, make the row durable, and fold it through the SAME reducer * replay uses — all inside one serialized step — answering where the row landed. */ append( diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 76a7dff8a303..790201d56f8a 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -9,6 +9,7 @@ import type { } from '../../../shared/agent-session-journal-types' import type { JournalHostDatabase } from './journal-host-database' import { JournalEpochController } from './journal-epoch-controller' +import { JournalEpochFounding } from './journal-epoch-founding' import { JournalItemAppender } from './journal-item-appender' import { JournalLifecycleBatchAppender } from './journal-lifecycle-batch-appender' import type { JournalLoad } from './journal-open' @@ -60,7 +61,9 @@ export type JournalStoreCollaborators = { } export function createJournalStoreCollaborators(host: JournalStoreHost): JournalStoreCollaborators { + const founding = new JournalEpochFounding(host.identity) const epochController = new JournalEpochController({ + founding, identity: host.identity, now: host.now, mintEpoch: host.mintEpoch, @@ -109,23 +112,21 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal // Every rejection is a dispatch row through this one writer; the draft // returned-transition rides it so no path can bypass the hook. inTransaction: (db, row) => queuedMessages.onRowInTransaction(db, row), - rolledBack: () => queuedMessages.invalidate() + rolledBack: () => queuedMessages.invalidate(), + founding }) const lifecycleBatchAppender = new JournalLifecycleBatchAppender({ state: host.state, cursor: host.cursor, - enqueueRows: (plan) => rowWriter.enqueueRows(plan) + enqueueRows: (plan, receipt) => rowWriter.enqueueRows(plan, receipt) }) return { epochController, queuedMessages, stopMarks: new JournalStopMarks({ state: host.state }), - // Behind the stored fact: settles drafts whose consumed submission the loaded journal shows - // refused (a downgrade wrote no hook), then prunes. Bookkeeping, never failing the open. - restore: () => - restoreJournalStore(host, { epochController }).then(() => - queuedMessages.repairAndPruneAtOpen() - ), + // Reads only: the drafts' repair and prune are startup's (`repairAndPrune`), and a chat with + // no journal yet is founded by its first write. + restore: () => restoreJournalStore(host, founding), rowWriter, submissionWriter: new JournalSubmissionWriter({ state: host.state, diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 290a59610366..9d4b4473509f 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -15,6 +15,7 @@ import type { AgentSessionMessageSource } from '../../../shared/agent-session-me import type { JournalHostDatabase } from './journal-host-database' import type { JournalLifecycleMutationInput } from './journal-row-builders' import type { JournalRow } from './journal-row-schema' +import type { JournalOperationReceipt } from './journal-row-writer' export type AgentSessionJournalOptions = { identity: AgentSessionJournalIdentity @@ -82,6 +83,20 @@ export type JournalLifecycleBatchInput = { rejectsQueued?: AgentJournalDispatchRejection } +/** A lifecycle batch whose rows are chosen at its own turn in the write queue, so a write queued + * ahead of it (an answer, a Stop) is what it plans from. */ +export type JournalPlannedLifecycleBatchInput = Pick< + JournalLifecycleBatchInput, + 'settlementId' | 'fence' | 'recovered' +> & { + plan: () => { + mutations: readonly JournalLifecycleMutationInput[] + dispatches: readonly ResolveDispatchInput[] + } + /** Committed in the same transaction as the rows, even when the plan is empty. */ + receipt?: JournalOperationReceipt +} + export type JournalResolvedLifecycleBatchInput = Omit< JournalLifecycleBatchInput, 'mutations' | 'rejectsQueued' | 'dispatches' diff --git a/src/main/native-chat/agent-session-journal/journal-store-open.ts b/src/main/native-chat/agent-session-journal/journal-store-open.ts index 1a5db4fe0d27..79fb36b64d77 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-open.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-open.ts @@ -1,23 +1,14 @@ -import type { - AgentJournalItemBody, - AgentJournalItemIdentity -} from '../../../shared/agent-session-journal-types' import type { JournalLoad } from './journal-open' import { failLoadOnUnloadableJournal } from './journal-open-failure' -import { staleSubagentRosterRevisions } from './journal-subagent-liveness' -export async function openJournalStoreState(input: { +/** Replays the saved journal. What a gone writer left live is read as saved: its owner's exit, the + * next acquisition, or startup settles it, never this open. */ +export function openJournalStoreState(input: { sessionId: string replay: () => JournalLoad | null start: () => void adopt: (loaded: JournalLoad) => void - appendItem: ( - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody, - fence: number - ) => Promise - highestFence: () => number -}): Promise { +}): void { const loaded = input.replay() // An epoch named but holding no row (a crash inside an older build's repair) has nothing to // keep, so it is founded afresh like a chat with no journal; no row is deleted. @@ -27,22 +18,4 @@ export async function openJournalStoreState(input: { } failLoadOnUnloadableJournal(input.sessionId, loaded) input.adopt(loaded) - await settleStaleSubagentRosters(input, loaded) -} - -/** Retires a `working` subagent roster the previous host never got to settle. */ -async function settleStaleSubagentRosters( - input: { - appendItem: ( - identity: AgentJournalItemIdentity, - body: AgentJournalItemBody, - fence: number - ) => Promise - highestFence: () => number - }, - loaded: JournalLoad -): Promise { - for (const revision of staleSubagentRosterRevisions(loaded.state.items.values())) { - await input.appendItem(revision.identity, revision.body, input.highestFence()) - } } diff --git a/src/main/native-chat/agent-session-journal/journal-store-restore.ts b/src/main/native-chat/agent-session-journal/journal-store-restore.ts index 0f5208dea9bb..94b9805305f9 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-restore.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-restore.ts @@ -1,13 +1,8 @@ -// Bringing a store's in-memory state up from disk. -// -// Split out of the store for the same reason its collaborators were: this is the -// ORDERING between replay and the notices an open owes, and none of it belongs -// to the store's public surface. Every step here reads or writes through the -// same host the collaborators use, so the store keeps the state and this owns -// the sequence. +// Bringing a store's in-memory state up from disk. It reads and never writes: a chat with no +// journal yet gets an empty one held in memory (`journal-epoch-founding.ts`), and what a gone +// writer left live is settled by the ownership event that ended it, never by this open. -import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' -import type { JournalEpochController } from './journal-epoch-controller' +import type { JournalEpochFounding } from './journal-epoch-founding' import { replayJournal } from './journal-open' import { journalOpenRefusalError } from './journal-open-failure' import type { JournalStoreHost } from './journal-store-collaborators' @@ -16,7 +11,7 @@ import { AgentSessionJournalError } from './journal-write-guards' export async function restoreJournalStore( host: JournalStoreHost, - collaborators: { epochController: JournalEpochController } + founding: Pick ): Promise { const database = host.database() if (database.readOnly) { @@ -28,14 +23,10 @@ export async function restoreJournalStore( ) ) } - return openJournalStoreState({ + openJournalStoreState({ sessionId: host.identity.sessionId, replay: () => replayJournal(database.db, host.identity.sessionId), - start: () => collaborators.epochController.start('session_created', 0), - adopt: host.adopt, - // Roster notices are about the conversation, not any turn in it. - appendItem: (identity, body, fence) => - host.journal().appendItem(identity, body, { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE }), - highestFence: () => host.state().highestFence + start: () => host.adopt(founding.hold(host.mintEpoch(), host.now())), + adopt: host.adopt }) } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 25e3a7e8d865..5b9d66cb9d69 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -21,7 +21,7 @@ import { currentAgentSessionThreadGoalBySequence } from '../../../shared/agent-s import type { AgentSessionContextUsage } from '../../../shared/agent-session-context-usage' import { latestStructuredAgentContextFacts } from '../../../shared/structured-agent-session-context-usage' import { - activeStructuredAgentSessionTurnIdBySequence, + currentStructuredAgentSessionTurnId, liveStructuredAgentSessionTurnScope, newestStructuredAgentSessionTurnBySequence } from '../../../shared/structured-agent-session-live-turn' @@ -48,6 +48,7 @@ import type { JournalItemAppendOptions, JournalItemLinkageVisitor, JournalLifecycleBatchInput, + JournalPlannedLifecycleBatchInput, JournalReadSince, JournalSubmissionConsume, JournalSubmissionInput, @@ -223,9 +224,9 @@ export class AgentSessionJournal { } /** The turn this journal has published as running — the same read a client's snapshot gives, - * without materialising one. */ - activeTurnId = (): string | null => - activeStructuredAgentSessionTurnIdBySequence(this.state.items.values()) + * without materialising one. Given the lease's fence, only one a current generation opened. */ + activeTurnId = (currentFence?: number | null): string | null => + currentStructuredAgentSessionTurnId(this.state.items.values(), this.itemFence, currentFence) /** Where a row written now belongs: the running turn, or the conversation. */ liveTurnScope = (): AgentJournalTurnScope => @@ -330,6 +331,13 @@ export class AgentSessionJournal { return this.lifecycleBatchAppender.append(input) } + /** A lifecycle batch planned at its own turn in the queue (`JournalPlannedLifecycleBatchInput`). */ + appendPlannedLifecycleBatch = (input: JournalPlannedLifecycleBatchInput) => + this.lifecycleBatchAppender.appendPlanned(input) + + /** An acquisition's write of a chat's first epoch (`JournalRowWriter.found`). */ + foundEpoch = (): Promise => this.rowWriter.found() + /** Several writes as one turn in the queue; see `JournalStepWriter`. */ appendSteps: JournalStepWriter['append'] = (steps) => this.stepWriter.append(steps) diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts index 89805d366081..67ea317ca56c 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.test.ts @@ -21,7 +21,8 @@ import { } from '../../codex/codex-subagent-roster' import type { openAgentSessionJournal } from './journal-store-factory' import { createTrackedJournalOpener } from './journal-host-database-test-support' -import { lostLiveWorkJournalBody, staleSubagentRosterRevisions } from './journal-subagent-liveness' +import { lostLiveWorkJournalBody } from './journal-subagent-liveness' +import { settleStaleStructuredAgentSessionState } from '../agent-session-wire/structured-agent-session-stale-state-settlement' import { endedUnseenMessageBody } from './journal-terminal-settlement' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' @@ -55,6 +56,26 @@ async function open(overrides: Partial { + const body = lostLiveWorkJournalBody(item.body) + return body ? [{ body }] : [] + }) +} + +/** What the next ownership event settles once the generation that wrote at fence 0 is gone. */ +async function settleGoneGeneration(journal: Awaited>): Promise { + await settleStaleStructuredAgentSessionState({ + journal, + sessionId: 'session-1', + fence: 1, + acquisitionGeneration: null, + deathEvidence: null, + below: 1 + }) +} + /** The row as the producer writes it: the structured block plus its twin. */ function rosterRow(agents: NativeChatSubagentEntry[]) { return { @@ -126,9 +147,9 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }) }) -describe('staleSubagentRosterRevisions', () => { +describe('lostLiveWorkJournalBody', () => { it('settles a child the previous host left working, and moves the twin with it', () => { - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ renderItem([ { id: 'a', label: 'read_readme', state: 'working', startedAt: 10 }, { id: 'b', label: 'read_package', state: 'completed', startedAt: 10, settledAt: 20 } @@ -146,7 +167,7 @@ describe('staleSubagentRosterRevisions', () => { it('settles a background task the previous host left live, and moves the twin with it', () => { const row = backgroundTaskRow() - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ { itemId: agentJournalItemKey(row.identity), revision: 1, @@ -164,7 +185,7 @@ describe('staleSubagentRosterRevisions', () => { // The child stopped being observable at an unknown moment. A stamp taken now // would report the time the app was down as how long the child ran. it('records no terminal timestamp for a child whose run length is unknown', () => { - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ renderItem([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }]) ]) @@ -173,7 +194,7 @@ describe('staleSubagentRosterRevisions', () => { it('records no terminal timestamp for a background task whose run length is unknown', () => { const row = backgroundTaskRow(backgroundTaskBlock({ settledAt: 20 })) - const revisions = staleSubagentRosterRevisions([ + const revisions = lostLiveWork([ { itemId: agentJournalItemKey(row.identity), revision: 1, @@ -188,15 +209,13 @@ describe('staleSubagentRosterRevisions', () => { it('owes nothing for a roster whose children all settled', () => { expect( - staleSubagentRosterRevisions([ - renderItem([{ id: 'a', label: 'read', state: 'completed', settledAt: 20 }]) - ]) + lostLiveWork([renderItem([{ id: 'a', label: 'read', state: 'completed', settledAt: 20 }])]) ).toEqual([]) }) it('leaves rows that carry no roster alone', () => { expect( - staleSubagentRosterRevisions([ + lostLiveWork([ { itemId: 'orca:plain', revision: 1, @@ -207,19 +226,9 @@ describe('staleSubagentRosterRevisions', () => { ]) ).toEqual([]) }) - - // Appending under a fresh identity would add a second row rather than revise - // the one on disk, so an unaddressable key is left exactly as it is. - it('skips a row whose key cannot be parsed back to its identity', () => { - expect( - staleSubagentRosterRevisions([ - { ...renderItem([{ id: 'a', label: 'r', state: 'working' }]), itemId: 'not-a-key' } - ]) - ).toEqual([]) - }) }) -describe('journal reopen after the writing host is gone', () => { +describe('the settlement after the writing host is gone', () => { it('writes composed roster, task, reasoning and text-twin corrections as one settlement step', async () => { const journal = await open() const roster = rosterRow([ @@ -308,6 +317,7 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const reopened = await open() + await settleGoneGeneration(reopened) const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(rosterOf(afterRestart.body)).toMatchObject([ @@ -328,6 +338,7 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const reopened = await open() + await settleGoneGeneration(reopened) expect(reopened.snapshot().items).toHaveLength(before) expect(reopened.snapshot().items.at(-1)?.revision).toBe(2) }) @@ -345,13 +356,14 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const reopened = await open() + await settleGoneGeneration(reopened) const afterRestart = reopened.snapshot().items.at(-1)! expect(afterRestart.itemId).toBe(beforeRestart.itemId) expect(taskOf(afterRestart.body)).toMatchObject({ state: 'unverifiable' }) expect(twinOf(afterRestart.body)).toBe('Background command "sleep 20" stopped reporting') }) - it('writes nothing on a second reopen once every child is settled', async () => { + it('writes nothing on a second settlement once every child is settled', async () => { const live = await open() const row = rosterRow([{ id: 'a', label: 'read', state: 'working', startedAt: 10 }]) await live.appendItem(row.identity, row.body, { @@ -361,10 +373,12 @@ describe('journal reopen after the writing host is gone', () => { await live.close() const once = await open() + await settleGoneGeneration(once) const revision = once.snapshot().items.at(-1)?.revision await once.close() const twice = await open() + await settleGoneGeneration(twice) expect(twice.snapshot().items.at(-1)?.revision).toBe(revision) }) }) diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts index ea4cebab1504..41dcc19f2a73 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts @@ -4,8 +4,8 @@ // the children, and sweeps whatever is still `working` when the provider goes // away. A host that DIED — crash, quit, force-restart — does neither: its last // revision goes on saying `working`, and nothing replays those children, so no -// later event can ever settle them. Opening the journal is the one moment a new -// host can state the truth about the old one: contact was lost. That is +// later event can ever settle them. The leftover settlement at the next ownership +// event states the truth about the old one: contact was lost. That is // `unverifiable`, never a synthesized exit — see // `docs/reference/ssh-execution-boundary.md`. // @@ -13,15 +13,7 @@ // from the journal reads these rows after this revision, so it inherits the // verdict rather than the dead host's `working`. -import { - agentJournalItemKey, - parseAgentJournalItemKey -} from '../../../shared/agent-session-journal-item-key' -import type { - AgentJournalItemBody, - AgentJournalItemIdentity, - AgentJournalRenderItem -} from '../../../shared/agent-session-journal-types' +import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' import { isSubagentGroupFallbackText, normalizeSubagentState, @@ -40,33 +32,6 @@ import { type NativeChatSubagentGroupBlock } from '../../../shared/native-chat-types' -export type JournalSubagentLivenessRevision = { - identity: AgentJournalItemIdentity - body: AgentJournalItemBody -} - -/** The revisions a reopened journal owes: one per row still claiming a live - * child. Empty — the common case — when nothing was left mid-flight. */ -export function staleSubagentRosterRevisions( - items: Iterable -): JournalSubagentLivenessRevision[] { - const revisions: JournalSubagentLivenessRevision[] = [] - for (const item of items) { - const body = lostLiveWorkJournalBody(item.body) - if (!body) { - continue - } - // A key that will not parse cannot be re-addressed, and appending under a - // fresh identity would duplicate the row rather than revise it. - const identity = parseAgentJournalItemKey(item.itemId) - if (!identity || agentJournalItemKey(identity) !== item.itemId) { - continue - } - revisions.push({ identity, body }) - } - return revisions -} - /** The row once the host lost the session running its live work: every working child and * in-flight background task `unverifiable`, plain-text twins restated. Null when none is live. */ export function lostLiveWorkJournalBody(body: AgentJournalItemBody): AgentJournalItemBody | null { diff --git a/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts b/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts index 5f9a95a53d20..c96e104096b7 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-bookkeeping-failure.test.ts @@ -91,7 +91,7 @@ afterEach(async () => { }) describe('draft bookkeeping inside a journal append', () => { - it('a throwing draft transition still commits the rejection row, and the next open recovers the draft', async () => { + it('a throwing draft transition still commits the rejection row, and the next startup recovers the draft', async () => { let journal = await open() await queueAndConsume(journal, 'draft-1') const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) @@ -113,6 +113,9 @@ describe('draft bookkeeping inside a journal append', () => { ) await journal.close() journal = await open() + // An open only reads; startup's repair (`settleOneAtStartup`) recovers it. + expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') + await journal.queuedMessages.repairAndPrune() expect(journal.submission('sub-draft-1')?.dispatchState).toBe('rejected') expect(journal.queuedMessages.get('draft-1')).toMatchObject({ state: 'returned', diff --git a/src/main/native-chat/agent-session-journal/queued-message-store.test.ts b/src/main/native-chat/agent-session-journal/queued-message-store.test.ts index 02e07bc53ee6..816c9a35d7e3 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-store.test.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-store.test.ts @@ -77,6 +77,13 @@ async function open(): Promise { return journal } +/** An open as startup makes it: then the cards' repair and prune (`settleOneAtStartup`). */ +async function openAtStartup(): Promise { + const journal = await open() + await journal.queuedMessages.repairAndPrune() + return journal +} + async function queueDraft(journal: AgentSessionJournal, messageId: string, text = 'queued text') { return journal.queuedMessages.insert({ messageId, @@ -573,23 +580,31 @@ describe('withdraw', () => { }) }) -describe('open-time repair and retention', () => { - it('a failed repair is reported and skipped, never failing the open', async () => { +describe('startup repair and retention', () => { + it('an open repairs nothing; a failed startup repair leaves the journal usable', async () => { + let journal = await open() + await queueDraft(journal, 'draft-1') + await consumeDraft(journal, 'draft-1') + await journal.rejectQueuedSubmissions(0, HOST_RESTARTED) + await journal.close() + const db = new Database(journalDatabasePath(root)) + db.prepare( + "UPDATE queued_messages SET state = 'dispatched', hold_reason = NULL, consumed_as = 'sub-draft-1' WHERE message_id = ?" + ).run('draft-1') + db.close() + journal = await open() + // Opening only reads: the cut-short hand-off waits for startup's repair. + expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') const repair = vi .spyOn(JournalQueuedMessages.prototype, 'repairAndPrune') .mockRejectedValueOnce(new Error('SQLITE_FULL')) - const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) try { - const journal = await open() - expect(warn).toHaveBeenCalledWith( - '[journal-open] queued-message repair skipped:', - expect.objectContaining({ error: 'SQLITE_FULL' }) - ) - await queueDraft(journal, 'draft-1') - expect(journal.queuedMessages.list()).toHaveLength(1) + // Startup reports it (`structured-agent-session-startup-settlement.ts`) and goes on. + await expect(journal.queuedMessages.repairAndPrune()).rejects.toThrow('SQLITE_FULL') + await queueDraft(journal, 'draft-2') + expect(journal.queuedMessages.list()).toHaveLength(2) } finally { repair.mockRestore() - warn.mockRestore() } }) @@ -600,7 +615,7 @@ describe('open-time repair and retention', () => { { origin: 'client' as const, cause: 'chatClosed' as const }, { origin: 'host' as const, cause: 'chatClosed' as const } ])( - 'a skipped hook for a $origin hand-off cut short ($cause) is repaired at open, back to waiting', + 'a skipped hook for a $origin hand-off cut short ($cause) is repaired at startup, back to waiting', async ({ origin, cause }) => { let journal = await open() await queueDraft(journal, 'draft-1') @@ -616,7 +631,7 @@ describe('open-time repair and retention', () => { "UPDATE queued_messages SET state = 'dispatched', hold_reason = NULL, consumed_as = 'sub-draft-1' WHERE message_id = ?" ).run('draft-1') db.close() - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.get('draft-1')).toMatchObject({ state: 'waiting', holdReason: null, @@ -644,7 +659,7 @@ describe('open-time repair and retention', () => { "UPDATE queued_messages SET state = 'dispatched', returned_reason = NULL, returned_rejection = NULL WHERE message_id = ?" ).run('draft-1') db.close() - journal = await open() + journal = await openAtStartup() const row = journal.queuedMessages.get('draft-1') expect(row?.state).toBe('returned') expect(row?.returnedReason).toBe(refusal('refused while downgraded').reason) @@ -663,7 +678,7 @@ describe('open-time repair and retention', () => { ).run('draft-1') db.close() clock += QUEUED_MESSAGE_REPLAY_WINDOW_MS + 1_000 - journal = await open() + journal = await openAtStartup() // The same settlement the live hook applies. expect(journal.queuedMessages.get('draft-1')).toMatchObject({ state: 'waiting', @@ -680,7 +695,7 @@ describe('open-time repair and retention', () => { await journal.close() // Reopen "25 hours" later: the submission is still queued/pending. clock += QUEUED_MESSAGE_REPLAY_WINDOW_MS + 60 * 60 * 1000 - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.get('draft-1')?.state).toBe('dispatched') // The delivery loop's leftover rejection now sends it back to waiting. await journal.rejectQueuedSubmissions(0, HOST_RESTARTED) @@ -716,7 +731,7 @@ describe('open-time repair and retention', () => { }) await journal.close() clock += QUEUED_MESSAGE_REPLAY_WINDOW_MS + 1_000 - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.list().map((row) => [row.messageId, row.state])).toEqual([ ['waiting-1', 'waiting'], ['returned-1', 'returned'] @@ -732,7 +747,7 @@ describe('open-time repair and retention', () => { }) await journal.close() clock += 1_000 - journal = await open() + journal = await openAtStartup() expect(journal.queuedMessages.get('withdrawn-1')?.state).toBe('withdrawn') }) }) diff --git a/src/main/native-chat/agent-session-journal/queued-message-table.ts b/src/main/native-chat/agent-session-journal/queued-message-table.ts index 303ea060c633..997e2ba886ec 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-table.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-table.ts @@ -129,6 +129,14 @@ export function listQueuedMessages(db: Database.Database, sessionId: string): Qu .flatMap((row) => readStoredQueuedMessageRow(row) ?? []) } +/** Every chat holding a card, of any state: the chats startup re-derives cards for. */ +export function listSessionsWithQueuedMessages(db: Database.Database): string[] { + return db + .prepare('SELECT DISTINCT session_id FROM queued_messages') + .all() + .flatMap((row) => (typeof row.session_id === 'string' ? [row.session_id] : [])) +} + export function getQueuedMessage( db: Database.Database, sessionId: string, diff --git a/src/main/native-chat/agent-session-record-test-fixture.ts b/src/main/native-chat/agent-session-record-test-fixture.ts index 21e7f1d5191e..485bdea37b01 100644 --- a/src/main/native-chat/agent-session-record-test-fixture.ts +++ b/src/main/native-chat/agent-session-record-test-fixture.ts @@ -36,7 +36,8 @@ export function agentSessionRecordFixture( claimKeyId: 'test-key', claimStatus: 'released', unreconciled: false, - deathEvidence: null + deathEvidence: null, + leftoverSettledAt: null }, ...rest, location: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index 29a033a4e7b9..78567db9f2c2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -50,6 +50,7 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-hand import { StructuredAgentArgumentsError } from '../structured-agent-arguments-error' import { USER_MESSAGE_SOURCE } from '../../../shared/agent-session-message-source' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { startUpOwingLeftovers } from './structured-agent-session-leftover-settlement.test-fixture' const CALLER = { callerKey: 'client-1' } @@ -557,8 +558,7 @@ describe('what an earlier host process left behind', () => { await writeAsEarlierProcess(async (journal, fence) => { await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence }) }) - - await host.revealSession(SESSION) + await startUpOwingLeftovers(host, store) expect(await submission('queued')).toMatchObject({ dispatchState: 'rejected', @@ -584,8 +584,7 @@ describe('what an earlier host process left behind', () => { }) await host.flushAllStreamedEvents() await startHost() - - await host.revealSession(SESSION) + await startUpOwingLeftovers(host, store) expect(await submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) expect(await submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true }) @@ -676,7 +675,7 @@ describe('a start that fails while messages wait on it', () => { }) describe('Stop withdraws what is queued', () => { - it('never meets a crash leftover: the open it runs settles it first (W17a)', async () => { + it('never meets a crash leftover: startup settles it first (W17a)', async () => { await writeAsEarlierProcess(async (journal, fence) => { await journal.appendSubmission({ ...earlierSubmission('person', 'p', true), @@ -686,6 +685,7 @@ describe('Stop withdraws what is queued', () => { }) await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) }) + await startUpOwingLeftovers(host, store) expect(await stop()).toMatchObject({ ok: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts new file mode 100644 index 000000000000..3ca7851f0592 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts @@ -0,0 +1,72 @@ +// An acquisition's share of the leftover settlement (`structured-agent-session-leftover-settlement.ts`): +// before the reservation, reading the lease directly, while it still holds the proof of how the +// last generation ended. The reservation clears that proof, so after it the same work could only +// settle `unverifiable`. Best effort: a failure is logged and the start goes on. + +import { + agentSessionLeaseIsReleased, + isProvenDeadProbe, + type AgentSessionOwnerProbe +} from '../../../shared/agent-session-lease-adjudication' +import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' +import { settleStructuredAgentSessionLeftoversOrLog } from './structured-agent-session-leftover-settlement' + +const SCOPE = 'acquisition-settlement' + +export async function settleStructuredAgentSessionLeftoversBeforeReserve( + context: Pick< + StructuredAgentSessionAttachContext, + 'deps' | 'sessions' | 'openConversation' | 'now' | 'runtimeState' + >, + sessionId: string, + probe: AgentSessionOwnerProbe +): Promise { + const { store, logger } = context.deps + const lease = store.getRecord(sessionId)?.lease + if (!lease) { + return + } + // A recorded owner the probe proved gone is ended by the reservation that follows; one alive, + // unproven, or not yet reconciled keeps its work, which is still its own. + const ownerProvenGone = + !lease.unreconciled && + lease.claimStatus !== 'conflicted' && + lease.ownerProcess !== null && + isProvenDeadProbe(probe) + if (!agentSessionLeaseIsReleased(lease) && !ownerProvenGone) { + return + } + if (ownerProvenGone && context.sessions.get(sessionId)?.child) { + // A child of this process that died unseen: what it wrote lands before its work is judged. + const barrier = await context.runtimeState.currentEventSink(sessionId)?.drained() + if (barrier && !barrier.ok) { + logger.warn("settling a gone agent's leftover work did not finish", { + scope: SCOPE, + sessionId, + error: barrier.error + }) + return + } + } + const conversation = await context.openConversation(sessionId).catch((error: unknown) => { + // The attach's own open meets the same failure and answers it. + logger.warn("settling a gone agent's leftover work did not finish", { + scope: SCOPE, + sessionId, + error + }) + return null + }) + if (!conversation) { + return + } + await settleStructuredAgentSessionLeftoversOrLog({ + store, + sessionId, + journal: conversation.journal, + now: context.now, + logger, + scope: SCOPE, + ...(ownerProvenGone ? { ownerProvenGone: true as const } : {}) + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 12c2f54a1793..468a48dfcae1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -15,7 +15,6 @@ import type { import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' -import type { StructuredAgentSessionConversationOpenOptions } from './structured-agent-session-conversation-open' export type StructuredAgentSessionAttachContext = { deps: StructuredAgentSessionHostDeps @@ -38,8 +37,5 @@ export type StructuredAgentSessionAttachContext = { endExitedChild: StructuredAgentSessionLifetimeContext['endExitedChild'] wakeDelivery?: StructuredAgentSessionLifetimeContext['wakeDelivery'] /** The conversation's one open journal, opened when closed; see `conversation-open`. */ - openConversation: ( - sessionId: string, - options?: StructuredAgentSessionConversationOpenOptions - ) => Promise + openConversation: (sessionId: string) => Promise } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index cd419133389c..b330893965f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -22,8 +22,7 @@ import { pinnedAgentSessionLaunchEnv } from './structured-agent-session-launch-env' import { refuseAgentSessionMutation } from './structured-agent-session-mutation-admission' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' -import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' +import { settleStructuredAgentSessionLeftoversBeforeReserve } from './structured-agent-session-acquisition-settlement' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { @@ -148,10 +147,10 @@ async function runAttachUnderAbort( const probe = await withAgentSessionCreatePhase('probe_owner', recordPhase, () => context.runtimeState.probeOwner(sessionId) ) - // Read before the reserve clears it: how the previous generation ended decides how whatever it - // left running is settled. + // Before the reserve wipes the proof of how the last generation ended: best effort, so a + // settlement that cannot commit never stops this start. + await settleStructuredAgentSessionLeftoversBeforeReserve(context, sessionId, probe) const priorRecord = context.deps.store.getRecord(sessionId) - const priorDeathEvidence = priorRecord?.lease.deathEvidence ?? null const launchDirectory = !priorRecord && isFloatingWorkspaceId(params.location.workspaceId) ? (options.hostLaunchDirectory ?? @@ -205,9 +204,7 @@ async function runAttachUnderAbort( optionRevision: () => context.runtimeState.optionRevisions.current(sessionId), ...(options.onAcquisitionFailed ? { onAcquisitionFailed: options.onAcquisitionFailed } : {}), openConversation: async (record) => { - const conversation = await context.openConversation(record.sessionId, { - acquisition: true - }) + const conversation = await context.openConversation(record.sessionId) if (!conversation) { throw new Error('agent_session_identity_required') } @@ -225,15 +222,14 @@ async function runAttachUnderAbort( ? attemptSink : (context.runtimeState.currentEventSink(sessionId) ?? attemptSink) if (acquiredOwner) { - // Before the drain: the buffered events are the new child's, never a stale row's. - await settleStaleStructuredAgentSessionState({ - journal: attached.journal, - sessionId, - fence, - acquisitionGeneration, - deathEvidence: priorDeathEvidence, - failureTextContext: structuredAgentSessionFailureWordsContext(priorRecord) - }) + // The chat is kept from its first start, written to or not; never a reason to fail it. + await attached.journal.foundEpoch().catch((error: unknown) => + context.deps.logger.warn("writing a chat's first journal row failed", { + scope: 'journal-founding', + sessionId, + error + }) + ) } await bindAndDrain(eventSink, attached.journal, fence, (activity) => context.subscribers.publish(sessionId, attached.journal, activity) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts index e22c5dea9fdc..8e7f39c6b5f8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-reconciliation.test.ts @@ -17,6 +17,7 @@ import { } from '../agent-session-journal/journal-host-database-test-support' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { openTestAttachConversation } from './structured-agent-session-attach-test-conversation' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-stale-state-settlement' import { attachJournal, journalIdentityFor, @@ -66,17 +67,27 @@ function adapterWith(providerHistoryWindow?: () => Promise Promise { return async (record) => - ( - await openStructuredAgentSessionConversationJournal( - { journalDatabase, logger: recordingStructuredAgentSessionLogger().logger }, - record, - { - acquisition: true - } - ) - ).session.journal + (await openStructuredAgentSessionConversationJournal({ journalDatabase }, record)).session + .journal } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts index b6cd948fa4dc..1e508636ebf0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-chat-stop.ts @@ -156,6 +156,8 @@ export function mutateWithChatStop( windDown = { owed, ctx } }, withdrewQueued: withdrew, + noteFence: () => + context.deps.store.getRecord(sessionId)?.lease.runtimeFence ?? ctx.fence, ...(ownsLatestStop ? { opensSettle: true as const } : {}) } ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 694268a1f8bf..24f144327305 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -9,7 +9,6 @@ import { } from '../../../shared/agent-session-failure' import { PROVIDER_EXIT_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionEndedEvent } from './structured-agent-session-adapter' import type { StructuredAgentSessionHostSession, @@ -21,7 +20,12 @@ import { type StructuredAgentSessionLeaseStore } from './structured-agent-session-lease-release' import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' -import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' +import { + settleStructuredAgentSessionLeftovers, + type StructuredAgentSessionLeftoverSettlementInput, + type StructuredAgentSessionLeftoverStore +} from './structured-agent-session-leftover-settlement' +import type { StaleStructuredAgentSessionStateJournal } from './structured-agent-session-stale-state-settlement' import { captureUnfinishedStructuredAgentSessionWork, type DeadGenerationJournal, @@ -47,12 +51,12 @@ export type StructuredAgentSessionChildExit = { export type StructuredAgentSessionChildExitSession = Pick< StructuredAgentSessionHostSession, 'child' | 'lastEndedChild' -> & { journal: DeadGenerationJournal & Pick } +> & { journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal } export type StructuredAgentSessionChildExitContext< TSession extends StructuredAgentSessionChildExitSession = StructuredAgentSessionHostSession > = { - store: StructuredAgentSessionLeaseStore + store: StructuredAgentSessionLeaseStore & StructuredAgentSessionLeftoverStore sessions: Map flushLifecycle: (sessionId: string) => Promise publishFence: (sessionId: string, session: TSession) => void @@ -168,6 +172,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< context.wakeDelivery?.(sessionId) return } + let settlement: StructuredAgentSessionLeftoverSettlementInput['exit'] | null = null try { // The exited child's own writes land first: its dead generation is settled from all of them. try { @@ -191,13 +196,11 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // Folded before the fallback's end is built, so the end reads it (`turnEndAfterStop`). await close?.recorded const generation = child.generation ?? 'unknown' - // The exit proves this child gone, as the record's death evidence later says: what it left - // `unverifiable` is revised now, not at the next open. + // The exit proves this child gone, as the record's death evidence says once released: what it + // left `unverifiable` is revised now. const watched = { ownerFence: child.fence, observedAt } - const settled = await settleStructuredAgentSessionDeadGeneration({ - journal: session.journal, - sessionId, - fence: child.fence, + settlement = { + ownerFence: child.fence, settlementId: `${expected ? 'expected-close:' : PROVIDER_EXIT_ROW_PREFIX}${sessionId}:${child.fence}:${generation}`, pendingSubmissionReason: expected ? 'provider_closed_before_acknowledgement' @@ -222,14 +225,11 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< : {}), exit: watched, ...(unrunRejection ? { unrunRejection } : {}) - }) - if (!settled.ok) { - logExitFailure(context, sessionId, 'exit-settlement', settled.error) } } finally { - // The root's exit was observed, so the owner is released even when terminal settlement could - // not be durably accepted. Bare cause: whatever this settlement could not write is settled from - // it later (the settle recording it queues, or the next open or acquire). + // The root's exit was observed, so the owner is released first, writing the proof and ending + // the generation; then what it left is settled at the new fence, with the mark that says so. + // A settlement that cannot commit stays owed: the next acquisition or startup settles it. let released = false try { await releaseStoredStructuredAgentSessionOwnerAfterExit({ @@ -244,6 +244,18 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< } catch (error) { logExitFailure(context, sessionId, 'exit-owner-release', error) } + if (settlement) { + const settled = await settleStructuredAgentSessionLeftovers({ + store: context.store, + sessionId, + journal: session.journal, + now: context.now, + exit: settlement + }) + if (!settled.ok) { + logExitFailure(context, sessionId, 'exit-settlement', settled.error) + } + } if (context.route) { const { runtimeState, acknowledgeRelease } = context.route await evictStructuredAgentSession({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index ec7f478501f7..cdee4f6f7e2f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -1,19 +1,14 @@ // The one way a conversation's journal becomes open on this host: for a send, for a reader, and // for an attach that finds none open. // -// A damaged journal fails the open, which refuses it as unloadable. It marks what an earlier host -// process handed over and left unanswered as in doubt, and settles what it left running — the -// crash boundary. That -// needs no lease: provider history decides such a row later, under a won lease, in the attach. A -// row an earlier process accepted and never handed over (it quit or crashed first) is settled here -// too, before any reader, command or child sees it: a person's message is kept as a card, the rest -// rejected (`journal-unsent-send-hold.ts`). The cards then wait for the chat's next turn -// (`queued-message-pause.ts`). Nothing here starts a provider child. +// It only reads. A damaged journal fails the open, which refuses it as unloadable. What an earlier +// generation or host process left running or queued is settled by the ownership event that ended +// it — its exit, the next acquisition, or startup (`structured-agent-session-leftover-settlement.ts`, +// `structured-agent-session-startup-settlement.ts`) — never here. Nothing here starts a provider +// child. import type { JournalHostDatabase } from '../agent-session-journal/journal-host-database' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { @@ -22,12 +17,6 @@ import { type AgentSessionAttachParams } from './structured-agent-session-attach' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' -import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import { - markStructuredQueueReopen, - structuredAgentSessionHostInstance -} from './structured-agent-session-queued-pause' import type { StructuredAgentSessionHostDeps, StructuredAgentSessionHostSession @@ -43,12 +32,6 @@ export type StructuredAgentSessionConversationOpenDeps = { logger: StructuredAgentSessionHostDeps['logger'] } -/** An acquisition's own open: its reserve cleared the record's death evidence, so it settles - * what the gone generation left running itself, from what it read before. */ -export type StructuredAgentSessionConversationOpenOptions = { - acquisition?: boolean -} - export type StructuredAgentSessionConversationOpenContext = { deps: StructuredAgentSessionConversationOpenDeps sessions: Map @@ -63,8 +46,7 @@ export type StructuredAgentSessionConversationOpenContext = { * session's serialize, which is what makes "not open yet" exact. */ export async function openStructuredAgentSessionConversation( context: StructuredAgentSessionConversationOpenContext, - sessionId: string, - options: StructuredAgentSessionConversationOpenOptions = {} + sessionId: string ): Promise { const open = context.sessions.get(sessionId) if (open) { @@ -74,103 +56,26 @@ export async function openStructuredAgentSessionConversation( if (!record) { return null } - const opened = await openStructuredAgentSessionConversationJournal(context.deps, record, options) + const opened = await openStructuredAgentSessionConversationJournal(context.deps, record) await context.adoptOpened(sessionId, opened) return opened.session } /** The open itself, indexed by nobody yet: the caller adopts the result. */ export async function openStructuredAgentSessionConversationJournal( - deps: Omit, - record: AgentSessionRecord, - options: StructuredAgentSessionConversationOpenOptions = {} + deps: Omit, + record: AgentSessionRecord ): Promise { const { sessionId } = record - const fence = record.lease.runtimeFence const params = attachParamsForRecord(record, { clientOperationId: `read-restore:${sessionId}`, - expectedRuntimeFence: fence + expectedRuntimeFence: record.lease.runtimeFence }) const identity = journalIdentityFor(record, params) const journal = await openAgentSessionJournal({ identity, database: deps.journalDatabase }) - try { - // A handed-over row found here is only doubt, which provider history decides under a won lease. - await journal.markPendingSubmissionsUnknown(fence) - } catch (error) { - deps.logger.warn('marking pending sends unknown on open failed', { - scope: 'open-pending-unknown', - sessionId, - error - }) - } - try { - // Before a Stop this open serves can withdraw one: a Stop never withdraws a card. - await holdUnsentSends(journal, { - fence, - hostInstance: structuredAgentSessionHostInstance(), - hold: { cause: 'hostRestarted' } - }) - } catch (error) { - // The row stays queued. The delivery loop's first step tries again before it hands anything - // over; if that fails too, the loop fails and rejects every queued send. - deps.logger.warn('settling sends an earlier process left queued failed on open', { - scope: 'open-leftover-sends', - sessionId, - error - }) - } - // After the leftovers became cards, so the mark follows every card this open found. Every open - // comes after the chat stopped running: the idle sweep never closes one with cards waiting. - await markStructuredQueueReopen(sessionId, journal, fence, deps.logger) - // No child in this process writes to a journal nobody had open, so whatever it shows running - // belongs to a generation that is gone, whatever the lease still claims. Settled before any - // reader or child sees it. - if (!options.acquisition) { - await settleGoneGeneration(deps, record, journal) - } return { session: { journal, params, child: null } } } -/** - * The open's settle again, for a conversation already open: a proof of death written since it - * opened (the startup reconcile, a recovery) revises what the open could only call `unverifiable`. - * A record holds a proof only while released, so no child here is writing. A no-op once revised. - */ -export async function resettleOpenStructuredAgentSessionConversation( - deps: StructuredAgentSessionConversationOpenDeps, - sessionId: string, - session: StructuredAgentSessionHostSession | undefined -): Promise { - const record = deps.store.getRecord(sessionId) - if (session && record?.lease.deathEvidence) { - await settleGoneGeneration(deps, record, session.journal) - } -} - -async function settleGoneGeneration( - deps: Pick, - record: AgentSessionRecord, - journal: AgentSessionJournal -): Promise { - try { - await settleStaleStructuredAgentSessionState({ - journal, - sessionId: record.sessionId, - fence: record.lease.runtimeFence, - acquisitionGeneration: null, - deathEvidence: record.lease.deathEvidence ?? null, - failureTextContext: structuredAgentSessionFailureWordsContext(record) - }) - } catch (error) { - // Best effort: the next open or acquire re-derives it. - deps.logger.warn("settling a gone agent's work on open failed", { - scope: 'open-dead-generation', - sessionId: record.sessionId, - error - }) - } -} - export function attachParamsForRecord( record: AgentSessionRecord, input: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index a36518c5e614..727e9108289a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -36,12 +36,10 @@ import { AgentSessionAcquisitionExitUnprovenError, type StructuredAgentSessionAdapter } from './structured-agent-session-adapter' -import { resettleOpenStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' import { StructuredAgentSessionHost } from './structured-agent-session-host' import { withNativeChatCutTurnNotices } from '../../../shared/native-chat-cut-turn-notice' import { latestNativeChatOrcaStopCut } from '../../../shared/native-chat-orca-stop-cut' import { beginAgentSessionRuntimeRecord } from '../../runtime/agent-session-runtime-end-record' -import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idle-sweep' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { hostTestAttachParams, @@ -104,7 +102,8 @@ function crashedClaudeRecord(): AgentSessionRecord { claimKeyId: 'key-1', claimStatus: 'live', unreconciled: false, - deathEvidence: null + deathEvidence: null, + leftoverSettledAt: null } } } @@ -302,10 +301,16 @@ function drainSession(): Promise { return host.collaboratorsForTests().serialize(SESSION, async () => {}) } -// On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner, -// so the open can only call the turn unverifiable; the proof, whoever writes it, then revises it. +/** Startup: the reconcile, then the settlement it begins. */ +async function startUp(): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() +} + +// On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner. +// The open writes nothing; the startup after the reconcile settles the turn by its proof. describe('a turn a read reached before the reconcile proved its owner dead', () => { - it('reads unverifiable, then interrupted at the last renewal, and a subscriber is sent both', async () => { + it('reads as the crash left it, then interrupted at the last renewal, and a subscriber is sent both', async () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) const events: AgentSessionSubscribeEvent[] = [] const unsubscribe = await host.subscribe({ @@ -313,15 +318,14 @@ describe('a turn a read reached before the reconcile proved its owner dead', () sessionId: SESSION, emit: (event) => events.push(event) }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toEqual(RUNNING_TURN) - await host.reconcileRestartLeases() - await drainSession() + await startUp() // A paired or phone client learns of the revision through the ordinary journal publish. await vi.waitFor(() => expect(turnStatesSent(events)).toMatchObject([ - { state: 'unverifiable' }, + { state: 'running' }, { state: 'interrupted', completedAt: LAST_RENEWED_AT } ]) ) @@ -343,10 +347,10 @@ describe('a turn a read reached before the reconcile proved its owner dead', () published .filter((summary) => summary.sessionId === SESSION && summary.turnOutcome) .map((summary) => summary.turnOutcome) - expect(outcomes().at(-1)).toBe('unconfirmed') + // Nothing settled yet: the turn has no outcome to report. + expect(outcomes()).toEqual([]) - await host.reconcileRestartLeases() - await drainSession() + await startUp() // The sidebar's red Failed until seen; the turn folds as "Worked for 27s" beside its notice row. await vi.waitFor(() => expect(outcomes().at(-1)).toBe('interruption')) @@ -380,10 +384,9 @@ describe('a turn a read reached before the reconcile proved its owner dead', () beginAgentSessionRuntimeRecord(root, 'runtime-crashed', TOOL_STARTED_AT - 60_000) store = await openTestAgentSessionRecordStore(root) openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toEqual(RUNNING_TURN) - await host.reconcileRestartLeases() - await drainSession() + await startUp() const { items } = await host.journalSnapshot(SESSION) const turnItemId = items.find((item) => item.body.kind === 'turn')?.itemId @@ -400,19 +403,11 @@ describe('a turn a read reached before the reconcile proved its owner dead', () it('revises nothing twice, whoever re-runs the settle', async () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) await host.history({ sessionId: SESSION, direction: 'tail' }) - await host.reconcileRestartLeases() - await drainSession() + await startUp() const settled = await host.journalSnapshot(SESSION) - await host - .collaboratorsForTests() - .serialize(SESSION, () => - resettleOpenStructuredAgentSessionConversation( - host.deps, - SESSION, - host.collaboratorsForTests().sessions.get(SESSION) - ) - ) + // A second startup pass and the restore's share find nothing left to do. + await startUp() await host.restoreReadableSessions() expect(await host.journalSnapshot(SESSION)).toEqual(settled) @@ -441,7 +436,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () stopOwnerProcess }) await host.history({ sessionId: SESSION, direction: 'tail' }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toEqual(RUNNING_TURN) // The reconcile only parks the live orphan in recovery; recovery's stop is what proves it gone. await host.restoreReadableSessions() @@ -510,27 +505,21 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ]) }) - it('stays unverifiable when the revision cannot be written, and a later open revises it', async () => { - let now = RELAUNCHED_AT + it('stays as it was when the startup settlement cannot be written, and the next trigger settles it', async () => { const log = recordingStructuredAgentSessionLogger() - openHost({ - probeOwner: async () => ({ outcome: 'pid-absent' }), - now: () => now, - logger: log.logger - }) + openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }), logger: log.logger }) await host.history({ sessionId: SESSION, direction: 'tail' }) const { journal } = host.collaboratorsForTests().sessions.get(SESSION)! - vi.spyOn(journal, 'appendLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + vi.spyOn(journal, 'appendPlannedLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) - await host.reconcileRestartLeases() - await drainSession() + await startUp() - expect(log.scopes()).toEqual(['open-dead-generation']) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) - // The proof is durable on the record, so the next open converges. - now += STRUCTURED_AGENT_SESSION_IDLE_MS + 1 - await host.collaboratorsForTests().lifetime.idleSweep.tick() - expect(host.hasSession(SESSION)).toBe(false) + expect(log.scopes()).toEqual(['startup-settlement']) + expect(await settledTurn()).toEqual(RUNNING_TURN) + expect(store.getRecord(SESSION)?.lease.leftoverSettledAt).toBeNull() + // The proof is durable on the record and the mark still owes it: the restore's startup share + // converges, as would the next acquisition. + await host.restoreReadableSessions() expect(await settledTurn()).toMatchObject({ state: 'interrupted', completedAt: LAST_RENEWED_AT @@ -583,15 +572,23 @@ async function hostWithFailingFirstStart(failure: Error) { } } -/** The turn as the crashed owner left it, with no end: nothing proves when that owner stopped. */ -const UNVERIFIABLE_TURN = { +/** The turn as the crashed owner left it, before anything settled it. */ +const RUNNING_TURN = { turnId: 'turn-1', - state: 'unverifiable', + state: 'running', startedAt: TOOL_STARTED_AT - 2_000 } -// The relaunch proved the fence-13 owner gone, but a start reserving fence 15 clears that proof -// before the turn is settled; what the record holds afterwards is about the start's own child. +/** The turn the relaunch's proof of the fence-13 owner's death settled at startup. */ +const PROVEN_TURN = { + turnId: 'turn-1', + state: 'interrupted', + startedAt: TOOL_STARTED_AT - 2_000, + completedAt: LAST_RENEWED_AT +} + +// The relaunch proved the fence-13 owner gone, and startup settled the turn by that proof before any +// start; what the record holds after a start reserving fence 15 fails is about the start's own child. describe('a turn a newer start could not settle before it failed', () => { it('is not judged by the death of the child it left for recovery', async () => { const { attach, advance } = await hostWithFailingFirstStart( @@ -613,7 +610,7 @@ describe('a turn a newer start could not settle before it failed', () => { }) await expect(attach(16)).resolves.toMatchObject({ ok: true }) - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + expect(await settledTurn()).toEqual(PROVEN_TURN) }) it('is not judged by the watched exit of a start that failed', async () => { @@ -628,7 +625,7 @@ describe('a turn a newer start could not settle before it failed', () => { advance(60_000) await expect(attach(16)).resolves.toMatchObject({ ok: true }) - // Main ended it at the failed start, an hour after the crash, with the start's reason. - expect(await settledTurn()).toEqual(UNVERIFIABLE_TURN) + // Never ended at the failed start, an hour after the crash, with the start's reason. + expect(await settledTurn()).toEqual(PROVEN_TURN) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index 07510c754e1c..ba466bf7dcb0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -279,7 +279,11 @@ describe('dead structured-session generation settlement', () => { const settledSnapshot = journal.snapshot() const closedJournal: Pick< AgentSessionJournal, - 'snapshot' | 'submissions' | 'rejectQueuedSubmissions' | 'appendLifecycleBatch' | 'itemFence' + | 'snapshot' + | 'submissions' + | 'rejectQueuedSubmissions' + | 'appendPlannedLifecycleBatch' + | 'itemFence' > = { itemFence: () => undefined, snapshot: () => ({ @@ -290,7 +294,7 @@ describe('dead structured-session generation settlement', () => { rejectQueuedSubmissions: async () => { throw new Error('journal_closed') }, - appendLifecycleBatch: async () => { + appendPlannedLifecycleBatch: async () => { throw new Error('journal_closed') } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index c9742e18881d..dd685da629c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -3,7 +3,6 @@ import { MAX_PROVIDER_DIAGNOSTIC_CHARS, type SubmissionRejectionFact } from '../../../shared/agent-session-failure' -import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { AGENT_JOURNAL_THREAD_SCOPE, @@ -20,7 +19,9 @@ import { runningCallEnd, terminalAgentJournalBody } from '../agent-session-journal/journal-terminal-settlement' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' +import type { ResolveDispatchInput } from '../agent-session-journal/journal-store-contracts' +import { lostLiveWorkJournalBody } from '../agent-session-journal/journal-subagent-liveness' import { agentSessionFailureWords, type AgentSessionFailureWordsContext @@ -30,30 +31,21 @@ import { hasStructuredAgentSessionStartFailureRow, structuredAgentSessionStartFailureRow } from './structured-agent-session-start-failure-row' -import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { - endedByPersonsStop, - provenUnverifiableTurnRevisions, - provenUnverifiedToolCallRevisions, runningTurnLifecycleRevisions, - stopFoundTurnLiveAt, - turnVerdictFromDeathEvidence, watchedExitRevisions, type StructuredAgentSessionTurnVerdict, type StructuredAgentSessionWatchedExit } from './structured-agent-session-stale-turn-verdict' -import { - exitedRootTurnScope, - settledRootTurnScope -} from './structured-agent-session-exit-turn-scope' -import { orcaStopRowBody } from './structured-agent-session-orca-stop-row' +import { exitedRootTurnScope } from './structured-agent-session-exit-turn-scope' import { hasUnfinishedStructuredAgentSessionWork, - isInProgressStructuredAgentSessionItem, type DeadGenerationJournal } from './structured-agent-session-unfinished-work' import { codexUnopenedSendResolutions } from './structured-agent-session-unopened-send-withdrawal' +export { settleStaleStructuredAgentSessionState } from './structured-agent-session-stale-state-settlement' + /** Bounds the exit reason the lease keeps as log evidence; a provider diagnostic is held to the * same cap. */ export const MAX_UNEXPECTED_EXIT_REASON_CHARS = MAX_PROVIDER_DIAGNOSTIC_CHARS @@ -63,9 +55,10 @@ export type StructuredAgentSessionDeadGenerationSettlement = | { ok: true } | { ok: false; error: unknown } -export async function settleStructuredAgentSessionDeadGeneration(input: { +export type StructuredAgentSessionDeadGenerationInput = { journal: DeadGenerationJournal sessionId: string + /** The fence the rows are written at: the lease's current one once the exit released it. */ fence: number settlementId: string verdict: StructuredAgentSessionTurnVerdict @@ -83,90 +76,28 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { exit?: StructuredAgentSessionWatchedExit /** A person's Stop ended a starting child before what it was handed could run: each is rejected so. */ unrunRejection?: SubmissionRejectionFact -}): Promise { + /** Committed in the same transaction as the rows, even when nothing is left to settle. */ + receipt?: JournalOperationReceipt +} + +/** The exited child's own settlement, planned at the batch's own turn in the write queue. */ +export async function settleStructuredAgentSessionDeadGeneration( + input: StructuredAgentSessionDeadGenerationInput +): Promise { try { - const hasUnfinishedWork = hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) - const showUnexpectedExitOutcome = input.showUnexpectedExitOutcome ?? hasUnfinishedWork - if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { + if ( + !input.receipt && + !input.showUnexpectedExitOutcome && + !hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) + ) { return { ok: true } } - // A queued message is the delivery loop's to settle: it was never handed to this child. A send - // a child still starting was handed and never echoed did not run, and its root is gone: it is - // rejected, with the child's own diagnostic or as the Stop that ended it. A proven child's - // handed-over sends stay in doubt. - const startupFailure = input.exitedDuringStartup - ? structuredAgentSessionStartFailure({ exit: input.exitFailure }, input.failureTextContext) - : null - const closed = input.unrunRejection - const unrun = - startupFailure ?? (closed && agentSessionFailureWords(closed, { surface: 'rejection' })) - const withdrawn = unrun ? [] : codexUnopenedSendResolutions(input.journal, input.fence) - const withdrawnIds = new Set(withdrawn.map((entry) => entry.clientMessageId)) - const dispatches = [ - ...withdrawn, - ...journalPendingSubmissionResolutions( - (input.journal.submissions?.() ?? input.journal.pendingSubmissions?.() ?? []).filter( - (entry) => !withdrawnIds.has(entry.clientMessageId) - ), - input.fence, - unrun ? { rejection: unrun } : { reason: input.pendingSubmissionReason } - ) - ] - const items = input.journal.snapshot().items - const proven = watchedExitRevisions(items, input.exit, input.journal) - const mutations: JournalLifecycleMutationInput[] = [] - if (showUnexpectedExitOutcome && input.exitedDuringStartup && startupFailure) { - const startKey = input.exitedDuringStartup.generation ?? input.settlementId - // A start a message waited on is the delivery loop's to record, before or after this exit, - // in the words it rejected the message with; this row is for a command, goal or rewind start. - // A row already written stays: rejected is terminal, so its words are not reworded. - const recordedByDeliveryLoop = - input.journal.submissions?.().some(isQueuedAgentJournalSubmission) || - hasStructuredAgentSessionStartFailureRow(items, startKey) - if (!recordedByDeliveryLoop) { - mutations.push(structuredAgentSessionStartFailureRow(startKey, startupFailure)) - } - } else if (showUnexpectedExitOutcome) { - // The turn the exit ended, and an error so no fold ever hides why it stopped. - mutations.push({ - kind: 'item', - identity: { provider: 'orca', clientMessageId: input.settlementId }, - body: { - kind: 'status', - ...agentSessionFailureWords( - input.exitFailure ?? agentSessionFailureFact('providerExited'), - { - ...input.failureTextContext, - surface: 'row' - } - ), - tone: 'error' - }, - turnScope: exitedRootTurnScope(withRevisions(items, proven), input.verdict) - }) - } - const bodies = new Map(items.map((item) => [item.itemId, item.body])) - for (const item of items) { - // Ended as its turn is: a proven death cuts a running call short. An open reasoning row is - // ended too, but is not unfinished work: its running turn already says so. - const end = runningCallEnd(item.turnScope, (id) => bodies.get(id), input.verdict.state) - const body = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) - if (body) { - mutations.push({ - kind: 'item', - itemId: item.itemId, - body, - turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE - }) - } - } - mutations.push(...runningTurnLifecycleRevisions(items, input.verdict), ...proven) - await input.journal.appendLifecycleBatch({ + await input.journal.appendPlannedLifecycleBatch({ settlementId: `dead-generation:${input.settlementId}`, fence: input.fence, recovered: true, - mutations, - dispatches + ...(input.receipt ? { receipt: input.receipt } : {}), + plan: () => planStructuredAgentSessionDeadGeneration(input) }) return { ok: true } } catch (error) { @@ -175,42 +106,81 @@ export async function settleStructuredAgentSessionDeadGeneration(input: { } } -/** - * Settles whatever a generation with no child in this process left running: found when a new child - * is acquired, or when a chat is reopened for reading. Derived from the journal and the lease's - * death evidence each time, so nothing is owed in between. Proven death ends the turn interrupted, - * and a proof written after an earlier settle revises what that settle left `unverifiable`, the - * turn and the calls it closed alike. Must run before a new child's buffered events land, or a - * live turn would be judged. - */ -export async function settleStaleStructuredAgentSessionState(input: { - journal: AgentSessionJournal - sessionId: string - fence: number - acquisitionGeneration: string | null - deathEvidence: AgentSessionDeathEvidence | null - /** Who the exit row names. */ - failureTextContext?: AgentSessionFailureWordsContext -}): Promise { - const { journal } = input - const items = journal.snapshot().items - // Each turn is judged by the evidence only if it names that turn's owner. - const verdictFor = (item: AgentJournalRenderItem) => - turnVerdictFromDeathEvidence( - input.deathEvidence, - journal.itemFence(item.itemId), - stopFoundTurnLiveAt(journal, item) +/** The exit's rows, read from the journal as it stands. `owns`: which fences are the exited + * generation's (and later); the rest are an earlier generation's, settled by its own rule. */ +export function planStructuredAgentSessionDeadGeneration( + input: StructuredAgentSessionDeadGenerationInput, + owns: (fence: number | undefined) => boolean = () => true +): { mutations: JournalLifecycleMutationInput[]; dispatches: ResolveDispatchInput[] } { + const hasUnfinishedWork = hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) + const showUnexpectedExitOutcome = input.showUnexpectedExitOutcome ?? hasUnfinishedWork + if (!showUnexpectedExitOutcome && !hasUnfinishedWork) { + return { mutations: [], dispatches: [] } + } + // A queued message is the delivery loop's to settle: it was never handed to this child. A send + // a child still starting was handed and never echoed did not run, and its root is gone: it is + // rejected, with the child's own diagnostic or as the Stop that ended it. A proven child's + // handed-over sends stay in doubt. + const startupFailure = input.exitedDuringStartup + ? structuredAgentSessionStartFailure({ exit: input.exitFailure }, input.failureTextContext) + : null + const closed = input.unrunRejection + const unrun = + startupFailure ?? (closed && agentSessionFailureWords(closed, { surface: 'rejection' })) + const withdrawn = unrun ? [] : codexUnopenedSendResolutions(input.journal, input.fence) + const withdrawnIds = new Set(withdrawn.map((entry) => entry.clientMessageId)) + const dispatches = [ + ...withdrawn, + ...journalPendingSubmissionResolutions( + (input.journal.submissions?.() ?? input.journal.pendingSubmissions?.() ?? []).filter( + (entry) => !withdrawnIds.has(entry.clientMessageId) && owns(entry.fence) + ), + input.fence, + unrun ? { rejection: unrun } : { reason: input.pendingSubmissionReason } ) - // A successful settlement leaves terminal items; a failed transaction leaves the same prefix. - const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` - const settlementId = `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}` - // Calls an earlier settle closed with no proof, revised once a proof names their owner. - const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal) + ] + const allItems = input.journal.snapshot().items + const items = allItems.filter((item) => owns(input.journal.itemFence(item.itemId))) + const proven = watchedExitRevisions(items, input.exit, input.journal) + const mutations: JournalLifecycleMutationInput[] = [] + if (showUnexpectedExitOutcome && input.exitedDuringStartup && startupFailure) { + const startKey = input.exitedDuringStartup.generation ?? input.settlementId + // A start a message waited on is the delivery loop's to record, before or after this exit, + // in the words it rejected the message with; this row is for a command, goal or rewind start. + // A row already written stays: rejected is terminal, so its words are not reworded. + const recordedByDeliveryLoop = + input.journal.submissions?.().some(isQueuedAgentJournalSubmission) || + hasStructuredAgentSessionStartFailureRow(allItems, startKey) + if (!recordedByDeliveryLoop) { + mutations.push(structuredAgentSessionStartFailureRow(startKey, startupFailure)) + } + } else if (showUnexpectedExitOutcome) { + // The turn the exit ended, and an error so no fold ever hides why it stopped. + mutations.push({ + kind: 'item', + identity: { provider: 'orca', clientMessageId: input.settlementId }, + body: { + kind: 'status', + ...agentSessionFailureWords( + input.exitFailure ?? agentSessionFailureFact('providerExited'), + { + ...input.failureTextContext, + surface: 'row' + } + ), + tone: 'error' + }, + turnScope: exitedRootTurnScope(withRevisions(items, proven), input.verdict) + }) + } + const bodies = new Map(allItems.map((item) => [item.itemId, item.body])) for (const item of items) { - // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running - // leaves them to the evidence. - const end = runningCallEnd(item.turnScope, (id) => journal.itemBody(id), verdictFor(item).state) - const body = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) + // Ended as its turn is: a proven death cuts a running call short. An open reasoning row is + // ended too, but is not unfinished work: its running turn already says so. + const end = runningCallEnd(item.turnScope, (id) => bodies.get(id), input.verdict.state) + const terminal = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) + // One revision per item: its live subagents and background tasks end with it. + const body = lostLiveWorkJournalBody(terminal ?? item.body) ?? terminal if (body) { mutations.push({ kind: 'item', @@ -220,44 +190,8 @@ export async function settleStaleStructuredAgentSessionState(input: { }) } } - const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal) - const turnEnds = [ - ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), - ...proven - ] - mutations.push(...turnEnds) - const evidence = input.deathEvidence - if ( - evidence && - (proven.length > 0 || - items.some( - (item) => - isInProgressStructuredAgentSessionItem(item) && verdictFor(item).state === 'interrupted' - )) && - !endedByPersonsStop(journal, turnEnds) - ) { - mutations.unshift({ - kind: 'item', - // Named by the death it explains, so a retry after a partly written settle adds no second row. - identity: { - provider: 'orca', - clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` - }, - // The death evidence is Orca's log text, never a sentence for a person: the row says only - // that the provider stopped, and how Orca ended when the provider died with it. - body: orcaStopRowBody(input.failureTextContext, evidence.runtimeEnd), - turnScope: settledRootTurnScope(items, turnEnds) - }) - } - if (mutations.length > 0) { - await journal.appendLifecycleBatch({ - settlementId, - fence: input.fence, - recovered: true, - mutations - }) - } - return mutations.length + mutations.push(...runningTurnLifecycleRevisions(items, input.verdict), ...proven) + return { mutations, dispatches } } function withRevisions( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts index 6fad454e993f..2dd6fa8c926f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts @@ -88,7 +88,8 @@ function ownerRecord(): AgentSessionRecord { claimKeyId: 'fixture-key', claimStatus: 'live', unreconciled: false, - deathEvidence: null + deathEvidence: null, + leftoverSettledAt: null } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 9ff8543d514e..dbc86bba0676 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -7,9 +7,7 @@ import { holdClosedStructuredAgentSessionSends } from './structured-agent-sessio import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openStructuredAgentSessionConversation, - resettleOpenStructuredAgentSessionConversation, - type OpenedStructuredAgentSessionConversation, - type StructuredAgentSessionConversationOpenOptions + type OpenedStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' import type { StructuredAgentSessionClientDelivery } from './structured-agent-session-client-delivery' import { StructuredAgentSessionDeliveryLoop } from './structured-agent-session-delivery-loop' @@ -21,16 +19,12 @@ import type { } from './structured-agent-session-host-types' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import { recoverStructuredRewind } from './structured-rewind-recovery' import { retireSignedOutStructuredAgentSessionChild } from './structured-agent-session-signed-out-child' export type StructuredAgentSessionConversationDelivery = { loop: StructuredAgentSessionDeliveryLoop /** For a caller inside the session's serialize. */ - open: ( - sessionId: string, - options?: StructuredAgentSessionConversationOpenOptions - ) => Promise + open: (sessionId: string) => Promise /** Every commit a conversation's journal makes: one may have ended the command that held its * queue. Enqueued through the session's serialize, never read here, so a commit that lands while * a step is deciding to stop wakes the loop after that step rather than being lost to it. */ @@ -38,7 +32,7 @@ export type StructuredAgentSessionConversationDelivery = { /** A person's Stop settle opened or closed. It writes no row, so only what reads the settle * moves: the session's status row and the steer hold's handover. Never activity. */ afterSettleEdge: (sessionId: string, journal: AgentSessionJournal) => void - /** Stops the loop and the resettle on a proof of death; quit's first step. */ + /** Stops the loop; quit's first step. */ dispose: () => void /** Indexes a conversation some other open produced, as `open` would have. */ adoptOpened: ( @@ -117,13 +111,10 @@ export function createStructuredAgentSessionConversationDelivery(input: { sessionId: string, opened: OpenedStructuredAgentSessionConversation ): Promise => { - const { session } = opened - sessions.set(sessionId, session) + // Indexing a chat writes nothing and wakes nothing: what an earlier process left queued is + // startup's to hold, and the loop's next wake holds whatever that missed before any hand-over. + sessions.set(sessionId, opened.session) input.clientDelivery.publishRestored(sessionId) - await settleInterruptedCommands(deps, sessionId, session) - if (session.journal.submissions().some(isQueuedAgentJournalSubmission)) { - loop.wake(sessionId) - } } const wakesQueued = new Set() const afterCommit = (sessionId: string, journal: AgentSessionJournal): void => { @@ -149,25 +140,6 @@ export function createStructuredAgentSessionConversationDelivery(input: { }) }) } - // A chat open before its owner's death was proven revises what its open settled. Queued, never - // awaited: the writer can hold this session's serialize (an attach recovering its lease). - const stopResettling = deps.store.onDeathEvidence((sessionId) => { - if (sessions.has(sessionId)) { - void input - .trackStart( - input.serialize(sessionId, () => - resettleOpenStructuredAgentSessionConversation(deps, sessionId, sessions.get(sessionId)) - ) - ) - .catch((error: unknown) => - deps.logger.warn('resettling an open chat after its owner died failed', { - scope: 'death-evidence-resettle', - sessionId, - error - }) - ) - } - }) return { loop, afterCommit, @@ -176,34 +148,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { afterCommit(sessionId, journal) }, adoptOpened, - dispose: () => { - loop.dispose() - stopResettling() - }, - open: (sessionId, options) => - openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId, options) - } -} - -/** - * A rewind found prepared when the conversation opens was started under a child this process no - * longer has — the open runs only when none is indexed — so nothing will finish it, and left alone - * it refuses every send until a view attaches. Settled here instead of by a start inside - * acceptance. A Codex rewind only its provider can prove stays for the attach. - */ -async function settleInterruptedCommands( - deps: StructuredAgentSessionHostDeps, - sessionId: string, - session: StructuredAgentSessionHostSession -): Promise { - const fence = structuredAgentSessionConversationFence(deps.store, sessionId) - try { - await recoverStructuredRewind(deps, sessionId, session.journal, fence) - } catch (error) { - deps.logger.warn('settling an interrupted rewind on open failed', { - scope: 'rewind-recovery', - sessionId, - error - }) + dispose: () => loop.dispose(), + open: (sessionId) => + openStructuredAgentSessionConversation({ deps, sessions, adoptOpened }, sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index 8e072e714b6d..ae1e1619380e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -195,6 +195,7 @@ export async function stopStructuredAgentSessionAgentUnderSerialize( journal: session.journal, sessionId, fence: child.fence, + rowFence: Math.max(child.fence, record?.lease.runtimeFence ?? child.fence), generation: child.generation ?? 'unknown', turnItemId: quitCuts, trigger: ending.quit, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts index dd4d0976bd88..c87ce9d485bc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts @@ -53,7 +53,8 @@ function reservedRecord(): AgentSessionRecord { claimKeyId: 'key-1', claimStatus: 'reserved', unreconciled: false, - deathEvidence: null + deathEvidence: null, + leftoverSettledAt: null }, createdAt: NOW, updatedAt: NOW diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts index d9b8e068c37e..1d98c835ca50 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts @@ -65,19 +65,19 @@ export async function stopEndsWork( const working = (drain === 'slow' && journal.stopMarks.latestAcceptedSendUnopened()) || isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), + journal.activeTurnId(child.fence), journal.submissions(), child.fence ) - return working && (ending.cause === 'user-close' || !defersToPersonsStop(session)) + return working && (ending.cause === 'user-close' || !defersToPersonsStop(session, child.fence)) } /** A host stop of work a person's Stop is already ending must not supersede that Stop's reason: * the turn it decides, or, with no turn running, the work under the queue pause it still holds, * with nothing sent since, which a host's event would lift. */ -function defersToPersonsStop(session: StructuredAgentSessionHostSession): boolean { +function defersToPersonsStop(session: StructuredAgentSessionHostSession, fence: number): boolean { const { journal } = session - const live = journal.activeTurnId() + const live = journal.activeTurnId(fence) if (live !== null) { return journal.stopMarks.personStopDecides(live) } @@ -101,12 +101,10 @@ export function recordStopEvent( if ('recorded' in ending) { return Promise.resolve(null) } - const turnId = session.journal.activeTurnId() + const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) + const turnId = session.journal.activeTurnId(fence) return session.journal - .appendStopEvent( - { reason: ending.cause, ...(turnId !== null ? { turnId } : {}) }, - structuredAgentSessionConversationFence(context.deps.store, sessionId) - ) + .appendStopEvent({ reason: ending.cause, ...(turnId !== null ? { turnId } : {}) }, fence) .then( () => (ending.cause === 'user-close' ? session.journal.stopMarks.beginSettle() : null), (error: unknown) => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 251ccc88e32b..2bd92b56a60d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -136,7 +136,8 @@ export class StructuredAgentSessionHost { // Site 10: cannot overwrite a live entry — the restorer returns early on // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened, - onUnopened: (sessionId) => this.tabs.markUnopened(sessionId) + onUnopened: (sessionId) => this.tabs.markUnopened(sessionId), + startup: { sessions: this.sessions, tasks: this.tasks } }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -234,6 +235,8 @@ export class StructuredAgentSessionHost { onSessionsHeld = (listener: () => void): (() => void) => this.deps.store.onFirstRecord(listener) reconcileRestartLeases = (): Promise => this.restore.reconcileRestartLeases() + /** Once the settlement the last startup reconcile began has run for every chat. */ + startupSettled = (): Promise => this.restore.startupSettled() restoreReadableSessions = (sessionIds?: readonly string[]): Promise => this.restore.restoreReadableSessions(sessionIds) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts new file mode 100644 index 000000000000..d8f224b080ba --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -0,0 +1,176 @@ +// What the leftover settlement never touches: an open or a re-attach to the live child writes +// nothing, a release nothing proved settles only as `unverifiable`, and an owner that may still run +// keeps its current work, which still holds the chat. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION +} from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueueHold } from './structured-agent-session-queued-messages' +import { + APPROVAL, + currentJournal, + exitWhileSettlementFails, + leaveUnfinishedWork, + logEveryWrite, + recoveredRows, + sendText, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Every row of every table, as stored. */ +function everyRow(db: ReturnType['db']): unknown[] { + return db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + .all() + .map((table) => [table.name, db.prepare(`SELECT * FROM main."${String(table.name)}"`).all()]) +} + +/** Whether the chat reads Working and holds a queue-if-active send, by its current fence. */ +function holds(current: QueuedMessageTestRig): { working: boolean; queueHeld: boolean } { + const journal = currentJournal(current) + const record = current.store.getRecord(SESSION) + const fence = record?.lease.runtimeFence ?? 0 + return { + working: isStructuredAgentSessionMainAgentWorking( + journal.activeTurnId(fence), + journal.submissions(), + fence + ), + queueHeld: structuredQueueHold({ journal, record, fence }) !== null + } +} + +describe('reading a chat', () => { + it('opens a chat whose leftovers are owed with zero writes, by every reader', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + // Answered, so no send left pending keeps the conversation open. + await current.settleAccepted(await current.workingSend(), 'work') + await leaveUnfinishedWork(current, { prompt: true }) + const { database } = await exitWhileSettlementFails(current) + database.db.exec('DROP TRIGGER reject_recovered') + await current.host.close(SESSION, 'evict') + expect(current.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) + const writes = logEveryWrite(database.db) + + await current.host.revealSession(SESSION) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + await current.host.journalSnapshot(SESSION) + const unsubscribe = await current.host.subscribe({ + id: 'reader', + sessionId: SESSION, + emit: () => undefined + }) + unsubscribe() + + expect(writes()).toEqual([]) + // Still owed, and still as the dead generation left it: only an ownership event settles it. + expect(current.store.getRecord(SESSION)?.lease.leftoverSettledAt).toBeNull() + expect(turnState(current)).toBe('running') + expect(holds(current)).toEqual({ working: false, queueHeld: false }) + }) + + it('re-attaches to the live child with zero writes', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { db } = openTestJournalHostDatabase(current.root) + const before = everyRow(db) + const writes = logEveryWrite(db) + + expect(await current.host.attach(CALLER, current.firstAttach)).toMatchObject({ ok: true }) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + + // The replayed operation's ledger row is upserted with the same bytes; nothing else is touched. + expect(writes().filter((entry) => entry !== 'UPDATE agent_session_operations')).toEqual([]) + expect(everyRow(db)).toEqual(before) + expect(turnState(current)).toBe('running') + }) +}) + +describe('a release nothing proved', () => { + it('settles what it left as unverifiable at startup, and the next send is delivered', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + + // The rig's probe answers indeterminate: the restart releases the owner with no proof. + await current.crashRestartHostProcess() + + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'released', + deathEvidence: null, + leftoverSettledAt: NOW + }) + expect(turnState(current)).toBe('unverifiable') + expect(currentJournal(current).itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ + resolution: { state: 'cancelled' } + }) + expect(holds(current)).toEqual({ working: false, queueHeld: false }) + + expect((await sendText(current, 'carry on').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an owner that may still run', () => { + it('keeps an unreconciled owner’s current work, which still holds, and settles nothing', async () => { + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => { + throw new Error('the probe could not run') + } + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + + // The restart cannot reconcile: no proof, no release. + await current.crashRestartHostProcess() + + expect(current.store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') + expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual([]) + expect(turnState(current)).toBe('running') + expect(holds(current)).toEqual({ working: true, queueHeld: true }) + }) + + it('leaves a live owner’s work unchanged at every trigger but its own exit', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const database = openTestJournalHostDatabase(current.root) + + // A startup pass and a second attach both find the owner live. + await current.host.reconcileRestartLeases() + await current.host.startupSettled() + expect(await current.host.attach(CALLER, current.firstAttach)).toMatchObject({ ok: true }) + const queued = await sendText(current, 'behind the live turn').result + + expect(queued).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + expect(recoveredRows(database.db)).toEqual([]) + expect(turnState(current)).toBe('running') + expect(holds(current)).toEqual({ working: true, queueHeld: true }) + expect(current.dispatch).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts new file mode 100644 index 000000000000..c9465ee4aed8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -0,0 +1,369 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + agentJournalItemKey, + agentJournalSubmissionKey +} from '../../../shared/agent-session-journal-item-key' +import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' +import { isSubagentGroupBlock } from '../../../shared/native-chat-types' +import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' +import { codexSubagentGroupIdentity } from '../../codex/codex-subagent-roster' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueueHold } from './structured-agent-session-queued-messages' +import { isMainAgentWorking } from './structured-agent-session-turns-cancel' +import { + APPROVAL, + currentJournal, + exitChild, + exitWhileSettlementFails, + leaveUnfinishedWork, + REASONING, + recoveredRows, + rowFence, + ROSTER_GROUP, + sendText, + TASK, + TOOL, + TURN_KEY, + turnState, + watchMarks +} from './structured-agent-session-leftover-settlement.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +function leftoverSettledAt(current: QueuedMessageTestRig): number | null | undefined { + return current.store.getRecord(SESSION)?.lease.leftoverSettledAt +} + +function settlementWarnings(): string[] { + return vi + .mocked(console.warn) + .mock.calls.flatMap(([, fields]) => + typeof fields === 'object' && fields !== null && 'scope' in fields + ? [String(fields.scope)] + : [] + ) +} + +describe('the observed exit, then the next start', () => { + it('settles at the start in one commit at the released fence, with its mark, before the successor binds', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const firstSend = await current.workingSend() + await leaveUnfinishedWork(current, { everything: true }) + const { session, database, deadFence } = await exitWhileSettlementFails(current) + database.db.exec('DROP TRIGGER reject_recovered') + const marks = watchMarks(database.db) + + const { clientOperationId, result } = sendText(current, 'continue after storage recovered') + + expect(await result).toMatchObject({ ok: true }) + // An ordinary send: no card waits behind the dead generation's turn. + expect(session.journal.queuedMessages.list()).toEqual([]) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + const settled = recoveredRows(database.db) + expect(settled.length).toBeGreaterThan(0) + // Every settlement row, at the released fence, was in the database when the mark was set. + expect(marks()).toEqual([{ fence: deadFence + 1, rows: settled.length }]) + expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 1)) + // The successor reserved after it, at the next fence. + expect(session.child?.fence).toBe(deadFence + 2) + expect(leftoverSettledAt(current)).toBe(NOW) + // The exit's proof named the dead generation: its turn was interrupted. + expect(turnState(current)).toBe('interrupted') + const body = (identity: AgentJournalItemIdentity) => + session.journal.itemBody(agentJournalItemKey(identity)) + expect(body(APPROVAL)).toMatchObject({ resolution: { state: 'cancelled' } }) + expect(body(TOOL)).toMatchObject({ state: 'failed', endedAs: 'interrupted' }) + expect(body(REASONING)).not.toMatchObject({ state: 'running' }) + const roster = body(codexSubagentGroupIdentity(ROSTER_GROUP)) + expect( + roster?.kind === 'message' ? roster.blocks.find(isSubagentGroupBlock)?.agents : undefined + ).toMatchObject([{ id: 'a', state: 'unverifiable' }]) + expect(body(TASK)).toMatchObject({ + blocks: [ + { text: 'Background command "sleep 20" stopped reporting' }, + { state: 'unverifiable' } + ] + }) + // Handed over and never answered: doubt, never "not delivered". + expect(session.journal.submission(firstSend)).toMatchObject({ + dispatchState: 'unknown', + reason: 'provider_exited_before_acknowledgement' + }) + }) + + it('settles at the exit itself when nothing refuses it, once', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const database = openTestJournalHostDatabase(current.root) + const marks = watchMarks(database.db) + + await exitChild(current) + + const settled = recoveredRows(database.db) + expect(turnState(current)).toBe('interrupted') + expect(marks()).toEqual([{ fence: 2, rows: settled.length }]) + // The next start finds nothing left: its settlement writes no row. + expect((await sendText(current, 'next').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(recoveredRows(database.db)).toEqual(settled) + }) +}) + +describe('a settlement that cannot commit never gates the user', () => { + it('delivers the queue-if-active send the STOP reproduction stranded', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { session } = await exitWhileSettlementFails(current) + + const { clientOperationId, result } = sendText(current, 'still stranded?') + + expect(await result).toMatchObject({ ok: true }) + expect((await result).ok && 'queued' in (await result)).toBe(false) + expect(session.journal.queuedMessages.list()).toEqual([]) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) + + it('reads a dead generation’s open turn as not Working, holds nothing, and delivers', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + await exitWhileSettlementFails(current) + const journal = currentJournal(current) + const record = current.store.getRecord(SESSION) + const fence = record?.lease.runtimeFence ?? 0 + + // Still saved as running: only its settlement would end it. + expect(turnState(current)).toBe('running') + expect( + isStructuredAgentSessionMainAgentWorking( + journal.activeTurnId(fence), + journal.submissions(), + fence + ) + ).toBe(false) + expect(structuredQueueHold({ journal, record, fence })).toBeNull() + expect(isMainAgentWorking({ journal, fence })).toBe(false) + const { items, submissions } = journal.snapshot() + expect( + projectStructuredAgentSessionStatusState(items, submissions, fence, journal.itemFence).summary + .status + ).not.toBe('working') + + expect((await sendText(current, 'go on').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) + + it('starts, delivers and logs when the start’s own settlement fails; the next trigger settles once as unverifiable', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { session, database } = await exitWhileSettlementFails(current) + + const { clientOperationId, result } = sendText(current, 'meets the storage fault') + + expect(await result).toMatchObject({ ok: true }) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(settlementWarnings()).toContain('acquisition-settlement') + // Nothing partial committed; the reservation left the mark as it was. + expect(recoveredRows(database.db)).toEqual([]) + expect(leftoverSettledAt(current)).toBeNull() + expect(turnState(current)).toBe('running') + // The successor's turn is the one that counts. + expect(session.journal.activeTurnId(session.child?.fence)).toBeNull() + + database.db.exec('DROP TRIGGER reject_recovered') + const marks = watchMarks(database.db) + await exitChild(current) + + // The reservation wiped the proof that named it: less specific, never wrong. + expect(turnState(current)).toBe('unverifiable') + expect(session.journal.itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ + resolution: { state: 'cancelled' } + }) + const revisions = recoveredRows(database.db).filter((row) => row.rowJson.includes(TURN_KEY)) + expect(revisions).toHaveLength(1) + expect(marks()).toHaveLength(1) + }) +}) + +describe('a stale prompt', () => { + it('holds no send, ends cancelled once settled, and refuses a stale answer', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { session, database } = await exitWhileSettlementFails(current) + const prompt = session.journal.item(agentJournalItemKey(APPROVAL)) + if (!prompt) { + throw new Error('expected the saved prompt') + } + const answer = () => { + const fields = { itemId: prompt.itemId, expectedRevision: prompt.revision, optionId: 'yes' } + return current.host.respondToPrompt(CALLER, { + envelope: { + ...current.envelope(fields, 'agentSession.respondTo:approval', hostTestOperationId()), + expectedRuntimeFence: current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + }, + kind: 'approval', + ...fields + }) + } + + // Unsettled, it holds nothing and takes no answer: the agent that asked is gone. + expect((await sendText(current, 'go on without it').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(session.journal.itemBody(prompt.itemId)).toMatchObject({ + resolution: { state: 'pending' } + }) + expect(await answer()).toMatchObject({ + ok: false, + refusal: { details: { reason: 'promptGone' } } + }) + expect(current.answerPrompt).not.toHaveBeenCalled() + + database.db.exec('DROP TRIGGER reject_recovered') + await exitChild(current) + + expect(session.journal.itemBody(prompt.itemId)).toMatchObject({ + resolution: { state: 'cancelled' } + }) + expect((await answer()).ok).toBe(false) + expect(current.answerPrompt).not.toHaveBeenCalled() + }) +}) + +describe('startup', () => { + it('settles a failed exit settlement once and sets the mark, starting nothing', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { deadFence } = await exitWhileSettlementFails(current) + let owed: number | null | undefined + // A new process loads the database and the record store back from disk, and starts up. + await current.crashReloadHostProcess(() => { + owed = leftoverSettledAt(current) + }) + const database = openTestJournalHostDatabase(current.root) + expect(owed).toBeNull() + + expect(turnState(current)).toBe('interrupted') + expect(currentJournal(current).itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ + resolution: { state: 'cancelled' } + }) + expect(leftoverSettledAt(current)).toBe(NOW) + const settled = recoveredRows(database.db) + expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 1)) + expect(current.starts).toHaveBeenCalledOnce() + // A second pass finds nothing owed. + await current.host.reconcileRestartLeases() + await current.host.startupSettled() + expect(recoveredRows(database.db)).toEqual(settled) + }) +}) + +describe('the mark is only an index', () => { + it('a wrong "settled" holds nothing: delivery goes on, and the next start settles and corrects it', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { database } = await exitWhileSettlementFails(current) + database.db.exec('DROP TRIGGER reject_recovered') + await current.store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { ...record.lease, leftoverSettledAt: 1 } + })) + await current.crashReloadHostProcess() + // Startup believed it and skipped the chat; a reader's open writes nothing either. + await current.host.journalSnapshot(SESSION) + expect(turnState(current)).toBe('running') + const marks = watchMarks(openTestJournalHostDatabase(current.root).db) + + expect((await sendText(current, 'after a wrong mark').result).ok).toBe(true) + + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(turnState(current)).toBe('interrupted') + // The receipt replaced the wrong mark; a set mark is not re-set, so no row logs it. + expect(marks()).toEqual([]) + expect(leftoverSettledAt(current)).toBe(NOW) + }) + + it('a wrong "owed" costs one journal open and a settlement that plans nothing', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + await exitChild(current) + expect(turnState(current)).toBe('interrupted') + await current.store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { ...record.lease, leftoverSettledAt: null } + })) + const settled = recoveredRows(openTestJournalHostDatabase(current.root).db) + + await current.crashReloadHostProcess() + + expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual(settled) + expect(leftoverSettledAt(current)).toBe(NOW) + expect((await sendText(current, 'after a wrong owed mark').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('a folder workspace', () => { + it('settles a folder chat’s dead generation the same way', async () => { + rig = await createQueuedMessageTestRig({ + restartable: true, + location: { + executionHostId: 'local', + wslDistro: null, + workspaceId: 'folder-1', + workspaceKind: 'folder' + } + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { session, database } = await exitWhileSettlementFails(current) + database.db.exec('DROP TRIGGER reject_recovered') + + const { clientOperationId, result } = sendText(current, 'continue in the folder') + + expect(await result).toMatchObject({ ok: true }) + expect(session.journal.submission(clientOperationId)).toBeDefined() + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + expect(turnState(current)).toBe('interrupted') + expect(leftoverSettledAt(current)).toBe(NOW) + expect( + session.journal.item(agentJournalSubmissionKey(clientOperationId))?.sequence + ).toBeGreaterThan(recoveredRows(database.db).at(-1)?.seq ?? 0) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts new file mode 100644 index 000000000000..0d72e8b43527 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts @@ -0,0 +1,278 @@ +// A dead generation's leftover work, for the leftover-settlement tests: a child that leaves a running +// turn (and what else a test asks for), an exit whose settlement the database refuses, and the +// database instruments the tests read. + +import { expect, vi } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + codexSubagentGroupBody, + codexSubagentGroupIdentity +} from '../../codex/codex-subagent-roster' +import { + liveTestJournalRows, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type Database from '../../sqlite/sync-database' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestMessage, + hostTestOperationId +} from './structured-agent-session-host-test-data' +import { + QUEUED_RIG_CALLER as CALLER, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' + +export const TURN: AgentJournalItemIdentity = { + provider: 'codex', + threadId: THREAD, + turnId: 'unfinished', + ordinal: 1 +} +export const TURN_KEY = agentJournalItemKey(TURN) +const IN_TURN = { kind: 'turn' as const, turnItemId: TURN_KEY } +export const APPROVAL: AgentJournalItemIdentity = { ...TURN, ordinal: 2 } +export const TOOL: AgentJournalItemIdentity = { ...TURN, ordinal: 3 } +export const REASONING: AgentJournalItemIdentity = { ...TURN, ordinal: 4 } +export const ROSTER_GROUP = `${THREAD}:unfinished` +export const TASK: AgentJournalItemIdentity = { + provider: 'orca', + clientMessageId: 'claude-background-task:task-1' +} +/** Refuses every row a settlement writes: its rows are all `recovered`. */ +export const REJECT_RECOVERED_ROWS = ` + CREATE TEMP TRIGGER reject_recovered BEFORE INSERT ON journal_rows + WHEN json_extract(NEW.row_json, '$.recovered') = 1 + BEGIN SELECT RAISE(ABORT, 'temporary storage failure'); END; +` + +/** The live child leaves a running turn, and what else the test asks for, in the journal. */ +export async function leaveUnfinishedWork( + current: QueuedMessageTestRig, + options: { prompt?: true; everything?: true } = {} +): Promise { + const events = current.providerEvents() + events.appendItem( + TURN, + { kind: 'turn', turnId: 'unfinished', state: 'running' }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + if (options.prompt || options.everything) { + events.appendItem( + APPROVAL, + { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }, + { turnScope: IN_TURN } + ) + } + if (options.everything) { + events.appendItem( + TOOL, + { kind: 'tool-call', name: 'shell', input: { command: 'pnpm test' }, state: 'running' }, + { turnScope: IN_TURN } + ) + events.appendItem( + REASONING, + { + kind: 'message', + role: 'assistant', + state: 'running', + blocks: [{ type: 'text', text: 'thinking' }] + }, + { turnScope: IN_TURN } + ) + events.appendItem( + codexSubagentGroupIdentity(ROSTER_GROUP), + codexSubagentGroupBody(ROSTER_GROUP, [ + { id: 'a', label: 'read', state: 'working', startedAt: 10 } + ]), + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + events.appendItem( + TASK, + { + kind: 'message', + role: 'system', + blocks: [ + { type: 'text', text: 'Started background command "sleep 20"' }, + { + type: 'background-task', + taskId: 'task-1', + kind: 'command', + label: 'sleep 20', + state: 'working', + startedAt: 10 + } + ] + }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + } + await current.host.flushStreamedEvents(SESSION) +} + +/** The child dies on its own while the database refuses every settlement row: the exit releases + * its lease (proof written, fence moved, leftovers owed) and its work stays saved as running. */ +export async function exitWhileSettlementFails(current: QueuedMessageTestRig) { + const context = current.host.collaboratorsForTests() + const session = context.sessions.get(SESSION) + const child = session?.child + if (!session || !child?.generation) { + throw new Error('expected the original child') + } + const database = openTestJournalHostDatabase(current.root) + const before = liveTestJournalRows(database.db, SESSION) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await context.serialize(SESSION, async () => {}) + expect(session.child).toBeNull() + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'released', + runtimeFence: child.fence + 1, + deathEvidence: { kind: 'exit-observed', ownerFence: child.fence }, + leftoverSettledAt: null + }) + expect(liveTestJournalRows(database.db, SESSION)).toEqual(before) + return { session, database, deadFence: child.fence } +} + +/** The child exits on its own with nothing refused. */ +export async function exitChild(current: QueuedMessageTestRig): Promise { + const context = current.host.collaboratorsForTests() + const child = context.sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected a child') + } + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await context.serialize(SESSION, async () => {}) +} + +/** A person's ordinary text send from the composer, with queueing on, at the current fence. */ +export function sendText(current: QueuedMessageTestRig, text: string) { + const fence = current.store.getRecord(SESSION)?.lease.runtimeFence + if (fence === undefined) { + throw new Error('expected the record') + } + const body = hostTestMessage(text) + const clientOperationId = hostTestOperationId() + const fields = { body, delivery: 'queue-if-active' as const } + const result = current.host.send(CALLER, { + ...fields, + userSend: true, + envelope: { + ...current.envelope(fields, 'agentSession.send', clientOperationId), + expectedRuntimeFence: fence + } + }) + return { clientOperationId, result } +} + +export function currentJournal(current: QueuedMessageTestRig) { + const journal = current.host.collaboratorsForTests().sessions.get(SESSION)?.journal + if (!journal) { + throw new Error('expected the open conversation') + } + return journal +} + +export function turnState(current: QueuedMessageTestRig): string | undefined { + return readAgentJournalTurn(currentJournal(current).itemBody(TURN_KEY) ?? undefined)?.state +} + +/** The rows settlements wrote: each is `recovered`. */ +export function recoveredRows(db: Database.Database) { + return liveTestJournalRows(db, SESSION).filter((row) => { + const parsed: unknown = JSON.parse(row.rowJson) + return typeof parsed === 'object' && parsed !== null && 'recovered' in parsed + }) +} + +export function rowFence(row: { rowJson: string }): number { + const parsed: unknown = JSON.parse(row.rowJson) + return typeof parsed === 'object' && parsed !== null && 'fence' in parsed + ? Number(parsed.fence) + : Number.NaN +} + +/** Each time the lease's mark is set: the fence it was set at, and how many settlement rows the + * database held at that moment, inside the transaction that set it. */ +export function watchMarks(db: Database.Database): () => { fence: number; rows: number }[] { + db.exec(` + CREATE TEMP TABLE mark_log (fence INTEGER, rows INTEGER); + CREATE TEMP TRIGGER note_mark AFTER UPDATE ON agent_session_records + WHEN json_extract(NEW.record_json, '$.lease.leftoverSettledAt') IS NOT NULL + AND json_extract(OLD.record_json, '$.lease.leftoverSettledAt') IS NULL + BEGIN + INSERT INTO mark_log + SELECT json_extract(NEW.record_json, '$.lease.runtimeFence'), COUNT(*) FROM journal_rows + WHERE session_id = NEW.session_id AND json_extract(row_json, '$.recovered') = 1; + END; + `) + return () => + db + .prepare('SELECT fence, rows FROM mark_log') + .all() + .map((row) => ({ fence: Number(row.fence), rows: Number(row.rows) })) +} + +/** Logs every INSERT, UPDATE and DELETE on every table of the file, from this connection on. */ +export function logEveryWrite(db: Database.Database): () => string[] { + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + .all() + .map((row) => String(row.name)) + db.exec('CREATE TEMP TABLE test_write_log (entry TEXT)') + for (const table of tables) { + for (const op of ['INSERT', 'UPDATE', 'DELETE']) { + db.exec(`CREATE TEMP TRIGGER test_log_${op}_${table} AFTER ${op} ON main.${table} + BEGIN INSERT INTO test_write_log VALUES ('${op} ${table}'); END;`) + } + } + return () => + db + .prepare('SELECT entry FROM test_write_log') + .all() + .map((row) => String(row.entry)) +} + +/** Startup as the runtime runs it, after an earlier process whose last generation ended with + * nothing settling it, as a crash leaves it. */ +export async function startUpOwingLeftovers( + host: Pick, + store: Pick +): Promise { + await store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { ...record.lease, leftoverSettledAt: null } + })) + await host.reconcileRestartLeases() + await host.startupSettled() +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts new file mode 100644 index 000000000000..05a6f7303751 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts @@ -0,0 +1,125 @@ +// The one settlement of what ended generations left unfinished, run only at the host's ownership +// events: an observed exit (after its release), an acquisition (before its reservation), and +// startup. Opening a chat never runs it. +// +// Re-derived each time from the journal and the lease: every turn, call, prompt, reasoning row, +// subagent roster and background task an ended generation wrote and never ended (below the lease's +// fence, or at it once released), and every send handed to one and never answered. Such a +// generation can no longer write, since every release moves the fence. Its turns end `interrupted` only when the lease's death evidence +// names it, else `unverifiable`. Rows land at the lease's current fence, in ONE transaction with the +// lease's `leftoverSettledAt` receipt, so a second run plans nothing. Best effort at every trigger: +// a failure is the caller's to log, and its rows hold nothing (holds count only current-fence work). + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + planStructuredAgentSessionDeadGeneration, + type StructuredAgentSessionDeadGenerationInput +} from './structured-agent-session-dead-generation-settlement' +import { + planStaleStructuredAgentSessionState, + type StaleStructuredAgentSessionStateJournal +} from './structured-agent-session-stale-state-settlement' +import type { DeadGenerationJournal } from './structured-agent-session-unfinished-work' +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' +import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' +import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' + +export type StructuredAgentSessionLeftoverStore = Pick & { + conversationReceipts: Pick +} + +export type StructuredAgentSessionLeftoverSettlementInput = { + store: StructuredAgentSessionLeftoverStore + sessionId: string + journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal + now: () => number + /** An observed exit: what that child's generation left is settled as the exit says. */ + exit?: Omit & { + ownerFence: number + } + /** An acquisition about to reserve over an owner a probe proved gone: that owner's generation + * ends with the reservation, so its work is settled with the rest. */ + ownerProvenGone?: true +} + +export type StructuredAgentSessionLeftoverSettlement = { ok: true } | { ok: false; error: unknown } + +export async function settleStructuredAgentSessionLeftovers( + input: StructuredAgentSessionLeftoverSettlementInput +): Promise { + try { + const record = input.store.getRecord(input.sessionId) + if (!record) { + return { ok: true } + } + const { lease } = record + const fence = lease.runtimeFence + const { exit, journal, sessionId } = input + // A released fence was never a child's (a reservation moves it first), so all of it is ended; + // what an older build left at it is settled with the rest. + const ended = input.ownerProvenGone || lease.claimStatus === 'released' + const below = exit ? exit.ownerFence : ended ? fence + 1 : fence + const failureTextContext = structuredAgentSessionFailureWordsContext(record) + await journal.appendPlannedLifecycleBatch({ + settlementId: exit + ? `dead-generation:${exit.settlementId}` + : `${STALE_SESSION_ROW_PREFIX}${sessionId}:${fence}:seq-${journal.cursor().sequence}`, + fence, + recovered: true, + // Only a released lease is owed anything: a reservation leaves the mark as it was. + ...(lease.claimStatus === 'released' + ? { + receipt: input.store.conversationReceipts.leftoverSettled(sessionId, fence, input.now()) + } + : {}), + plan: () => { + const exited = exit + ? planStructuredAgentSessionDeadGeneration( + { ...exit, journal, sessionId, fence }, + (itemFence) => itemFence === undefined || itemFence >= exit.ownerFence + ) + : { mutations: [], dispatches: [] } + const settledByExit = new Set(exited.dispatches.map((entry) => entry.clientMessageId)) + const stale = planStaleStructuredAgentSessionState({ + journal, + sessionId, + fence, + acquisitionGeneration: null, + // Read with the rows: the evidence the release before this settlement wrote. + deathEvidence: input.store.getRecord(sessionId)?.lease.deathEvidence ?? null, + failureTextContext, + below + }) + return { + mutations: [...exited.mutations, ...stale.mutations], + dispatches: [ + ...exited.dispatches, + ...stale.dispatches.filter((entry) => !settledByExit.has(entry.clientMessageId)) + ] + } + } + }) + return { ok: true } + } catch (error) { + return { ok: false, error } + } +} + +/** The settlement at a trigger that must go on whatever happens: a failure is logged under the + * trigger's scope, and the leftovers stay owed for the next trigger. */ +export async function settleStructuredAgentSessionLeftoversOrLog( + input: StructuredAgentSessionLeftoverSettlementInput & { + logger: StructuredAgentSessionLogger + scope: string + } +): Promise { + const settled = await settleStructuredAgentSessionLeftovers(input) + if (!settled.ok) { + input.logger.warn("settling a gone agent's leftover work did not finish", { + scope: input.scope, + sessionId: input.sessionId, + error: settled.error + }) + } + return settled.ok +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts index 1778f7e43977..c4d7333b9eb6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-orca-stop-row.ts @@ -80,7 +80,10 @@ function cutByNobody(item: AgentJournalRenderItem | undefined): boolean { export async function recordStructuredAgentSessionShutdownCut(input: { journal: OrcaStopRowJournal sessionId: string + /** The stopped child's fence, which keys the row. */ fence: number + /** The fence the row is written at: the lease's, which the child's end moved past `fence`. */ + rowFence: number generation: string turnItemId: string | null trigger: AgentSessionResumeTrigger @@ -108,7 +111,7 @@ export async function recordStructuredAgentSessionShutdownCut(input: { await input.journal.appendItem( { provider: 'orca', clientMessageId }, orcaStopRowBody(input.failureTextContext, input.trigger), - { fence: input.fence, turnScope: { kind: 'turn', turnItemId } } + { fence: input.rowFence, turnScope: { kind: 'turn', turnItemId } } ) } catch (error) { input.logger.warn('recording why a quit cut a reply failed', { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts index c7a9997117e3..e9bc67d48ebe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -617,10 +617,13 @@ describe('a quit with a message still queued', () => { surface: 'rejection' }) - /** Read by the next launch, through the same open any reader takes. */ + /** Read after the next launch's startup: its reconcile and settlement, then the restore of the + * chat's tab, whose share holds what the quit left unsent. */ async function afterRelaunch(id: string): Promise { startHost() - await host.revealSession(SESSION) + await host.reconcileRestartLeases() + await host.startupSettled() + await host.restoreReadableSessions([SESSION]) return await submission(id) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts index 0feaf809f2a5..6c432486a19b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts @@ -231,7 +231,7 @@ describe('after a restart, nothing sends by itself', () => { await rig.workingSend() const first = await queuedDraft('A') const second = await queuedDraft('B') - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() // The new host opens the chat for its first reader. await published() return { first, second } @@ -391,7 +391,7 @@ describe('after a restart, nothing sends by itself', () => { await eventually(async () => expect(await rig.handoff(first)).toBeDefined()) const cutShort = await rig.handoffId(first) expect((await rig.handoff(first))?.handedOverAt).toBeUndefined() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() release() await published() // The new host refuses the leftover hand-off, and A waits again in its own place. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts index ee7a26af161e..76aa19dddb19 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-compact.test.ts @@ -409,7 +409,7 @@ describe('a /compact that waits in line', () => { expect(await rig.sendNow(compactId)).toMatchObject({ ok: true }) } await eventually(() => expect(rig.starts.mock.calls.length).toBeGreaterThan(startsBefore)) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() release() return compactId } @@ -488,7 +488,7 @@ describe('a /compact that waits in line', () => { const startsBefore = rig.starts.mock.calls.length const { id } = compact() await eventually(() => expect(rig.starts.mock.calls.length).toBeGreaterThan(startsBefore)) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() release() // A command in flight is not resumed: the person runs it again. No card, kept or otherwise. expect(await rig.drafts()).toEqual([]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts index ef7e36953bd3..316a79400a46 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts @@ -11,11 +11,15 @@ import { agentSessionFailureWords } from '../../../shared/agent-session-failure- import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionQueuePause } from '../../../shared/agent-session-wire' +import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' import type { AgentMessageSource } from '../../../shared/agent-session-message-source' import { agentSessionMessagePayload } from '../../../shared/structured-agent-session-send-mutation' import { openTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' import { AgentSessionRecoveryCapsule } from '../../runtime/agent-session-recovery-capsule' -import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + StructuredAgentSessionHost, + type StructuredAgentSessionHostDeps +} from './structured-agent-session-host' import { rotateStructuredAgentSessionHostInstanceForTests } from './structured-agent-session-queued-pause' import { HOST_TEST_NOW as NOW, @@ -26,7 +30,10 @@ import { hostTestOperationId, resetHostTestOperationIds } from './structured-agent-session-host-test-data' -import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + closeTestJournalHostDatabase, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { createQueuedRigProvider, @@ -49,12 +56,16 @@ export async function createQueuedMessageTestRig( idleSweep?: { idleMs: number; intervalMs: number } /** Teardown records its restart offers, which `restartOffers` lists. */ recoveryCapsule?: true + /** Where the chat runs, when not the default worktree. */ + location?: AgentSessionExecutionLocation + /** How the host probes a recorded owner; absent answers indeterminate. */ + probeOwner?: StructuredAgentSessionHostDeps['probeOwner'] } = {} ) { const root = await mkdtemp(join(tmpdir(), 'orca-queued-messages-')) resetHostTestOperationIds() - const store = await openTestAgentSessionRecordStore(root) - const provider = createQueuedRigProvider(store, options) + let store = await openTestAgentSessionRecordStore(root) + const provider = createQueuedRigProvider({ getRecord: (id) => store.getRecord(id) }, options) const { dispatch, compact, cancelTurn, closeSession } = provider const makeHost = () => new StructuredAgentSessionHost({ @@ -66,13 +77,17 @@ export async function createQueuedMessageTestRig( claimKeyId: 'key-1', mintSpawnToken: () => 'spawn-1', now: () => NOW, + ...(options.probeOwner ? { probeOwner: options.probeOwner } : {}), ...(options.idleSweep ? { idleSweep: options.idleSweep } : {}), ...(options.recoveryCapsule ? { recoveryCapsule: new AgentSessionRecoveryCapsule(root) } : {}) }) let host = makeHost() - expect(await host.attach(QUEUED_RIG_CALLER, hostTestAttachParams(null))).toMatchObject({ - ok: true - }) + // Kept, so a test can re-attach under the same operation: a reconnect to the live child. + const firstAttach = hostTestAttachParams( + null, + options.location ? { location: options.location } : {} + ) + expect(await host.attach(QUEUED_RIG_CALLER, firstAttach)).toMatchObject({ ok: true }) function envelope( fields: Record, @@ -212,20 +227,36 @@ export async function createQueuedMessageTestRig( /** A host-process restart: the app quits (its own teardown runs) and a new host opens the same * state. A quit writes no close's Stop event, so a person's Stop pause survives it. */ - async function restartHostProcess(): Promise { - await quitRestartHostProcess() + async function restartHostProcess(beforeStartup?: () => unknown): Promise { + await quitRestartHostProcess(beforeStartup) } - /** A host process that dies with no close: a new host opens the same state directory. */ - function crashRestartHostProcess(): void { + /** A host process that dies with no close: a new process loads the same state directory, its + * records with it, and runs its startup as the runtime does before any client reaches it: the + * restart reconcile and its settlement, then the restore of the chat's tab. */ + async function crashRestartHostProcess(beforeStartup?: () => unknown): Promise { + await crashReloadHostProcess(beforeStartup) + await host.restoreReadableSessions([SESSION]) + await host.startupSettled() + } + + /** The same crash, with only the restart reconcile and its settlement: no tab is restored. + * `beforeStartup` sees the state directory between the two processes. */ + async function crashReloadHostProcess(beforeStartup?: () => unknown): Promise { + // Its connection closes with it, and every temporary trigger a test added with it. + closeTestJournalHostDatabase(root) + await beforeStartup?.() + store = await openTestAgentSessionRecordStore(root) rotateStructuredAgentSessionHostInstanceForTests() host = makeHost() + await host.reconcileRestartLeases() + await host.startupSettled() } /** The app quits — the host's own teardown runs — and a new host opens the same state. */ - async function quitRestartHostProcess(): Promise { + async function quitRestartHostProcess(beforeStartup?: () => unknown): Promise { await host.flushAllStreamedEvents() - crashRestartHostProcess() + await crashRestartHostProcess(beforeStartup) } /** The queue's published pause: null when it sends on its own. */ @@ -276,7 +307,10 @@ export async function createQueuedMessageTestRig( return { root, - store, + firstAttach, + get store() { + return store + }, get host() { return host }, @@ -285,6 +319,7 @@ export async function createQueuedMessageTestRig( closeSession, compact, starts: provider.starts, + answerPrompt: provider.adapter.answerPrompt, holdNextStart: provider.holdNextStart, failNextStart: provider.failNextStart, finishCompact: provider.finishCompact, @@ -303,6 +338,7 @@ export async function createQueuedMessageTestRig( settleRejected, restartHostProcess, crashRestartHostProcess, + crashReloadHostProcess, quitRestartHostProcess, queuePause, restartOffers, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 249396d38b5d..631bd5d8e466 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -19,7 +19,10 @@ import { queuedSendAnswer } from './structured-agent-session-queued-send-answer' import { structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { queuedMessagesPublishedBytesRefusal } from './structured-agent-session-queued-published-bytes' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import { + isStructuredAgentSessionEndedGenerationWork, + isStructuredAgentSessionMainAgentWorking +} from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { AgentSessionTurnContext } from './structured-agent-session-turns' import { QueuedMessageNotConsumableError } from '../agent-session-journal/journal-queued-messages' @@ -40,14 +43,19 @@ export function queuedMessageBodyIsTextOnly(body: AgentJournalMessageItem): bool } /** Walks the reduced items in place: the gate runs on every admission and - * drain step, so it must not render a snapshot of the whole journal. */ -export function pendingPromptExists(journal: Pick): boolean { + * drain step, so it must not render a snapshot of the whole journal. Only a prompt a generation + * at `fence` raised: an ended one's can take no answer (`isStructuredAgentSessionEndedGenerationWork`). */ +export function pendingPromptExists( + journal: Pick, + fence: number +): boolean { let pending = false - journal.visitItems((_itemId, _sequence, body) => { + journal.visitItems((itemId, _sequence, body) => { if ( !pending && (body.kind === 'approval' || body.kind === 'question') && - body.resolution.state === 'pending' + body.resolution.state === 'pending' && + !isStructuredAgentSessionEndedGenerationWork(journal.itemFence(itemId), fence) ) { pending = true } @@ -102,12 +110,12 @@ export function structuredQueueHold(input: { const { journal } = input // `prompt` outranks `working`: it is the one wait Send-now may not override, // so a prompt raised mid-turn must not read as merely `working`. - if (pendingPromptExists(journal)) { + if (pendingPromptExists(journal, input.fence)) { return 'prompt' } if ( isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), + journal.activeTurnId(input.fence), journal.submissions(), input.fence ) @@ -131,7 +139,11 @@ export function nextStructuredQueuedMessage(input: { // prompt check walks the whole fold. if ( next === null || - isStructuredAgentSessionMainAgentWorking(journal.activeTurnId(), journal.submissions(), fence) + isStructuredAgentSessionMainAgentWorking( + journal.activeTurnId(fence), + journal.submissions(), + fence + ) ) { return null } @@ -291,7 +303,7 @@ export class StructuredAgentSessionQueuedMessageDrain { !journal.queuedMessages.settlementOwed() && (oldestActionableQueuedMessage(journal) === null || isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(), + journal.activeTurnId(this.deps.conversationFence(sessionId)), journal.submissions(), this.deps.conversationFence(sessionId) )) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts index eeaafbbaff8d..e844aff6d08b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-newer-orca.test.ts @@ -38,11 +38,16 @@ async function queuedDraft(text: string): Promise { return queued.value.queued.messageId } -/** The chat reopens on a database a newer Orca stamped. The host's database object is the one - * the rig built, so its open-time verdict is set in place; every store write checks that. */ +/** The database a newer Orca stamped. The host's database object is the one the rig built, so + * its open-time verdict is set in place; every store write checks that. */ +function stampNewerOrca(): void { + Object.defineProperty(openTestJournalHostDatabase(rig.root), 'readOnly', { value: true }) +} + +/** The chat reopens on a database a newer Orca stamped. */ async function reopenOnNewerOrcaDatabase(close: () => Promise): Promise { await close() - Object.defineProperty(openTestJournalHostDatabase(rig.root), 'readOnly', { value: true }) + stampNewerOrca() } /** The history read a newer Orca's chat gets: refused, as every reader's is. */ @@ -80,7 +85,8 @@ describe("a newer Orca's journal", () => { it('refuses Resume of the queue a restart paused with the update words', async () => { await twoCardsBehindWork() - await reopenOnNewerOrcaDatabase(() => rig.restartHostProcess()) + // Stamped before the new process starts, as a newer Orca's run leaves it. + await rig.restartHostProcess(stampNewerOrca) await expectHistoryRefused() expect(await rig.resume()).toMatchObject(REFUSED_BY_NEWER_ORCA) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts index 418c51f5cc74..a802043bab9e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause-lift.test.ts @@ -104,7 +104,7 @@ describe("a Stop's queue pause", () => { const inFlight = await handedOverUserSend('sent before the restart') // Derived from the journal, not remembered: a restart forgets nothing it needs. The process // dies with no close, as a quit writes no Stop event to end the pause either. - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() // Still derived, though after a restart no pause is shown. expect(await rig.queuePause()).toBeNull() expect(derivedPauses()).toEqual(['stopped', 'restarted']) @@ -448,7 +448,7 @@ describe('Resume', () => { it("of a Stop from before a restart, which no client offers there, also lifts the restart's pause", async () => { const draftId = await stoppedDraft() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.queuePause()).toBeNull() expect(derivedPauses()).toEqual(['stopped', 'restarted']) expect(await rig.resume()).toMatchObject({ ok: true, value: { resumed: true } }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts index e59a56a7c795..39f79d13a2b3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts @@ -30,8 +30,8 @@ export function structuredQueuePauses(journal: PauseJournal): DerivedQueuePause[ } /** The mark of a chat that stopped running with cards waiting — Orca quit or crashed, or the chat - * was closed — so they wait for its next turn (`queued-message-pause.ts`). Every open marks, and - * so does a person's close: the idle sweep never closes a chat with cards waiting, so its + * was closed — so they wait for its next turn (`queued-message-pause.ts`). Startup marks each + * chat with cards the earlier process left, and so does a person's close: the idle sweep never closes a chat with cards waiting, so its * eviction never reopens one. `since`: where the chat stopped, for a mark written after a send * that came later, which must still lift it. Bookkeeping, so a failure is reported and never * thrown; the pause then starts where the mark would have gone, holding no less. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts index e6ecad009d4c..fc03a0d0716f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop-row.test.ts @@ -177,7 +177,7 @@ describe("Stop's event", () => { await eventually(async () => expect((await rig.handoff(sentId))?.handedOverAt).toBeDefined()) await rig.stop() // The process dies with no close; a new host opens the same state directory. - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect((await rig.handoff(sentId))?.dispatchState).toBe('unknown') // After a restart no row shows, the Stop's included, and nothing sends. await expectHeld(null, waiting) @@ -198,15 +198,12 @@ describe("a restart's hold over a card queued after a Stop", () => { const mail = await mailTurn() const typed = await queuedDraft('typed during the mail turn') // The process dies with no close: a quit writes no Stop event, so only a turn ends the pauses. - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.settleAccepted(mail, 'mail') - // The new host opens the conversation for its first reader. The card came after the Stop, so - // only the restart holds it, and a restart's hold is never published. + // The mail turn, sent after the Stop, was accepted, which lifts the Stop's pause. The card came + // after the Stop anyway, so only the restart holds it, and a restart's hold is never published. await rig.queuePause() - expect(structuredQueuePauses(journal()).map((pause) => pause.reason)).toEqual([ - 'stopped', - 'restarted' - ]) + expect(structuredQueuePauses(journal()).map((pause) => pause.reason)).toEqual(['restarted']) await expectHeld(null, typed) }) @@ -219,7 +216,7 @@ describe("a restart's hold over a card queued after a Stop", () => { expect((await rig.handoff(correction))?.handedOverAt).toBeDefined() ) const typed = await queuedDraft('typed during that send') - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.settleAccepted(await rig.handoffId(correction), 'correction') await rig.queuePause() expect(structuredQueuePauses(journal()).map((pause) => pause.reason)).toEqual(['restarted']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts index fea528147494..a351a73793e8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts @@ -68,7 +68,7 @@ export function stopReachesUnrecordedWork( ctx: Pick, namedTurnId: string | undefined ): 'unrecorded' | 'repeat' | 'late' { - const live = ctx.journal.activeTurnId() + const live = ctx.journal.activeTurnId(ctx.fence) // No turn published yet while the agent works: the named one may still be opening. if ( structuredAgentSessionStopNamesTurnNotLive(namedTurnId, live) && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts index e17315b43e83..e9518febe80b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-interruption-test-harness.ts @@ -180,6 +180,9 @@ export async function interruptedRestart( now: () => clock.now }) replaceHostTestState({ store, host }) + // As the runtime starts it, before any client reaches it. + await host.reconcileRestartLeases() + await host.startupSettled() previous.acquire.mockClear() previous.releaseAcquisition.mockClear() previous.dispatch.mockClear() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts index 932bd700fbf7..aa63dc5f889e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-offer-newer-orca.test.ts @@ -31,12 +31,13 @@ afterEach(() => vi.restoreAllMocks()) function newerOrcaRow(root: string, sessionId: string = SESSION): void { const { db } = openTestJournalHostDatabase(root) const last = liveTestJournalRows(db, sessionId).at(-1) - const parsed: unknown = last ? JSON.parse(last.rowJson) : null - if (!last || typeof parsed !== 'object' || parsed === null || !('epoch' in parsed)) { + const parsed: unknown = last ? JSON.parse(last.rowJson) : { epoch: 'newer-orca-epoch' } + if (typeof parsed !== 'object' || parsed === null || !('epoch' in parsed)) { throw new Error('the chat has no journal row to follow') } + // A chat nothing was written to has no journal yet: the row founds one, after its epoch row. const { epoch } = parsed - const seq = last.seq + 1 + const seq = (last?.seq ?? 1) + 1 insertTestJournalRowJson( db, sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 6c6efc49d728..d7d59add30fb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -40,6 +40,12 @@ export type StructuredAgentSessionReadRestoreDeps = { sessionId: string, opened: OpenedStructuredAgentSessionConversation ) => Promise | void + /** Inside the chat's serialize, once its recovery is resolved: what that ended is settled, for + * a chat this restore opened (`opened`) before `onReadable` publishes it. */ + afterReadable?: ( + sessionId: string, + opened?: OpenedStructuredAgentSessionConversation + ) => Promise } /** One session's share of the restart restore. Startup maps this over every supported record. */ @@ -56,19 +62,26 @@ async function restoreOneStructuredAgentSessionRead( /** The serialized half of the restore. */ async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick, + input: Pick< + StructuredAgentSessionReadRestoreDeps, + 'openDeps' | 'hasSession' | 'onReadable' | 'afterReadable' + >, sessionId: string ): Promise { + // A read or a send mid-restore may already have opened this one. if (input.hasSession(sessionId)) { - // A read or a send mid-restore already opened this one. + await input.afterReadable?.(sessionId) return } const opened = await restoreStructuredAgentSessionRead(input.openDeps, sessionId) if (!opened) { return } - // The open settled what a gone generation left running, so no reader sees it run. - await input.onReadable(sessionId, opened) + try { + await input.afterReadable?.(sessionId, opened) + } finally { + await input.onReadable(sessionId, opened) + } } export async function restoreStructuredAgentSessionsOnRestart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 9a6e3aca5917..1c11acb1ef47 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -23,6 +23,12 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal } from './structured-agent-session-host-types' +import { + settleOneAtStartup, + settleStructuredAgentSessionsAtStartup, + type StructuredAgentSessionStartupSettlementContext +} from './structured-agent-session-startup-settlement' +import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' /** Throws its refusal as the code itself. */ export async function revealStructuredAgentSession( @@ -52,27 +58,46 @@ export async function revealStructuredAgentSession( } } -/** The host's startup readable-restore sweep: reconcile, resolve, then open each chat's journal. +/** The host's startup: reconcile, then settle what the earlier process left (in the background: + * no chat's tab waits on it), and the readable-restore sweep that resolves and opens each chat. * Its lease bookkeeping is a reader's, which never fails a read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'openDeps' | 'reconcile' | 'resolveRecovery' + 'openDeps' | 'reconcile' | 'resolveRecovery' | 'afterReadable' > & { reconcileLeases: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise + startup: Pick & { + tasks: Pick + } } ): { reconcileRestartLeases: () => Promise + /** The startup settlement the last reconcile began; resolved once every chat's share ran. */ + startupSettled: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise } { - const { reconcileLeases, resolveRecovery, ...rest } = wiring + const { reconcileLeases, resolveRecovery, startup, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) const reconcile = createReaderReconcile(reconcileLeases, failures) + const settlement: StructuredAgentSessionStartupSettlementContext = { + deps, + sessions: startup.sessions, + serialize: rest.serialize, + adopt: async (sessionId, opened) => { + await rest.onReadable(sessionId, opened) + }, + now: () => deps.now?.() ?? Date.now(), + sharesDone: new WeakSet() + } + let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: deps, reconcile, + // A recovery the restore resolved can end a generation the startup pass found still owned. + afterReadable: (sessionId, opened) => settleOneAtStartup(settlement, sessionId, opened), // The next attach or send resolves recovery again, strictly, before it acts. resolveRecovery: (sessionId) => resolveRecovery(sessionId).then( @@ -88,7 +113,10 @@ export function createStructuredAgentSessionHostRestore( return { reconcileRestartLeases: async () => { await reconcile('startup') + // Tracked like a start, so a quit waits for it before it closes what it writes to. + startupSettled = startup.tasks.trackAttach(settleStructuredAgentSessionsAtStartup(settlement)) }, + startupSettled: () => startupSettled, restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts index 631b9cac3c37..63337ea24e3a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -1,8 +1,9 @@ import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' // A send can be what opens a conversation this process has not read yet: a chat nobody has on // screen after the app died, sent to from a phone or the CLI. Whatever that journal shows running -// belongs to a generation that is gone, so it is settled when the journal opens, not only when a -// new child starts: a start that then fails would leave the turn running for every reader. +// belongs to a generation that is gone, so the send's acquisition settles it before it reserves: +// a start that then fails leaves no turn running for any reader. A reader's open writes nothing; +// startup settles what it reads. import { cp, rm } from 'node:fs/promises' import { afterEach, expect, it, vi } from 'vitest' @@ -133,26 +134,31 @@ it.each(PROBES)( ) it.each(PROBES)( - 'settles the same turn when a reader opens the chat, when %s', + 'settles the same turn at startup, not when a reader opens the chat, when %s', async (_when, probe, settled) => { - const { host } = await relaunchAfterCrashMidTurn(probe) + const { host } = await relaunchAfterCrashMidTurn(probe, { reconcile: false }) await host.revealSession(SESSION) + expect(await turnStates(host)).toEqual(['running']) + await host.reconcileRestartLeases() + await host.startupSettled() + await host.restoreReadableSessions([SESSION]) expect(await turnStates(host)).toEqual([settled]) await host.flushAllStreamedEvents() } ) -// On desktop the chat on screen at relaunch reads before startup reconciles the leases: nothing -// has proved its owner gone yet, and the reconcile's proof then revises what the open settled. -it('settles it when a read reaches the chat before the startup reconcile, then revises it', async () => { +// On desktop the chat on screen at relaunch reads before startup reconciles the leases: that read +// writes nothing, and the startup that follows settles the turn by the reconcile's proof. +it('settles it at startup when a read reaches the chat before the startup reconcile', async () => { const { host } = await relaunchAfterCrashMidTurn({ outcome: 'pid-absent' }, { reconcile: false }) expect(hostTestState().store.getRecord(SESSION)?.lease.claimStatus).toBe('live') await host.history({ sessionId: SESSION, direction: 'tail' }) - expect(await turnStates(host)).toEqual(['unverifiable']) + expect(await turnStates(host)).toEqual(['running']) await host.reconcileRestartLeases() + await host.startupSettled() await host.restoreReadableSessions([SESSION]) expect(await turnStates(host)).toEqual(['interrupted']) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts new file mode 100644 index 000000000000..67dd06ed8c1e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts @@ -0,0 +1,65 @@ +// Doubles for unit tests that drive a settlement without a journal database: a planned batch is +// planned at once and handed to an `appendLifecycleBatch` mock as the batch it would write, and a +// store's leftover receipt commits nothing. + +import type { + AgentJournalCursor, + AgentJournalRenderItem, + AgentJournalSubmission +} from '../../../shared/agent-session-journal-types' +import type { StructuredAgentSessionChildExitSession } from './structured-agent-session-child-exit' +import type { + JournalLifecycleBatchInput, + JournalPlannedLifecycleBatchInput +} from '../agent-session-journal/journal-store-contracts' +import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' + +const NO_CURSOR: AgentJournalCursor = { epoch: 'epoch-1', sequence: 0 } + +/** `appendPlannedLifecycleBatch` for a double: a plan with no row reaches no `append`, as the + * writer writes none. */ +export function plannedBatchThrough( + append: (input: JournalLifecycleBatchInput) => Promise +): (input: JournalPlannedLifecycleBatchInput) => Promise { + return async (input) => { + const { mutations, dispatches } = input.plan() + if (mutations.length === 0 && dispatches.length === 0) { + input.receipt?.committed() + return NO_CURSOR + } + const cursor = await append({ + settlementId: input.settlementId, + fence: input.fence, + ...(input.recovered ? { recovered: input.recovered } : {}), + mutations, + dispatches + }) + input.receipt?.committed() + return cursor + } +} + +/** A store's receipts, for a store double: each commits nothing. */ +export const NO_STORE_RECEIPTS = { + leftoverSettled: (): JournalOperationReceipt => ({ write: () => {}, committed: () => {} }) +} + +/** An exited child's journal for a test without a database: the reads a settlement makes, with a + * planned batch handed to `appendLifecycleBatch` (`plannedBatchThrough`). */ +export function exitJournalDouble(fields: { + items: () => readonly AgentJournalRenderItem[] + appendLifecycleBatch: (input: JournalLifecycleBatchInput) => Promise + itemFence?: (itemId: string) => number | undefined + submissions?: () => readonly Partial[] +}): StructuredAgentSessionChildExitSession['journal'] { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a settlement reads only these members; the rest of the journal is unreachable from it. + return { + cursor: () => NO_CURSOR, + itemBody: () => null, + itemFence: fields.itemFence ?? (() => undefined), + snapshot: () => ({ items: fields.items() }), + submissions: fields.submissions ?? (() => []), + stopMarks: { latest: () => null, personStopDecides: () => false }, + appendPlannedLifecycleBatch: plannedBatchThrough(fields.appendLifecycleBatch) + } as unknown as StructuredAgentSessionChildExitSession['journal'] +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts index 4ed41e8c6884..5fe3bc74da53 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts @@ -15,6 +15,7 @@ import { type StructuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' import { projectStructuredAgentSessionStatus } from '../../../shared/structured-agent-session-projection' +import type { StructuredAgentSessionItemFence } from '../../../shared/structured-agent-session-main-agent-working' /** The full row the sidebar's fold would show, for callers that need the lead's own state beside * the working answer — the teardown snapshot records both from this one computation. */ @@ -25,9 +26,16 @@ export function structuredAgentSessionShownStatus( }, childWork: readonly AgentChildWorkView[] | undefined, /** The session's lease fence, as the status feed passes it: a send from an older one is not work. */ - fence: number | undefined + fence: number | undefined, + /** The host's item fences, as the status feed passes them: an ended generation's turn is not work. */ + itemFence?: StructuredAgentSessionItemFence ): StructuredAgentSessionAgentStatus { - const status = projectStructuredAgentSessionStatus(journal.items, journal.submissions, fence) + const status = projectStructuredAgentSessionStatus( + journal.items, + journal.submissions, + fence, + itemFence + ) return structuredAgentSessionAgentStatus({ status, ...(childWork ? { childWork } : {}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts new file mode 100644 index 000000000000..974fbff54dbf --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts @@ -0,0 +1,171 @@ +// What a generation that can no longer write left unfinished, settled from the journal and the +// lease's death evidence each time it is asked. Proven death ends a turn interrupted; anything else +// is `unverifiable`, until a proof naming its owner revises it. Working subagents and background +// tasks become `unverifiable`. Planned at the batch's own turn in the write queue, so an answer or +// a Stop queued ahead of it is what it settles from. + +import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem +} from '../../../shared/agent-session-journal-types' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import type { AgentSessionFailureWordsContext } from '../../../shared/agent-session-failure-words' +import { journalPendingSubmissionResolutions } from '../agent-session-journal/journal-pending-submission-recovery' +import { DISPATCH_DOUBT_PROVIDER_EXITED } from '../agent-session-journal/journal-dispatch-doubt-reasons' +import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' +import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' +import type { ResolveDispatchInput } from '../agent-session-journal/journal-store-contracts' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { lostLiveWorkJournalBody } from '../agent-session-journal/journal-subagent-liveness' +import { + endedUnseenMessageBody, + runningCallEnd, + terminalAgentJournalBody +} from '../agent-session-journal/journal-terminal-settlement' +import { + endedByPersonsStop, + provenUnverifiableTurnRevisions, + provenUnverifiedToolCallRevisions, + runningTurnLifecycleRevisions, + stopFoundTurnLiveAt, + turnVerdictFromDeathEvidence +} from './structured-agent-session-stale-turn-verdict' +import { settledRootTurnScope } from './structured-agent-session-exit-turn-scope' +import { orcaStopRowBody } from './structured-agent-session-orca-stop-row' +import { isInProgressStructuredAgentSessionItem } from './structured-agent-session-unfinished-work' + +/** What the settlement reads and writes of a chat's journal. */ +export type StaleStructuredAgentSessionStateJournal = Pick< + AgentSessionJournal, + | 'snapshot' + | 'itemFence' + | 'itemBody' + | 'submissions' + | 'stopMarks' + | 'cursor' + | 'appendPlannedLifecycleBatch' +> + +export type StaleStructuredAgentSessionStateInput = { + journal: StaleStructuredAgentSessionStateJournal + sessionId: string + /** The fence the settlement's rows are written at: the lease's current one. */ + fence: number + acquisitionGeneration: string | null + deathEvidence: AgentSessionDeathEvidence | null + /** Who the exit row names. */ + failureTextContext?: AgentSessionFailureWordsContext + /** Only what generations below this fence wrote, which can no longer write, is settled; their + * sends handed over and never answered become doubt. Absent: every item, and no send. */ + below?: number + /** Committed in the same transaction as the rows, even when nothing is left to settle. */ + receipt?: JournalOperationReceipt +} + +/** Settles it, and answers how many items it revised. */ +export async function settleStaleStructuredAgentSessionState( + input: StaleStructuredAgentSessionStateInput +): Promise { + const { journal } = input + // A successful settlement leaves terminal items; a failed transaction leaves the same prefix. + const generation = input.acquisitionGeneration ?? `seq-${journal.cursor().sequence}` + let planned = 0 + await journal.appendPlannedLifecycleBatch({ + settlementId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}`, + fence: input.fence, + recovered: true, + ...(input.receipt ? { receipt: input.receipt } : {}), + plan: () => { + const plan = planStaleStructuredAgentSessionState(input) + planned = plan.mutations.length + return plan + } + }) + return planned +} + +/** The settlement's rows, read from the journal as it stands. */ +export function planStaleStructuredAgentSessionState( + input: StaleStructuredAgentSessionStateInput +): { + mutations: JournalLifecycleMutationInput[] + dispatches: ResolveDispatchInput[] +} { + const { journal, below } = input + const ended = (fence: number | undefined) => + below === undefined || (fence !== undefined && fence < below) + const items = journal.snapshot().items.filter((item) => ended(journal.itemFence(item.itemId))) + const dispatches = + below === undefined + ? [] + : journalPendingSubmissionResolutions( + journal.submissions().filter((entry) => ended(entry.fence)), + input.fence, + // Doubt, never non-delivery: what a gone owner was handed and never answered. + { reason: DISPATCH_DOUBT_PROVIDER_EXITED } + ) + return { mutations: staleMutations(input, items), dispatches } +} + +function staleMutations( + input: StaleStructuredAgentSessionStateInput, + items: readonly AgentJournalRenderItem[] +): JournalLifecycleMutationInput[] { + const { journal } = input + // Each turn is judged by the evidence only if it names that turn's owner. + const verdictFor = (item: AgentJournalRenderItem) => + turnVerdictFromDeathEvidence( + input.deathEvidence, + journal.itemFence(item.itemId), + stopFoundTurnLiveAt(journal, item) + ) + // Calls an earlier settle closed with no proof, revised once a proof names their owner. + const mutations = provenUnverifiedToolCallRevisions(items, input.deathEvidence, journal) + for (const item of items) { + // A turn already settled (a person's Stop) ends its calls as it ended; only a turn still running + // leaves them to the evidence. + const end = runningCallEnd(item.turnScope, (id) => journal.itemBody(id), verdictFor(item).state) + const terminal = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) + // One revision per item: its live subagents and background tasks end with it. + const body = lostLiveWorkJournalBody(terminal ?? item.body) ?? terminal + if (body) { + mutations.push({ + kind: 'item', + itemId: item.itemId, + body, + turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE + }) + } + } + const proven = provenUnverifiableTurnRevisions(items, input.deathEvidence, journal) + const turnEnds = [ + ...items.flatMap((item) => runningTurnLifecycleRevisions([item], verdictFor(item))), + ...proven + ] + mutations.push(...turnEnds) + const evidence = input.deathEvidence + if ( + evidence && + (proven.length > 0 || + items.some( + (item) => + isInProgressStructuredAgentSessionItem(item) && verdictFor(item).state === 'interrupted' + )) && + !endedByPersonsStop(journal, turnEnds) + ) { + mutations.unshift({ + kind: 'item', + // Named by the death it explains, so a retry after a partly written settle adds no second row. + identity: { + provider: 'orca', + clientMessageId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:death-${evidence.ownerFence ?? 'unowned'}-${evidence.observedAt}` + }, + // The death evidence is Orca's log text, never a sentence for a person: the row says only + // that the provider stopped, and how Orca ended when the provider died with it. + body: orcaStopRowBody(input.failureTextContext, evidence.runtimeEnd), + turnScope: settledRootTurnScope(items, turnEnds) + }) + } + return mutations +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts index 204da81a2e6c..7bd37bcaa590 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts @@ -1,6 +1,6 @@ -// A turn an exited agent left running, whose settlement write failed, is settled by the next open of -// the conversation, not only by the next send: a reader reopening a chat the idle sweep closed, or -// reading it before the restart restore reaches it. +// A turn an exited agent left running, whose settlement write failed, is settled at the next +// ownership event: the next send's acquisition, or the next startup. A reader opening the chat in +// between writes nothing, and the turn holds nothing: it is an ended generation's. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' @@ -11,7 +11,9 @@ import { createRestTestRig, foundRestTestChat, IDLE_MS, + REST_TEST_CALLER, REST_TEST_SESSION as SESSION, + restTestSend, REST_TEST_THREAD, sweepOnce, type RestTestRig @@ -29,9 +31,8 @@ afterEach(async () => { await rig.dispose() }) -/** A turn in flight whose agent exits, and whose exit settlement the journal refuses: by default - * also the retry that recording the exit queues, so only an open is left to settle it. */ -async function exitWithUnwrittenSettlement(refusedWrites = 2): Promise { +/** A turn in flight whose agent exits, and whose exit settlement the journal refuses. */ +async function exitWithUnwrittenSettlement(refusedWrites = 1): Promise { await foundRestTestChat(rig) const open = rig.host.collaboratorsForTests().sessions.get(SESSION)! const running = open.child! @@ -43,7 +44,7 @@ async function exitWithUnwrittenSettlement(refusedWrites = 2): Promise { { turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await rig.host.flushStreamedEvents(SESSION) - const append = vi.spyOn(open.journal, 'appendLifecycleBatch') + const append = vi.spyOn(open.journal, 'appendPlannedLifecycleBatch') for (let refused = 0; refused < refusedWrites; refused += 1) { append.mockRejectedValueOnce(new Error('disk full')) } @@ -72,8 +73,8 @@ async function workingTurnState(): Promise { } describe('a turn its gone agent left running', () => { - it('is settled in place by the retry recording the exit queues, with no reopen', async () => { - await exitWithUnwrittenSettlement(1) + it('is settled in place by the exit, with no reopen', async () => { + await exitWithUnwrittenSettlement(0) expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(true) expect(await workingTurnState()).toBe('interrupted') @@ -84,7 +85,7 @@ describe('a turn its gone agent left running', () => { expect(rig.adapter.acquire).toHaveBeenCalledOnce() }) - it('is settled when the idle-closed conversation is opened again, and its reader sees it', async () => { + it('stays as it was through an idle-closed reopen, and the next send settles it first', async () => { await exitWithUnwrittenSettlement() expect(await workingTurnState()).toBe('running') rig.clock.now += IDLE_MS + 1 @@ -93,21 +94,28 @@ describe('a turn its gone agent left running', () => { const reader = collectSubscriber() await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) + expect(await workingTurnState()).toBe('running') - expect(await workingTurnState()).toBe('interrupted') + const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence + expect(await rig.host.send(REST_TEST_CALLER, restTestSend('next', fence))).toMatchObject({ + ok: true + }) + await vi.waitFor(async () => expect(await workingTurnState()).toBe('interrupted')) // The death evidence is Orca's log text: the row says only that the provider stopped. expect(JSON.stringify(reader.events)).toContain( 'Codex stopped while this response was in progress. You can continue in this conversation.' ) - expect(rig.adapter.acquire).toHaveBeenCalledOnce() }) - it('is settled by a read after a restart that reaches the chat before the restore does', async () => { + it('is settled by the startup after a restart, even when a read reaches the chat first', async () => { await exitWithUnwrittenSettlement() await rig.restart() setStructuredAgentSessionHost(rig.host) - // The client's read opens the chat first; the restore then finds it open and skips it. + // The client's read opens the chat first and writes nothing. + expect(await workingTurnState()).toBe('running') + await rig.host.reconcileRestartLeases() + await rig.host.startupSettled() expect(await workingTurnState()).toBe('interrupted') await rig.host.restoreReadableSessions([SESSION]) expect(await workingTurnState()).toBe('interrupted') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts index c0652dff2d59..f7b2b67dd459 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-verdict.test.ts @@ -17,6 +17,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { plannedBatchThrough } from './structured-agent-session-settlement-double.test-fixture' import { runningTurnLifecycleRevisions, turnVerdictFromDeathEvidence, @@ -313,7 +314,8 @@ describe('stale session state on a cold acquire', () => { itemFence: () => 1, stopMarks: { latest: () => null }, cursor: () => ({ epoch: 'epoch-1', sequence: 8 }), - appendLifecycleBatch + submissions: () => [], + appendPlannedLifecycleBatch: plannedBatchThrough(appendLifecycleBatch) } as unknown as AgentSessionJournal return { journal, appendLifecycleBatch } } @@ -338,6 +340,7 @@ describe('stale session state on a cold acquire', () => { settlementId: 'stale-session:session-1:14:generation-2', fence: 14, recovered: true, + dispatches: [], mutations: [ { kind: 'item', @@ -368,6 +371,7 @@ describe('stale session state on a cold acquire', () => { settlementId: 'stale-session:session-1:14:generation-2', fence: 14, recovered: true, + dispatches: [], mutations: [ { kind: 'item', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index fed25d7dc9b0..d192a0d01b14 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -11,25 +11,32 @@ import { type StructuredAgentSessionChildExitSession } from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' +import { + exitJournalDouble, + NO_STORE_RECEIPTS +} from './structured-agent-session-settlement-double.test-fixture' const SESSION = 'session-1' const GENERATION = 'generation-1' const REASON = 'claude stream-json exited (code 1): session limit reached' const STARTUP_TEXT = 'Claude stopped before it finished starting. Send your message to try again.' -function startedSession(): StructuredAgentSessionChildExitSession & { - journal: { appendLifecycleBatch: ReturnType } +function startedSession( + submissions?: () => readonly Partial[] +): StructuredAgentSessionChildExitSession & { + appendLifecycleBatch: ReturnType } { + const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })) return { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - // Nothing ran: the start failed before any response or acknowledged prompt. - snapshot: () => ({ items: [] }), - appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })) - } + appendLifecycleBatch, + // Nothing ran: the start failed before any response or acknowledged prompt. + journal: exitJournalDouble({ + items: () => [], + appendLifecycleBatch, + ...(submissions ? { submissions } : {}) + }) } } @@ -53,7 +60,8 @@ function contextFor(session: StructuredAgentSessionChildExitSession) { transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)) + ) => (record = transition(record)), + conversationReceipts: NO_STORE_RECEIPTS }, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), @@ -84,7 +92,7 @@ describe('a provider that ends before it finished starting', () => { startupUnproven: true }) - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect(session.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ mutations: [ expect.objectContaining({ @@ -108,7 +116,7 @@ describe('a provider that ends before it finished starting', () => { await settleStructuredAgentSessionChildExit(contextFor(session), ended) expect(session.child).toBeNull() - expect(session.journal.appendLifecycleBatch).not.toHaveBeenCalled() + expect(session.appendLifecycleBatch).not.toHaveBeenCalled() }) it("reads a start that failed off the host's own phase when the provider omits the flag", async () => { @@ -122,7 +130,7 @@ describe('a provider that ends before it finished starting', () => { failure: { kind: 'providerExited', detail: { text: REASON, audience: 'log' } } }) - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect(session.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ mutations: [ expect.objectContaining({ @@ -145,28 +153,27 @@ describe('a provider that ends before it finished starting', () => { }) it('names /compact as the next step when the start that failed was carrying it', async () => { - const base = startedSession() + const base = startedSession(() => [ + { clientMessageId: 'compact-1', dispatchState: 'pending' as const } + ]) const session = { ...base, child: { generation: GENERATION, fence: 7, phase: 'starting' as const }, - journal: { - ...base.journal, - submissions: () => [{ clientMessageId: 'compact-1', dispatchState: 'pending' as const }], - itemBody: () => structuredAgentSessionCompactBody() - } + journal: { ...base.journal, itemBody: () => structuredAgentSessionCompactBody() } } await settleStructuredAgentSessionChildExit(contextFor(session), ended) const text = 'Claude stopped before it finished starting. Run /compact again.' - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + expect(session.appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ dispatches: [ expect.objectContaining({ clientMessageId: 'compact-1', state: 'rejected', reason: text, - fence: 7, + // At the fence the exit's release moved to. + fence: 8, recovered: true }) ], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts new file mode 100644 index 000000000000..2dde6314cbf9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -0,0 +1,177 @@ +// What the host owes its chats when it comes up as a new instance: the earlier process's leftovers. +// Run after the whole-host restart reconcile, per chat under its serialize, queued before anything +// else can reach that chat, so its share runs first. +// +// For each chat it opens, in the order an open used to: the cards' repair and prune; sends the +// earlier process accepted and never handed over, kept as cards or rejected (`holdUnsentSends`); +// the queue's reopen mark; the leftover settlement (`structured-agent-session-leftover-settlement.ts`) +// when the lease says it is owed; and a rewind it left in doubt. Each step is best effort: a failure +// is logged, and the next trigger re-derives it. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import { agentSessionLeaseIsReleased } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { PrioritySemaphore } from '../../../shared/priority-semaphore' +import { listSessionsWithQueuedMessages } from '../agent-session-journal/queued-message-table' +import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import type { + StructuredAgentSessionHostDeps, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' +import { settleStructuredAgentSessionLeftoversOrLog } from './structured-agent-session-leftover-settlement' +import { + markStructuredQueueReopen, + structuredAgentSessionHostInstance +} from './structured-agent-session-queued-pause' +import { recoverStructuredRewind } from './structured-rewind-recovery' +import type { OpenedStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' + +const STARTUP_SETTLEMENT_CONCURRENCY = 4 + +export type StructuredAgentSessionStartupSettlementContext = { + deps: Pick + sessions: Map + serialize: (sessionId: string, task: () => Promise) => Promise + /** Indexes and publishes a chat this share opened, once it is settled. */ + adopt: (sessionId: string, opened: OpenedStructuredAgentSessionConversation) => Promise + now: () => number + /** Journals whose cards and sends this startup already settled: the restore's share after the + * pass's settles only what its recovery ended, or a second reopen mark would hold a later send. */ + sharesDone: WeakSet +} + +/** Owed: released, and no settlement since the release that ended its last generation. */ +export function structuredAgentSessionLeftoversOwed(record: AgentSessionRecord | null): boolean { + return ( + record !== null && + agentSessionLeaseIsReleased(record.lease) && + record.lease.leftoverSettledAt === null + ) +} + +function rewindInDoubt(record: AgentSessionRecord): boolean { + return record.rewind?.phase === 'prepared' || record.rewind?.phase === 'provider-succeeded' +} + +/** The chats startup opens: owed ones and ones with a rewind in doubt, from the record store, and + * ones holding cards, from an index of the cards' own table. Every other chat opens only to read. */ +function startupCandidates(context: StructuredAgentSessionStartupSettlementContext): string[] { + const { store, journalDatabase, logger } = context.deps + const ids = new Set( + store + .listRecords() + .filter((record) => structuredAgentSessionLeftoversOwed(record) || rewindInDoubt(record)) + .map((record) => record.sessionId) + ) + try { + for (const sessionId of listSessionsWithQueuedMessages(journalDatabase.db)) { + if (store.getRecord(sessionId)) { + ids.add(sessionId) + } + } + } catch (error) { + logger.warn('listing chats with saved cards at startup failed', { + scope: 'startup-settlement', + error + }) + } + return [...ids] +} + +/** Every candidate's share is queued on its chat's serialize at once, so whatever that chat runs + * next (a reader's open and the queue it wakes, a send, an attach) runs after its share; a few + * chats at a time do the work. */ +export async function settleStructuredAgentSessionsAtStartup( + context: StructuredAgentSessionStartupSettlementContext +): Promise { + const slots = new PrioritySemaphore(STARTUP_SETTLEMENT_CONCURRENCY) + await Promise.all( + startupCandidates(context).map((sessionId) => + context + .serialize(sessionId, async () => { + const release = await slots.acquire(0) + try { + // A journal open is synchronous SQLite: a macrotask per chat keeps startup responsive. + await yieldToEventLoop() + await settleOneAtStartup(context, sessionId) + } finally { + release() + } + }) + .catch((error: unknown) => + context.deps.logger.warn('settling a chat at startup failed', { + scope: 'startup-settlement', + sessionId, + error + }) + ) + ) + ) +} + +/** One chat's share, for a caller inside its serialize: the startup pass, or the restore once it + * resolved the chat's recovery (which can end a generation after the pass ran). A chat not yet + * open is settled before it is published, so no reader sees what the share ends; `opened` is + * one the restore opened and publishes itself. A share already done finds nothing left to do. */ +export async function settleOneAtStartup( + context: StructuredAgentSessionStartupSettlementContext, + sessionId: string, + opened?: OpenedStructuredAgentSessionConversation +): Promise { + const indexed = opened?.session ?? context.sessions.get(sessionId) + // No journal: nothing was ever written, so nothing is left to settle. + const fresh = indexed ? null : await restoreStructuredAgentSessionRead(context.deps, sessionId) + const session = indexed ?? fresh?.session + try { + // A child here was acquired since: its acquisition settled first, and its loop owns the queue. + if (session && !session.child) { + await settleShare(context, sessionId, session) + } + } finally { + if (fresh) { + await context.adopt(sessionId, fresh) + } + } +} + +async function settleShare( + context: StructuredAgentSessionStartupSettlementContext, + sessionId: string, + { journal }: StructuredAgentSessionHostSession +): Promise { + const { store, logger } = context.deps + const warn = (scope: string) => (error: unknown) => + logger.warn('startup bookkeeping for a chat failed', { scope, sessionId, error }) + const fence = store.getRecord(sessionId)?.lease.runtimeFence + const firstShare = !context.sharesDone.has(journal) + context.sharesDone.add(journal) + if (firstShare && fence !== undefined) { + await journal.queuedMessages.repairAndPrune().catch(warn('startup-queued-repair')) + // Before a Stop can withdraw one: a Stop never withdraws a card. + await holdUnsentSends(journal, { + fence, + hostInstance: structuredAgentSessionHostInstance(), + hold: { cause: 'hostRestarted' } + }).catch(warn('startup-leftover-sends')) + // After the leftovers became cards, so the mark follows every card the earlier process left. + await markStructuredQueueReopen(sessionId, journal, fence, logger) + } + if (structuredAgentSessionLeftoversOwed(store.getRecord(sessionId))) { + await settleStructuredAgentSessionLeftoversOrLog({ + store, + sessionId, + journal, + now: context.now, + logger, + scope: 'startup-settlement' + }) + } + if (firstShare && fence !== undefined) { + // A rewind the earlier process left prepared refuses every send until settled; one only its + // provider can prove stays for the next acquisition. + await recoverStructuredRewind(context.deps, sessionId, journal, fence).catch( + warn('rewind-recovery') + ) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts index 9fa62fd170e3..0a24a6b805b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.test.ts @@ -360,6 +360,8 @@ describe('StructuredAgentSessionStatusFeed', () => { it('carries the record model and the running tool line the sidebar row shows', async () => { const journal = await openJournal() + // Written by the record's current generation: work an ended one left never reads working. + const fence = agentSessionRecordFixture().lease.runtimeFence const { feed, events } = feedFor(new Map([[SESSION, { journal }]]), { location: indexed({ journal }).params.location, options: { model: 'gpt-5-codex' }, @@ -368,12 +370,12 @@ describe('StructuredAgentSessionStatusFeed', () => { await journal.appendItem( USER_IDENTITY, { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'run the tests' }] }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await journal.appendItem( TURN_IDENTITY, { kind: 'status', text: 'Working', turnLifecycle: { turnId: 'turn-1', state: 'running' } }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) feed.publish(SESSION) expect(events.at(-1)).toEqual({ @@ -384,7 +386,7 @@ describe('StructuredAgentSessionStatusFeed', () => { await journal.appendItem( { ...USER_IDENTITY, ordinal: 2 }, { kind: 'tool-call', name: 'shell', input: { command: 'pnpm test' }, state: 'running' }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) feed.publish(SESSION) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index f28a49c81aea..e672be5a673e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -82,10 +82,12 @@ export class StructuredAgentSessionJournalProjections { submissionCount: snapshot.submissions.length, fence, stopRevision, + // The host knows who wrote each item: an ended generation's work is not Working. state: projectStructuredAgentSessionStatusState( snapshot.items, snapshot.submissions, - fence + fence, + journal.itemFence ), acceptedSendKey: newestAcceptedSendKey(cursor.epoch, snapshot.submissions), stopping: structuredAgentSessionStopping(journal, snapshot.items, snapshot.submissions) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts index c09d2eba456c..03e9108f2e5d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts @@ -180,7 +180,7 @@ describe('a Stop of a start that never landed binds no later turn', () => { await stopOfStart() await mailTurn() const owner = fence() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.host.journalSnapshot(HOST_TEST_SESSION) await settleStaleStructuredAgentSessionState({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts index 55eb9080a5b1..50b25503d7ec 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts @@ -74,7 +74,7 @@ async function runningTurn( async function restartAndSettle( proof: 'pid-absent' | 'exit-observed' | 'unproven' = 'pid-absent' ): Promise { - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.host.journalSnapshot(HOST_TEST_SESSION) const now = Date.now() const deathEvidence: AgentSessionDeathEvidence | null = diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts index a3fee6b2ed40..26e5a0475678 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts @@ -169,11 +169,13 @@ describe("a person's close pressed before its send's turn showed", () => { await rig.host['lifetime'].stopAgent(HOST_TEST_SESSION, { cause: 'user-close' }) expect(journal().stopMarks.latest()?.event).toMatchObject({ reason: 'user-close' }) expect(journal().stopMarks.latest()?.event).not.toHaveProperty('turnId') + // The next child's turn, at the fence the stopped one's end moved to. + const fence = rig.store.getRecord(HOST_TEST_SESSION)!.lease.runtimeFence await journal().appendItem( LATER, { kind: 'turn', turnId: 'turn-later', state: 'running', startedAt: Date.now() }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await journal().appendItem( LATER, @@ -184,7 +186,7 @@ describe("a person's close pressed before its send's turn showed", () => { startedAt: Date.now(), completedAt: Date.now() + 5 }, - { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) const later = journal() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts index 8750fe909a0a..f2001d8da1b4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts @@ -51,7 +51,7 @@ export async function endStoppedStructuredAgentSession( await stopChild() } catch (error) { onError(error) - failedOn = structuredAgentSessionFailedStopMark(ctx.journal) + failedOn = structuredAgentSessionFailedStopMark(ctx.journal, ctx.fence) await reviseStopNoteUnconfirmed(ctx, windDown.stopNote).catch(onError) } finally { windDown.settled?.(failedOn) @@ -85,7 +85,7 @@ async function reviseStopNoteUnconfirmed( // Stop of that turn writes it: its proven end finds it there. A row keeps the scope it was created // with, so a note a Stop wrote before the turn showed is re-keyed, in one batch. Known limit: with // no turn open yet, the note keeps its key and scope, and no turn's end revises it. - const running = ctx.journal.activeTurnId() + const running = ctx.journal.activeTurnId(ctx.fence) const turnScope = running !== null ? structuredAgentSessionNamedTurnScope(ctx.journal, running) : null const onTurn = running !== null ? structuredAgentSessionStopNoteIdentity(running) : identity diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts index 92cd6f574331..479874892264 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts @@ -13,6 +13,7 @@ import { type AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' import type { AgentSessionCancelOutcome } from './structured-agent-session-adapter' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionStatusEvent, AgentSessionStatusSummary @@ -98,7 +99,12 @@ function stopAnswers(): (string | undefined)[] { } /** A send whose turn `turn-1` is running, and the session's status. */ -async function runningTurn(options: { stopEndsSession?: true } = {}) { +async function runningTurn( + options: Pick< + NonNullable[0]>, + 'stopEndsSession' | 'probeOwner' + > = {} +) { rig = await createQueuedMessageTestRig(options) const sent = await rig.workingSend() await rig.settleAccepted(sent, 'sent') @@ -468,12 +474,12 @@ describe('a Stop whose provider ends its session', () => { // The close lives on the child, in memory, and dies with the host. The new host's settlement, // which ends the turn on a proof of the old owner's death, is what says the Stop took. - async function crashAfterUnconfirmedStop(): Promise { - await runningTurn({ stopEndsSession: true }) + async function crashAfterUnconfirmedStop(proof: AgentSessionOwnerProbe): Promise { + await runningTurn({ stopEndsSession: true, probeOwner: async () => proof }) rig.closeSession.mockRejectedValueOnce(new Error('the kill timed out')) expect(await rig.stop()).toMatchObject({ ok: true }) await eventually(() => expect(stopAnswers()).toEqual(['cancelUnconfirmed'])) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() } function turnOneState(): string | undefined { @@ -483,21 +489,9 @@ describe('a Stop whose provider ends its session', () => { .find((turn) => turn?.turnId === 'turn-1')?.state } - /** What the host's restart reconciliation or recovery writes once a probe finds the old pid gone. */ - async function proveOldOwnerGone(): Promise { - const record = rig.store.getRecord(HOST_TEST_SESSION)! - await rig.store.evictProvenDeadOwner({ - sessionId: HOST_TEST_SESSION, - expectedFence: record.lease.runtimeFence, - probe: { outcome: 'pid-absent' }, - now: Date.now() - }) - } - it("says the Stop took once the new host proves the old child's exit after a crash", async () => { - await crashAfterUnconfirmedStop() - // Proven before the chat opens, as the restart's reconciliation does on most machines. - await proveOldOwnerGone() + // Proven by the restart's reconciliation, as on most machines; its startup settles the chat. + await crashAfterUnconfirmedStop({ outcome: 'pid-absent' }) await rig.queuePause() expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ @@ -506,22 +500,21 @@ describe('a Stop whose provider ends its session', () => { }) }) - it('keeps the note unconfirmed while the old exit stays unverifiable, then says it took once proven', async () => { - await crashAfterUnconfirmedStop() + it('keeps the note unconfirmed while the old exit stays unverifiable, through a later restart', async () => { + // The restart cannot prove the old owner gone: it releases it unproven, and startup settles. + await crashAfterUnconfirmedStop({ outcome: 'indeterminate', reason: 'no answer' }) await rig.queuePause() expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ turn: 'unverifiable', notes: ['cancelUnconfirmed'] }) - // A later proof naming the old owner, as recovery writes it, revises the open chat. - await proveOldOwnerGone() - await eventually(() => - expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ - turn: 'interrupted', - notes: ['took'] - }) - ) + // A later restart that proves nothing more leaves them as they are. + await rig.crashRestartHostProcess() + expect({ turn: turnOneState(), notes: stopAnswers() }).toEqual({ + turn: 'unverifiable', + notes: ['cancelUnconfirmed'] + }) }) // A Stop pressed before its turn showed keys its note by itself, with no turn to sit on; once it is diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts index 284a474b9c65..0cf15693b3ce 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts @@ -64,9 +64,10 @@ function firstSendHandedOverAfter( /** Where a Stop that failed leaves "Stopping…" (display only): the turn running now, or, with * none, the first that opens after this position. */ export function structuredAgentSessionFailedStopMark( - journal: Pick + journal: Pick, + fence: number ): JournalStopFailedOn { - const live = journal.activeTurnId() + const live = journal.activeTurnId(fence) return live !== null ? { turnId: live } : { openedAfter: journal.cursor().sequence } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index 19fac5d20b8c..568922e9cd4f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -38,7 +38,7 @@ export function isMainAgentWorking( ctx: Pick ): boolean { return isStructuredAgentSessionMainAgentWorking( - ctx.journal.activeTurnId(), + ctx.journal.activeTurnId(ctx.fence), ctx.journal.submissions(), ctx.fence ) @@ -57,7 +57,7 @@ function stillRunsStoppedTurn( return false } // Working with no turn open after the Stop's turn is a later send whose turn has not opened. - return stoppedTurnId === null || ctx.journal.activeTurnId() === stoppedTurnId + return stoppedTurnId === null || ctx.journal.activeTurnId(ctx.fence) === stoppedTurnId } /** Whether the Stop took back every send it found in flight, none of them having run @@ -147,6 +147,8 @@ type PerformCancelInput = { childWork?: () => readonly AgentChildWorkView[] | undefined /** The latest Stop event is this press's own, or the in-force one it repeats: its settle binds. */ opensSettle?: true + /** The fence the note is written at: a child end this Stop caused moved it past `ctx.fence`. */ + noteFence?: () => number } async function cancelAndNote( @@ -168,7 +170,7 @@ async function cancelAndNote( // Only the provider's end or the child's ends a command. A command the provider has not opened a // turn for, would not interrupt, or was already asked to stop, ends with its child; that child's // dead-generation settlement writes the command's verdict. - const liveTurnId = ctx.journal.activeTurnId() + const liveTurnId = ctx.journal.activeTurnId(ctx.fence) // Read with the live turn, before the cancel settles it: the note is keyed by this turn. const stoppedTurnId = structuredAgentSessionStoppedTurnId(ctx.journal, input.turnId) const namesTurnNotLive = structuredAgentSessionStopNamesTurnNotLive(input.turnId, liveTurnId) @@ -212,7 +214,7 @@ async function cancelAndNote( ...(input.turnId !== undefined ? { turnId: input.turnId } : {}), fence: ctx.fence, // The journal is what the client read to name a turn, so it is what judges the request. - resolveLiveTurnId: () => ctx.journal.activeTurnId(), + resolveLiveTurnId: () => ctx.journal.activeTurnId(ctx.fence), ...(dispatchStatus ? { dispatchStatus } : {}), ...(input.prompt ? { prompt: { itemId: input.prompt.itemId } } : {}) }) @@ -309,7 +311,7 @@ async function cancelAndNote( binding.turnId = stoppedTurn ?? stoppedTurnId ?? undefined } else if (interruptFailed) { // A Stop that failed reads "Stopping…" through the turn it could not stop; its end stays its own. - binding.failedOn = structuredAgentSessionFailedStopMark(ctx.journal) + binding.failedOn = structuredAgentSessionFailedStopMark(ctx.journal, ctx.fence) } if (taken === true && stoppedTurn !== undefined && !binding.closedByWindDown && !input.scope) { const opened = await settleTakenStop(ctx, stoppedTurn) @@ -328,6 +330,7 @@ async function cancelAndNote( (stoppedTurn !== undefined ? structuredAgentSessionNamedTurnScope(ctx.journal, stoppedTurn) : null) ?? turnScope - await ctx.journal.appendItem(noteIdentity, note, { fence: ctx.fence, turnScope: noteScope }) + const fence = Math.max(ctx.fence, input.noteFence?.() ?? ctx.fence) + await ctx.journal.appendItem(noteIdentity, note, { fence, turnScope: noteScope }) return { ok: true, value } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts index 3a2b9a205f28..301022c42c6a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts @@ -24,6 +24,7 @@ import { AgentSessionPromptUnavailableError } from './structured-agent-session-adapter' import { settledPrompt, validatePendingPrompt } from './structured-agent-session-prompt-state' +import { isStructuredAgentSessionEndedGenerationWork } from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' export type AgentSessionPromptRequest = { @@ -97,6 +98,10 @@ export async function performPrompt( return held ? { ok: true, value: held } : validated } const { prompt } = validated + if (isStructuredAgentSessionEndedGenerationWork(ctx.journal.itemFence(input.itemId), ctx.fence)) { + // The agent that asked has ended: an answer would reach whichever agent runs now. + return invalid('promptGone', `Item ${input.itemId} was raised by an agent that has ended.`) + } const choice = readPromptChoice(prompt, input) if (!choice) { return invalid( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index ab73a2a786aa..aef5e47314dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -14,6 +14,12 @@ import { type StructuredAgentSessionChildExitSession } from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' +import { + exitJournalDouble, + NO_STORE_RECEIPTS, + plannedBatchThrough +} from './structured-agent-session-settlement-double.test-fixture' +import type { JournalLifecycleBatchInput } from '../agent-session-journal/journal-store-contracts' const exitOutcome = (agent: string): string => `${agent} stopped while this response was in progress. You can continue in this conversation.` @@ -67,7 +73,8 @@ function mutableStore() { transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)) + ) => (record = transition(record)), + conversationReceipts: NO_STORE_RECEIPTS } } } @@ -83,7 +90,8 @@ describe('provider-exit settlement', () => { transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)) + ) => (record = transition(record)), + conversationReceipts: NO_STORE_RECEIPTS } const appendLifecycleBatch = vi .fn() @@ -100,7 +108,8 @@ describe('provider-exit settlement', () => { }), itemFence: () => 7, stopMarks: { latest: () => null, personStopDecides: () => false }, - appendLifecycleBatch + submissions: () => [], + appendPlannedLifecycleBatch: plannedBatchThrough(appendLifecycleBatch) } } as unknown as StructuredAgentSessionHostSession @@ -172,7 +181,8 @@ describe('provider-exit settlement', () => { itemBody: () => null, itemFence: () => undefined, snapshot: () => ({ items }), - appendLifecycleBatch + submissions: () => [], + appendPlannedLifecycleBatch: plannedBatchThrough(appendLifecycleBatch) } } as unknown as StructuredAgentSessionHostSession const store = { @@ -186,7 +196,8 @@ describe('provider-exit settlement', () => { reservedSpawnToken: null } }), - transitionHandoff: async () => ({ lease: { runtimeFence: 8 } }) + transitionHandoff: async () => ({ lease: { runtimeFence: 8 } }), + conversationReceipts: NO_STORE_RECEIPTS } await settleStructuredAgentSessionChildExit( @@ -277,19 +288,13 @@ describe('provider-exit settlement', () => { ...(initialState === 'running' ? {} : { completedAt: 40 }) }) ] - const appendLifecycleBatch = vi.fn(async (_input: { mutations: readonly unknown[] }) => ({ + const appendLifecycleBatch = vi.fn(async (_input: JournalLifecycleBatchInput) => ({ epoch: 'epoch-1', sequence: 3 })) const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - snapshot: () => ({ items }), - appendLifecycleBatch - } + journal: exitJournalDouble({ items: () => items, appendLifecycleBatch }) } const { store } = mutableStore() @@ -338,16 +343,15 @@ describe('provider-exit settlement', () => { ) it('settles a submission the dead child never acknowledged', async () => { + const appendLifecycleBatch = vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })) const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - snapshot: () => ({ items: [] }), - appendLifecycleBatch: vi.fn(async () => ({ epoch: 'epoch-1', sequence: 1 })), - submissions: () => [{ clientMessageId: 'client-1', dispatchState: 'pending' }] - } + journal: exitJournalDouble({ + items: () => [], + appendLifecycleBatch, + // Handed to the child that exited, at its fence. + submissions: () => [{ clientMessageId: 'client-1', dispatchState: 'pending', fence: 7 }] + }) } const { store } = mutableStore() @@ -369,14 +373,16 @@ describe('provider-exit settlement', () => { acquisitionGeneration: GENERATION }) - expect(session.journal.appendLifecycleBatch).toHaveBeenCalledWith( + // Written at the fence the exit's release moved to, as is every row its settlement writes. + expect(appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ + fence: 8, dispatches: [ { clientMessageId: 'client-1', state: 'unknown', reason: 'provider_exited_before_acknowledgement', - fence: 7, + fence: 8, recovered: true } ], @@ -397,17 +403,12 @@ describe('provider-exit settlement', () => { it('releases without offering a restart while terminal settlement is failing', async () => { const session: StructuredAgentSessionChildExitSession = { child: { generation: GENERATION, fence: 7, phase: 'ready' }, - journal: { - cursor: () => ({ epoch: 'epoch-1', sequence: 0 }), - itemBody: () => null, - itemFence: () => undefined, - snapshot: () => ({ - items: [lifecycleItem('turn-failing', 1, { state: 'running', startedAt: 1 })] - }), + journal: exitJournalDouble({ + items: () => [lifecycleItem('turn-failing', 1, { state: 'running', startedAt: 1 })], appendLifecycleBatch: vi.fn(async () => { throw new Error('journal still unavailable') }) - } + }) } const log = recordingStructuredAgentSessionLogger() const publishFence = vi.fn() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts index ed1e642da39e..e0a98e7d7716 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unfinished-work.ts @@ -11,7 +11,7 @@ import { import type { UnopenedSendJournal } from './structured-agent-session-unopened-send-withdrawal' export type DeadGenerationJournal = UnopenedSendJournal & { - appendLifecycleBatch: AgentSessionJournal['appendLifecycleBatch'] + appendPlannedLifecycleBatch: AgentSessionJournal['appendPlannedLifecycleBatch'] pendingSubmissions?: AgentSessionJournal['pendingSubmissions'] itemFence: AgentSessionJournal['itemFence'] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts index e1022722955f..deeb87fb23e9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unopened-send-withdrawal.ts @@ -24,7 +24,7 @@ export type UnopenedSendJournal = { | 'handoverRecorded' | 'handedOverAt' | 'acceptedSequence' - > & { reason?: string | null })[] + > & { reason?: string | null; fence?: number })[] resolveDispatch?: AgentSessionJournal['resolveDispatch'] } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts index e4de581bf067..1a2f7b1f2bb1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unrun-send-hold.test.ts @@ -106,7 +106,7 @@ describe('a message held behind a start that never answered, then the chat ends' async (cause) => { const id = await heldBehindHungStart('kept at close') await rig.host.close(SESSION, cause) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts index 872d08c1da40..b80c5cd51878 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts @@ -110,9 +110,10 @@ async function closeWhileStarting(cause: 'user-close' | 'evict'): Promise await closed } -/** The app quits: delivery stops first, so the held spawn finishing hands nothing over. */ -async function quitRestart(): Promise { - const quit = rig.quitRestartHostProcess() +/** The app quits: delivery stops first, so the held spawn finishing hands nothing over. + * `beforeStartup` reads the state directory before the next process starts. */ +async function quitRestart(beforeStartup?: () => unknown): Promise { + const quit = rig.quitRestartHostProcess(beforeStartup) releaseStart() releaseStart = () => {} await quit @@ -161,17 +162,19 @@ describe('a message accepted while the agent starts, then Orca stops', () => { }) // Nothing is sent on its own, however many times the chat reopens. await rig.host.close(SESSION, 'evict') - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }]) expect(rig.dispatch).not.toHaveBeenCalled() } ) - it('quit writes nothing for it: no closed-chat rejection, still queued until the next open', async () => { + it('quit writes nothing for it: no closed-chat rejection, still pending until the next startup', async () => { const id = await acceptWhileStarting(sendRequest('kept through quit')) - await quitRestart() - // Read beside the host, without opening the chat through it: the row the quit left. - const leftover = await peekSubmission(id) + let leftover: Awaited> | undefined + // Read beside the host, between the two processes: the row the quit left. + await quitRestart(async () => { + leftover = await peekSubmission(id) + }) expect(leftover).toMatchObject({ dispatchState: 'pending', handoverRecorded: true }) expect(leftover?.handedOverAt).toBeUndefined() }) @@ -290,7 +293,7 @@ describe('the next accepted turn releases a kept card', () => { const id = await acceptWhileStarting(sendRequest('kept twice over')) await quitRestart() expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }]) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await new Promise((resolve) => setTimeout(resolve, 100)) expect(rig.dispatch).not.toHaveBeenCalled() const working = await rig.workingSend() @@ -352,11 +355,7 @@ describe('cards queued behind a working turn, then Orca stops', () => { ok: true, value: { queued: { state: 'waiting' } } }) - if (how === 'quit') { - await quitRestart() - } else { - rig.crashRestartHostProcess() - } + await (how === 'quit' ? quitRestart() : rig.crashRestartHostProcess()) expect(await rig.drafts()).toEqual([ { messageId: first.id, state: 'waiting' }, @@ -429,7 +428,7 @@ describe('the queue at a quit', () => { } owed.mockRestore() healing.mockRestore() - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.drafts()).toEqual([{ messageId: queued.id, state: 'waiting' }]) expect(await rig.queuePause()).toBeNull() @@ -470,7 +469,7 @@ describe('a message accepted while the agent starts, then the chat closes', () = async (cause) => { const id = await acceptWhileStarting(sendRequest('kept at close')) await closeWhileStarting(cause) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', ...CHAT_CLOSED }) expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }]) @@ -583,7 +582,7 @@ describe('what is not kept', () => { vi.spyOn(JournalQueuedMessages.prototype, 'holdInTransaction').mockImplementation(() => { throw new Error('disk full') }) - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await eventually(async () => expect(await rig.submission(id)).toMatchObject({ @@ -758,7 +757,7 @@ describe("the reopen's mark", () => { await queued.result void working const sent = rig.dispatch.mock.calls.length - rig.crashRestartHostProcess() + await rig.crashRestartHostProcess() await rig.host.restoreReadableSessions([SESSION]) expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(true) await new Promise((resolve) => setTimeout(resolve, 100)) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 436d7075643e..05b46d867d1e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -38,7 +38,8 @@ import { } from '../../runtime/agent-session-record-store-test-harness' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { JournalLifecycleBatchAppender } from '../agent-session-journal/journal-lifecycle-batch-appender' import * as recoveryResolution from './structured-agent-session-recovery-resolution' import { StructuredAgentSessionHost } from './structured-agent-session-host' import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' @@ -462,9 +463,9 @@ describe('already-wedged profiles become usable on load', () => { : { outcome: 'pid-absent' }, stopOwnerProcess }) - // The read restore's settlement fails, and nothing retries it. + // The startup settlement fails, and nothing retries it before the next acquisition. const failing = vi - .spyOn(AgentSessionJournal.prototype, 'appendLifecycleBatch') + .spyOn(JournalLifecycleBatchAppender.prototype, 'appendPlanned') .mockRejectedValue(new Error('journal unavailable')) await host.restoreReadableSessions() failing.mockRestore() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index aae177bbd650..cd901310d8dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -200,7 +200,8 @@ export function structuredAgentSessionWorkingAtStop(input: { const status = structuredAgentSessionShownStatus( { items: current.items, submissions: handedOver }, childWork, - session.child.fence + session.child.fence, + session.journal.itemFence ) if (status.state === 'done') { return null diff --git a/src/main/runtime/agent-session-conversation-receipts.ts b/src/main/runtime/agent-session-conversation-receipts.ts index 6935f8bb0836..c6c5713fbe08 100644 --- a/src/main/runtime/agent-session-conversation-receipts.ts +++ b/src/main/runtime/agent-session-conversation-receipts.ts @@ -5,6 +5,7 @@ import { } from './agent-session-conversation-command-record' import { settleAgentSessionOperationInto } from './agent-session-operation-admission' import { commitAgentSessionRewindCompletion } from './agent-session-rewind-completion' +import { markAgentSessionLeftoverSettled } from './agent-session-lease-transitions' export function createAgentSessionConversationReceipts( transactions: Pick @@ -30,6 +31,14 @@ export function createAgentSessionConversationReceipts( ...args: Parameters extends [unknown, ...infer A] ? A : never - ) => transactions.receipt((draft) => commitAgentSessionRewindCompletion(draft, ...args)) + ) => transactions.receipt((draft) => commitAgentSessionRewindCompletion(draft, ...args)), + /** Committed with the settlement of what ended generations left: see `leftoverSettledAt`. */ + leftoverSettled: (sessionId: string, plannedFence: number, now: number) => + transactions.receipt((draft) => { + const record = draft.records.get(sessionId) + if (record) { + draft.records.set(sessionId, markAgentSessionLeftoverSettled(record, plannedFence, now)) + } + }) } } diff --git a/src/main/runtime/agent-session-lease-transitions.ts b/src/main/runtime/agent-session-lease-transitions.ts index 950e05a8e78e..0581dff6904f 100644 --- a/src/main/runtime/agent-session-lease-transitions.ts +++ b/src/main/runtime/agent-session-lease-transitions.ts @@ -33,11 +33,31 @@ export type AgentSessionReservation = { now: number } +/** Every lease write. One that ends a generation (moves the fence and leaves no owner) marks what + * that generation left as owed; only the settlement's own receipt marks it settled. */ export function withLease( record: AgentSessionRecord, lease: AgentSessionLease ): AgentSessionRecord { - return { ...record, lease, updatedAt: lease.lastRenewedAt } + const endsGeneration = + lease.claimStatus === 'released' && lease.runtimeFence !== record.lease.runtimeFence + return { + ...record, + lease: endsGeneration ? { ...lease, leftoverSettledAt: null } : lease, + updatedAt: lease.lastRenewedAt + } +} + +/** The settlement's receipt: marks the leftovers settled only while the lease still has the fence + * the settlement was planned at, so a generation that ended meanwhile stays owed. */ +export function markAgentSessionLeftoverSettled( + record: AgentSessionRecord, + plannedFence: number, + now: number +): AgentSessionRecord { + return record.lease.runtimeFence === plannedFence && record.lease.claimStatus === 'released' + ? { ...record, lease: { ...record.lease, leftoverSettledAt: now } } + : record } export function assertFence(lease: AgentSessionLease, fence: number): void { diff --git a/src/main/runtime/agent-session-record-founding.ts b/src/main/runtime/agent-session-record-founding.ts index 41386401351e..57d5300096b4 100644 --- a/src/main/runtime/agent-session-record-founding.ts +++ b/src/main/runtime/agent-session-record-founding.ts @@ -64,7 +64,9 @@ export function foundAgentSessionRecord( claimKeyId: lease.claimKeyId, claimStatus: 'released', unreconciled: false, - deathEvidence: null + deathEvidence: null, + // No generation ran yet, so none left anything. + leftoverSettledAt: lease.now } } } diff --git a/src/main/runtime/agent-session-reservation-admission.ts b/src/main/runtime/agent-session-reservation-admission.ts index ff840c755b35..4ae12274c1b3 100644 --- a/src/main/runtime/agent-session-reservation-admission.ts +++ b/src/main/runtime/agent-session-reservation-admission.ts @@ -300,7 +300,9 @@ function createAgentSessionRecord( claimKeyId: reservation.claimKeyId, claimStatus: 'reserved', unreconciled: false, - deathEvidence: null + deathEvidence: null, + // A new record: no earlier generation left anything. + leftoverSettledAt: reservation.now } } } diff --git a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts index d29159ec1116..c6ae8a29d9d9 100644 --- a/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts +++ b/src/main/runtime/structured-agent-session-startup-tab-restore.test.ts @@ -339,6 +339,8 @@ describe('restoring the chat tabs open at quit', () => { const tabWrites = spyOnTabWrites() const { runtime, published } = startupRuntime({ profileChats: [CHAT_A, CHAT_B] }) await runtime.prepareStructuredAgentSessionStartupRestoration() + // Startup's settlement of what the legacy chats' last process left lands first. + await getStructuredAgentSessionHost()?.startupSettled() // One more write, then failing: a seed written chat by chat would stop part way. writes.grants = 1 diff --git a/src/main/runtime/structured-worker-identity.test.ts b/src/main/runtime/structured-worker-identity.test.ts index 17ba897de479..7776e04cde85 100644 --- a/src/main/runtime/structured-worker-identity.test.ts +++ b/src/main/runtime/structured-worker-identity.test.ts @@ -56,7 +56,8 @@ function record(overrides: { claimKeyId: 'k', claimStatus: overrides.claimStatus ?? 'live', unreconciled: false, - deathEvidence: null + deathEvidence: null, + leftoverSettledAt: null }, createdAt: 0, updatedAt: 0 diff --git a/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx b/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx index d78e370619cc..e99acdc69e25 100644 --- a/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx +++ b/src/renderer/src/components/native-chat/NativeChatSubagentRun.tsx @@ -156,8 +156,8 @@ export function NativeChatSubagentEntries({ * reporting into this group long after a newer turn opened, so a turn boundary * is a fact about the turn and never evidence that contact with a child was * lost. Only a host can say that, and one does: `CodexSubagentRoster.settleSession` - * when the provider goes away, and `staleSubagentRosterRevisions` on the next - * journal open when the host itself died mid-flight. */ + * when the provider goes away, and the leftover settlement at the next ownership + * event when the host itself died mid-flight. */ export function NativeChatSubagentRun({ block, open: heldOpen, diff --git a/src/shared/agent-session-lease-adjudication.test.ts b/src/shared/agent-session-lease-adjudication.test.ts index 35e794efd0d2..9c71b87c2f39 100644 --- a/src/shared/agent-session-lease-adjudication.test.ts +++ b/src/shared/agent-session-lease-adjudication.test.ts @@ -34,6 +34,7 @@ function lease(overrides: Partial = {}): AgentSessionLease { claimStatus: 'live', unreconciled: false, deathEvidence: null, + leftoverSettledAt: null, ...overrides } } diff --git a/src/shared/agent-session-legacy-handoff-lease.ts b/src/shared/agent-session-legacy-handoff-lease.ts index ce06ddce0eb7..ea99a66d472d 100644 --- a/src/shared/agent-session-legacy-handoff-lease.ts +++ b/src/shared/agent-session-legacy-handoff-lease.ts @@ -24,9 +24,14 @@ export type PersistedAgentSessionRuntimeKind = export type PersistedAgentSessionHandoffStage = AgentSessionHandoffStage | LegacyHandoffStage /** A lease as it may appear on disk. */ -export type PersistedAgentSessionLease = Omit & { +export type PersistedAgentSessionLease = Omit< + AgentSessionLease, + 'runtimeKind' | 'handoffStage' | 'leftoverSettledAt' +> & { runtimeKind: PersistedAgentSessionRuntimeKind handoffStage: PersistedAgentSessionHandoffStage | null + /** Absent on a lease an older build wrote: read as owed, so startup settles it once. */ + leftoverSettledAt?: number | null } /** A record as a row stores it. Decode through `decodePersistedAgentSessionRecord`. */ @@ -71,11 +76,13 @@ export function normalizeLegacyHandoffLease(lease: PersistedAgentSessionLease): const { runtimeKind, handoffStage } = lease // Why: every one of these awaited proof about an owner, which is what `recovering` resolves. const stage = isLegacyHandoffStage(handoffStage) ? 'recovering' : handoffStage + const leftoverSettledAt = lease.leftoverSettledAt ?? null if (runtimeKind === 'native') { - return { ...lease, runtimeKind, handoffStage: stage } + return { ...lease, runtimeKind, handoffStage: stage, leftoverSettledAt } } return { ...lease, + leftoverSettledAt, runtimeKind: 'native', handoffStage: stage, // Why: a recorded terminal is the user's foreground agent. `conflicted` is the claim every diff --git a/src/shared/agent-session-record.test-fixture.ts b/src/shared/agent-session-record.test-fixture.ts index 68a190fbbcae..7ba49f60b17e 100644 --- a/src/shared/agent-session-record.test-fixture.ts +++ b/src/shared/agent-session-record.test-fixture.ts @@ -34,6 +34,7 @@ export function agentSessionLeaseFixture( claimStatus: 'live', unreconciled: false, deathEvidence: null, + leftoverSettledAt: null, ...overrides } } diff --git a/src/shared/agent-session-record.ts b/src/shared/agent-session-record.ts index 185c62c8333e..94ce27cea808 100644 --- a/src/shared/agent-session-record.ts +++ b/src/shared/agent-session-record.ts @@ -141,6 +141,9 @@ export type AgentSessionLease = { minimumNextFence?: number /** Null on a released lease when nothing proved its owner gone. */ deathEvidence: AgentSessionDeathEvidence | null + /** When the work the generations before this fence left unfinished was last settled, or null + * while it is owed. An index for startup only: every settlement re-derives from the journal. */ + leftoverSettledAt: number | null } export type AgentSessionRecord = { @@ -339,7 +342,8 @@ function isPersistedAgentSessionLease(value: unknown): value is PersistedAgentSe lease.claimStatus === 'conflicted' || lease.claimStatus === 'released') && typeof lease.unreconciled === 'boolean' && - (lease.deathEvidence === null || isAgentSessionDeathEvidence(lease.deathEvidence)) + (lease.deathEvidence === null || isAgentSessionDeathEvidence(lease.deathEvidence)) && + (lease.leftoverSettledAt == null || Number.isSafeInteger(lease.leftoverSettledAt)) ) } diff --git a/src/shared/structured-agent-session-live-turn.ts b/src/shared/structured-agent-session-live-turn.ts index a716b8f4a131..5049e55409d0 100644 --- a/src/shared/structured-agent-session-live-turn.ts +++ b/src/shared/structured-agent-session-live-turn.ts @@ -27,6 +27,10 @@ import { isRootAgentJournalItem } from './agent-session-journal-producer' import type { AgentSessionLatestTurn, AgentSessionSubscribeEvent } from './agent-session-wire' import { readAgentJournalTurn } from './agent-session-turn-record' import { isAgentSessionContextClear } from './agent-session-context-clear' +import { + isStructuredAgentSessionEndedGenerationWork, + type StructuredAgentSessionItemFence +} from './structured-agent-session-main-agent-working' import type { NativeChatToolCallBlock } from './native-chat-types' import { isRunningStructuredAgentSessionToolAction, @@ -35,16 +39,20 @@ import { type StructuredAgentSessionToolAction } from './structured-agent-session-tool-call-block' +/** `current`: whether the running turn's item is current work; one an ended generation opened is + * none (`isStructuredAgentSessionEndedGenerationWork`). */ export function activeStructuredAgentSessionTurnId( - items: readonly AgentJournalRenderItem[] + items: readonly AgentJournalRenderItem[], + current?: (item: AgentJournalRenderItem) => boolean ): string | null { for (let index = items.length - 1; index >= 0; index -= 1) { - if (isAgentSessionContextClear(items[index]?.body)) { + const item = items[index] + if (isAgentSessionContextClear(item?.body)) { return null } - const turn = readAgentJournalTurn(items[index]?.body) + const turn = readAgentJournalTurn(item?.body) if (turn) { - return turn.state === 'running' ? turn.turnId : null + return turn.state === 'running' && (!current || (item && current(item))) ? turn.turnId : null } } return null @@ -81,6 +89,14 @@ export function newestStructuredAgentSessionTurnBySequence( export function liveStructuredAgentSessionTurnScope( items: Iterable ): AgentJournalTurnScope { + const running = runningStructuredAgentSessionTurnItemBySequence(items) + return running ? { kind: 'turn', turnItemId: running.item.itemId } : AGENT_JOURNAL_THREAD_SCOPE +} + +/** The newest turn record's item when that turn is running, by sequence. */ +export function runningStructuredAgentSessionTurnItemBySequence( + items: Iterable +): { item: AgentJournalRenderItem; turnId: string } | null { let newest: AgentJournalRenderItem | null = null for (const item of items) { if ( @@ -90,17 +106,22 @@ export function liveStructuredAgentSessionTurnScope( newest = item } } - return newest && readAgentJournalTurn(newest.body)?.state === 'running' - ? { kind: 'turn', turnItemId: newest.itemId } - : AGENT_JOURNAL_THREAD_SCOPE + const turn = newest ? readAgentJournalTurn(newest.body) : null + return newest && turn?.state === 'running' ? { item: newest, turnId: turn.turnId } : null } -/** Whether that newest turn is still running, which is all most callers want. */ -export function activeStructuredAgentSessionTurnIdBySequence( - items: Iterable +/** The running turn by sequence, given the lease's fence: one an ended generation opened is none + * (`isStructuredAgentSessionEndedGenerationWork`). The host's one read of its active turn. */ +export function currentStructuredAgentSessionTurnId( + items: Iterable, + itemFence: StructuredAgentSessionItemFence, + currentFence?: number | null ): string | null { - const newest = newestStructuredAgentSessionTurnBySequence(items) - return newest?.state === 'running' ? newest.turnId : null + const running = runningStructuredAgentSessionTurnItemBySequence(items) + return running && + !isStructuredAgentSessionEndedGenerationWork(itemFence(running.item.itemId), currentFence) + ? running.turnId + : null } /** The newest turn record whatever state it ended in, STATE INCLUDED. Restart resume compares both diff --git a/src/shared/structured-agent-session-main-agent-working.ts b/src/shared/structured-agent-session-main-agent-working.ts index 05e3b775aaa9..0b03765cded8 100644 --- a/src/shared/structured-agent-session-main-agent-working.ts +++ b/src/shared/structured-agent-session-main-agent-working.ts @@ -2,7 +2,8 @@ import type { AgentJournalSubmission } from './agent-session-journal-types' import { hasUnansweredStructuredAgentSessionDispatch } from './structured-agent-session-unanswered-dispatch' /** Whether the session's own agent is working: a running turn, or a send it has not answered. - * The one rule behind every session list's Working and the chat's own Stop. */ + * The one rule behind every session list's Working and the chat's own Stop. Callers pass the turn + * scoped to `currentFence` (`isStructuredAgentSessionEndedGenerationWork`). */ export function isStructuredAgentSessionMainAgentWorking( activeTurnId: string | null, submissions: readonly AgentJournalSubmission[], @@ -12,3 +13,17 @@ export function isStructuredAgentSessionMainAgentWorking( activeTurnId !== null || hasUnansweredStructuredAgentSessionDispatch(submissions, currentFence) ) } + +/** The fence of the writer that created a journal item, where the reader knows it: the host does, + * a client does not. */ +export type StructuredAgentSessionItemFence = (itemId: string) => number | undefined + +/** Work a generation below the lease's fence wrote — a running turn, a pending prompt. Every release + * moves the fence, so that generation can no longer finish it: it holds nothing and is not + * Working, whether or not its settlement has landed yet. */ +export function isStructuredAgentSessionEndedGenerationWork( + itemFence: number | undefined, + currentFence: number | null | undefined +): boolean { + return currentFence != null && itemFence !== undefined && itemFence < currentFence +} diff --git a/src/shared/structured-agent-session-owed-work.ts b/src/shared/structured-agent-session-owed-work.ts index fe7c9695ae73..3024e2af867b 100644 --- a/src/shared/structured-agent-session-owed-work.ts +++ b/src/shared/structured-agent-session-owed-work.ts @@ -1,6 +1,10 @@ import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' import { activeStructuredAgentSessionTurnId } from './structured-agent-session-live-turn' -import { isStructuredAgentSessionMainAgentWorking } from './structured-agent-session-main-agent-working' +import { + isStructuredAgentSessionEndedGenerationWork, + isStructuredAgentSessionMainAgentWorking, + type StructuredAgentSessionItemFence +} from './structured-agent-session-main-agent-working' import { agentSessionCurrentContextRows } from './agent-session-context-clear' /** A running turn or an unanswered send: what a `working` status means, and what an `attention` @@ -8,11 +12,17 @@ import { agentSessionCurrentContextRows } from './agent-session-context-clear' export function owesStructuredAgentSessionWork( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[], - currentFence?: number | null + currentFence?: number | null, + itemFence?: StructuredAgentSessionItemFence ): boolean { const current = agentSessionCurrentContextRows(items, submissions) return isStructuredAgentSessionMainAgentWorking( - activeStructuredAgentSessionTurnId(current.items), + activeStructuredAgentSessionTurnId( + current.items, + itemFence && + ((item) => + !isStructuredAgentSessionEndedGenerationWork(itemFence(item.itemId), currentFence)) + ), current.submissions, currentFence ) diff --git a/src/shared/structured-agent-session-projection.ts b/src/shared/structured-agent-session-projection.ts index d5b08c833665..7c9b1a4076d7 100644 --- a/src/shared/structured-agent-session-projection.ts +++ b/src/shared/structured-agent-session-projection.ts @@ -39,6 +39,10 @@ import { sha256 } from './sha256' import { readAgentMessageSource } from './agent-session-message-source' import { structuredAgentSessionStatusStartedAt } from './structured-agent-session-status-started-at' import { owesStructuredAgentSessionWork } from './structured-agent-session-owed-work' +import { + isStructuredAgentSessionEndedGenerationWork, + type StructuredAgentSessionItemFence +} from './structured-agent-session-main-agent-working' import { agentSessionCurrentContextRows } from './agent-session-context-clear' // Re-exported so the live-turn readers' and the unanswered-send rule's existing consumers keep one @@ -211,23 +215,33 @@ export function structuredAgentSessionTabId(sessionId: string): string { return `structured-agent-session-${sessionId}` } -function isPendingStructuredAgentSessionPrompt(item: AgentJournalRenderItem): boolean { - return ( +/** A prompt waiting on the user, unless an ended generation raised it: nothing can take its answer. */ +function pendingStructuredAgentSessionPrompt( + currentFence?: number | null, + itemFence?: StructuredAgentSessionItemFence +): (item: AgentJournalRenderItem) => boolean { + return (item) => (item.body.kind === 'approval' || item.body.kind === 'question') && - item.body.resolution.state === 'pending' - ) + item.body.resolution.state === 'pending' && + !( + itemFence && isStructuredAgentSessionEndedGenerationWork(itemFence(item.itemId), currentFence) + ) } +/** `itemFence`: a host's, so work an ended generation left neither holds nor reads as Working. */ export function projectStructuredAgentSessionStatus( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[] = [], - currentFence?: number | null + currentFence?: number | null, + itemFence?: StructuredAgentSessionItemFence ): StructuredAgentSessionProjectedStatus { ;({ items, submissions } = agentSessionCurrentContextRows(items, submissions)) - if (items.some(isPendingStructuredAgentSessionPrompt)) { + if (items.some(pendingStructuredAgentSessionPrompt(currentFence, itemFence))) { return 'attention' } - return owesStructuredAgentSessionWork(items, submissions, currentFence) ? 'working' : 'idle' + return owesStructuredAgentSessionWork(items, submissions, currentFence, itemFence) + ? 'working' + : 'idle' } /** The activity fields a sidebar row shows beside the prompt, named as the agent-status @@ -266,7 +280,8 @@ export function projectStructuredAgentSessionStatusSummary( export function projectStructuredAgentSessionStatusState( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[] = [], - currentFence?: number | null + currentFence?: number | null, + itemFence?: StructuredAgentSessionItemFence ): { summary: StructuredAgentSessionStatusProjection latestRequest: StructuredAgentSessionLatestRequest | null @@ -285,7 +300,7 @@ export function projectStructuredAgentSessionStatusState( } } ;({ items, submissions } = agentSessionCurrentContextRows(items, submissions)) - const status = projectStructuredAgentSessionStatus(items, submissions, currentFence) + const status = projectStructuredAgentSessionStatus(items, submissions, currentFence, itemFence) const statusToolCall = status === 'working' ? statusStructuredAgentSessionToolCall(items) : null const toolName = statusToolCall ? normalizeOptionalField(statusToolCall.name, AGENT_STATUS_TOOL_NAME_MAX_LENGTH) @@ -315,10 +330,14 @@ export function projectStructuredAgentSessionStatusState( ) return { latestRequest, - owesWork: status !== 'idle' && owesStructuredAgentSessionWork(items, submissions, currentFence), + owesWork: + status !== 'idle' && + owesStructuredAgentSessionWork(items, submissions, currentFence, itemFence), pendingPromptIds: status === 'attention' - ? items.filter(isPendingStructuredAgentSessionPrompt).map((item) => item.itemId) + ? items + .filter(pendingStructuredAgentSessionPrompt(currentFence, itemFence)) + .map((item) => item.itemId) : [], summary: { status, From 19612d57419727922f5fcfdcdd12aad940bb6152 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:49:12 -0700 Subject: [PATCH 02/22] feat(native-chat): one host view of an agent's current work, with execution provenance The host decides which turn, prompts and sends belong to the agent generation that can still act, from the lease, its own observed exits and each item's owner generation. Every host reader and the desktop and mobile chats use that view, so a dead generation's leftover rows never hold a send, show Working, or take a prompt answer. A generation ending publishes the view and wakes queued cards without a journal write, and only sends this process accepted are handed over. --- .../scripts/vitest-sqlite-runtime-files.mjs | 2 + ...obile-structured-actionable-prompt.test.ts | 66 ++++ .../mobile-structured-actionable-prompt.ts | 33 ++ .../mobile-structured-agent-session-cancel.ts | 14 +- .../use-mobile-structured-agent-session.ts | 25 +- .../use-mobile-structured-prompt-responses.ts | 10 +- ...use-mobile-structured-send-with-outcome.ts | 11 +- .../acp/acp-structured-host-crash.test.ts | 12 + ...acp-structured-host-restore-failed.test.ts | 9 +- .../first-work-branch-rename.test.ts | 12 +- .../claude-background-stop-records.test.ts | 9 +- .../claude-background-wake-retention.test.ts | 5 +- ...claude-subagent-permission-request.test.ts | 6 +- ...tructured-journal-exit-convergence.test.ts | 21 +- .../journal-context-rewind.ts | 1 + .../journal-damage.test.ts | 4 +- .../journal-epoch-replacement.test.ts | 4 +- .../journal-epoch-replacement.ts | 4 + .../journal-item-fold.ts | 10 +- .../journal-item-provenance.test.ts | 186 +++++++++++ .../journal-legacy-import.test.ts | 4 + .../journal-legacy-import.ts | 4 +- .../journal-lifecycle-batch-appender.ts | 18 +- .../journal-lifecycle-batch-partition.test.ts | 1 + .../journal-lifecycle-batch-partition.ts | 2 + .../journal-lifecycle-mutation.ts | 7 +- .../journal-pending-submission-recovery.ts | 23 +- .../agent-session-journal/journal-reducer.ts | 10 +- .../journal-row-builders.ts | 32 +- .../journal-row-schema.ts | 33 +- .../journal-row-unusable-annotations.ts | 10 + .../journal-store-contracts.ts | 11 +- .../agent-session-journal/journal-store.ts | 22 +- .../journal-submission-fold.ts | 3 +- .../agent-session-history-page.ts | 80 +++-- .../agent-session-subscriber-catch-up.ts | 41 ++- ...structured-agent-session-attach-context.ts | 1 + ...structured-agent-session-attach-failure.ts | 2 + .../structured-agent-session-attach-flow.ts | 8 +- ...d-agent-session-background-task-channel.ts | 13 +- .../structured-agent-session-child-exit.ts | 21 +- ...red-agent-session-claude-root-exit.test.ts | 2 +- ...tructured-agent-session-client-delivery.ts | 62 +++- .../structured-agent-session-command-turn.ts | 14 +- ...red-agent-session-conversation-lifetime.ts | 1 + ...uctured-agent-session-current-work.test.ts | 299 ++++++++++++++++++ .../structured-agent-session-current-work.ts | 206 ++++++++++++ .../structured-agent-session-delivery-loop.ts | 30 +- ...structured-agent-session-event-recovery.ts | 5 +- ...tructured-agent-session-event-sink.test.ts | 9 +- .../structured-agent-session-event-sink.ts | 4 +- ...structured-agent-session-history-result.ts | 13 +- .../structured-agent-session-host-delivery.ts | 10 +- .../structured-agent-session-host-lifetime.ts | 22 +- ...tructured-agent-session-host-stop-event.ts | 31 +- .../structured-agent-session-host.ts | 14 +- ...tructured-agent-session-idle-sweep.test.ts | 1 + .../structured-agent-session-idle-sweep.ts | 55 ++-- ...ed-agent-session-journal-append-options.ts | 4 + ...-session-leftover-settlement-scope.test.ts | 16 +- ...ssion-leftover-settlement-triggers.test.ts | 42 ++- ...ession-leftover-settlement.test-fixture.ts | 16 + ...ctured-agent-session-mutation-admission.ts | 3 + ...ructured-agent-session-mutation-context.ts | 1 + ...red-agent-session-mutation-turn-context.ts | 13 +- .../structured-agent-session-prompt-cancel.ts | 7 +- ...ed-agent-session-queued-card-holds.test.ts | 36 ++- ...ructured-agent-session-queued-gate.test.ts | 10 +- ...tructured-agent-session-queued-messages.ts | 110 +++---- ...ructured-agent-session-queued-mutations.ts | 6 +- ...ctured-agent-session-queued-publication.ts | 31 +- .../structured-agent-session-queued-stop.ts | 11 +- .../structured-agent-session-queued-wiring.ts | 10 +- ...gent-session-restart-context-clear.test.ts | 7 +- ...ent-session-restart-resume-test-harness.ts | 7 +- ...tured-agent-session-restart-resume.test.ts | 12 +- ...ed-agent-session-rewind-provenance.test.ts | 154 +++++++++ .../structured-agent-session-rewind.ts | 21 +- ...tructured-agent-session-send-settlement.ts | 8 +- .../structured-agent-session-shown-work.ts | 9 +- ...red-agent-session-signed-out-child.test.ts | 7 +- ...ructured-agent-session-signed-out-child.ts | 7 +- ...uctured-agent-session-start-failure-row.ts | 13 +- ...agent-session-startup-failure-exit.test.ts | 2 +- .../structured-agent-session-status-feed.ts | 16 +- ...agent-session-status-journal-projection.ts | 24 +- ...d-agent-session-stop-event-binding.test.ts | 3 +- ...structured-agent-session-stop-wind-down.ts | 12 +- .../structured-agent-session-stopping.ts | 17 +- .../structured-agent-session-subscribers.ts | 33 +- .../structured-agent-session-thread-goal.ts | 3 +- .../structured-agent-session-transition.ts | 1 + ...tructured-agent-session-turn-stop-notes.ts | 7 +- .../structured-agent-session-turns-cancel.ts | 34 +- .../structured-agent-session-turns-prompt.ts | 16 +- .../structured-agent-session-turns.ts | 12 +- ...ured-agent-session-unexpected-exit.test.ts | 16 +- ...red-agent-session-unsent-send-hold.test.ts | 16 +- ...tured-agent-session-working-at-teardown.ts | 42 ++- ...red-conversation-command-admission.test.ts | 13 +- ...ructured-conversation-command-admission.ts | 23 +- .../structured-conversation-command.ts | 1 + ...structured-conversation-compaction.test.ts | 9 +- .../structured-rewind-retained-host-rows.ts | 24 +- .../structured-mailbox-pointer-host.test.ts | 5 +- .../structured-mailbox-pointer-host.ts | 7 +- .../structured-session-pointer-delivery.ts | 13 +- ...structured-worker-group-addressing.test.ts | 3 +- ...ructured-agent-session-rpc.test-fixture.ts | 5 +- .../structured-worker-cleared-session.test.ts | 2 + ...d-agent-session-actionable-prompts.test.ts | 57 ++++ ...ctured-agent-session-message-projection.ts | 9 +- .../structured-attention-read-observation.ts | 6 +- ...tructured-agent-session-transport-state.ts | 3 + .../use-structured-agent-session.ts | 5 +- src/shared/agent-session-refusal-details.ts | 2 + .../agent-session-refusal-notice.test.ts | 6 + src/shared/agent-session-refusal-notice.ts | 4 + .../agent-session-refusal-reason-words.ts | 2 + src/shared/agent-session-rewind.ts | 3 + src/shared/agent-session-wire.ts | 7 + .../structured-agent-session-coalescer.ts | 12 +- .../structured-agent-session-live-turn.ts | 61 ++-- ...ctured-agent-session-main-agent-working.ts | 37 +-- .../structured-agent-session-owed-work.ts | 12 +- .../structured-agent-session-projection.ts | 35 +- .../structured-agent-session-reducer.test.ts | 74 ++++- .../structured-agent-session-reducer.ts | 17 +- 128 files changed, 2272 insertions(+), 568 deletions(-) create mode 100644 mobile/src/session/mobile-structured-actionable-prompt.test.ts create mode 100644 mobile/src/session/mobile-structured-actionable-prompt.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts create mode 100644 src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 2a237b0f13ea..0d359f63de59 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -97,6 +97,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-owner-verdict.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-failed-create-sink-release.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-grouped-prompt.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-held-child-lease-renewal.test.ts', @@ -141,6 +142,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-status-publication.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-at-rest.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-live-send.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-rewind.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-clear-rewind.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-send-idempotency.test.ts', diff --git a/mobile/src/session/mobile-structured-actionable-prompt.test.ts b/mobile/src/session/mobile-structured-actionable-prompt.test.ts new file mode 100644 index 000000000000..159418b1889e --- /dev/null +++ b/mobile/src/session/mobile-structured-actionable-prompt.test.ts @@ -0,0 +1,66 @@ +// The phone answers only the prompts the host says still wait on the person: one an agent that +// has ended raised gets no card, and an older host, which names none, keeps today's rule. + +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from '../../../src/shared/agent-session-journal-types' +import { + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' + +const PENDING = { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null +} + +function approval(itemId: string, sequence: number): AgentJournalRenderItem { + return { + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: PENDING + } + } +} + +function question(itemId: string, sequence: number): AgentJournalRenderItem { + return { + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'question', + title: 'Which one?', + options: [{ id: 'a', label: 'A' }], + resolution: PENDING + } + } +} + +const ITEMS = [approval('dead-approval', 1), question('dead-question', 2)] + +describe('the prompt a phone answers', () => { + it('skips one the host does not name, and answers one it does', () => { + const items = [...ITEMS, approval('live-approval', 3)] + expect( + actionableStructuredApproval({ items, actionablePromptIds: ['live-approval'] })?.itemId + ).toBe('live-approval') + expect( + actionableStructuredQuestion({ items, actionablePromptIds: ['live-approval'] }) + ).toBeNull() + }) + + it('is the first pending one when an older host names none', () => { + expect(actionableStructuredApproval({ items: ITEMS })?.itemId).toBe('dead-approval') + expect(actionableStructuredQuestion({ items: ITEMS })?.itemId).toBe('dead-question') + }) +}) diff --git a/mobile/src/session/mobile-structured-actionable-prompt.ts b/mobile/src/session/mobile-structured-actionable-prompt.ts new file mode 100644 index 000000000000..478927a9088e --- /dev/null +++ b/mobile/src/session/mobile-structured-actionable-prompt.ts @@ -0,0 +1,33 @@ +import { isActionableStructuredAgentSessionPrompt } from '../../../src/shared/structured-agent-session-live-turn' +import type { StructuredAgentSessionState } from '../../../src/shared/structured-agent-session-reducer' +import { + pendingStructuredApproval, + pendingStructuredQuestion, + type StructuredApprovalItem, + type StructuredQuestionItem +} from './mobile-structured-agent-prompts' + +type PromptState = Pick + +/** The approval a person can answer: pending, and named by the host as waiting when it names them + * (`isActionableStructuredAgentSessionPrompt`): one an agent that ended raised is not. */ +export function actionableStructuredApproval(state: PromptState): StructuredApprovalItem | null { + return ( + state.items.find( + (item): item is StructuredApprovalItem => + pendingStructuredApproval(item) && + isActionableStructuredAgentSessionPrompt(item.itemId, state.actionablePromptIds) + ) ?? null + ) +} + +/** The question a person can answer, by the same rule. */ +export function actionableStructuredQuestion(state: PromptState): StructuredQuestionItem | null { + return ( + state.items.find( + (item): item is StructuredQuestionItem => + pendingStructuredQuestion(item) && + isActionableStructuredAgentSessionPrompt(item.itemId, state.actionablePromptIds) + ) ?? null + ) +} diff --git a/mobile/src/session/mobile-structured-agent-session-cancel.ts b/mobile/src/session/mobile-structured-agent-session-cancel.ts index 49232afeb326..457a960c26cc 100644 --- a/mobile/src/session/mobile-structured-agent-session-cancel.ts +++ b/mobile/src/session/mobile-structured-agent-session-cancel.ts @@ -1,7 +1,9 @@ import type { AgentSessionCancelResult } from '../../../src/shared/agent-session-wire' -import type { AgentJournalRenderItem } from '../../../src/shared/agent-session-journal-types' import type { StructuredAgentSessionState } from '../../../src/shared/structured-agent-session-reducer' -import { runningStructuredAgentSessionTurnId } from '../../../src/shared/structured-agent-session-live-turn' +import { + isActionableStructuredAgentSessionPrompt, + runningStructuredAgentSessionTurnId +} from '../../../src/shared/structured-agent-session-live-turn' import type { RpcClient } from '../transport/rpc-client' import { requestStructuredAgentSessionMutation, @@ -10,12 +12,14 @@ import { type PromptIdentity = { itemId: string; expectedRevision: number } +/** The prompt a Stop cancels: one the host says still waits on the person, when it says. */ export function pendingStructuredPromptIdentity( - items: readonly AgentJournalRenderItem[] + state: Pick ): PromptIdentity | undefined { - const prompt = items.find((item) => + const prompt = state.items.find((item) => item.body.kind === 'approval' || item.body.kind === 'question' - ? item.body.resolution.state === 'pending' + ? item.body.resolution.state === 'pending' && + isActionableStructuredAgentSessionPrompt(item.itemId, state.actionablePromptIds) : false ) return prompt ? { itemId: prompt.itemId, expectedRevision: prompt.revision } : undefined diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index fb3c69c51aa6..b9871df085a5 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -11,8 +11,10 @@ import { } from '../../../src/shared/structured-agent-session-live-turn' import { selectStructuredAgentTurnActivity } from '../../../src/shared/native-chat-turn-activity' import { - pendingStructuredApproval, - pendingStructuredQuestion, + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' +import { projectStructuredPermission, projectStructuredQuestion } from './mobile-structured-agent-prompts' @@ -196,13 +198,22 @@ export function useMobileStructuredAgentSession(args: { [thinking, activityText, stopping, stopRequestInFlight, queueCapable, state.items] ) const status = state.status === 'idle' ? 'idle' : state.status + // The host's answer where it gives one: a prompt an agent that ended raised takes no answer. const approvalPrompt = useMemo( - () => state.items.find(pendingStructuredApproval) ?? null, - [state.items] + () => + actionableStructuredApproval({ + items: state.items, + actionablePromptIds: state.actionablePromptIds + }), + [state.items, state.actionablePromptIds] ) const questionPrompt = useMemo( - () => state.items.find(pendingStructuredQuestion) ?? null, - [state.items] + () => + actionableStructuredQuestion({ + items: state.items, + actionablePromptIds: state.actionablePromptIds + }), + [state.items, state.actionablePromptIds] ) // What a refused command's line on the phone stands on. const commandRefusalCauses = useMemo( @@ -279,7 +290,7 @@ export function useMobileStructuredAgentSession(args: { void stopPress.track(() => requestCancel()) }, cancelPrompt: (prompt?: { itemId: string; expectedRevision: number }) => - requestCancel(prompt ?? pendingStructuredPromptIdentity(stateRef.current.items)), + requestCancel(prompt ?? pendingStructuredPromptIdentity(stateRef.current)), permission: projectStructuredPermission(approvalPrompt), question: projectStructuredQuestion(questionPrompt, groupedDraft), respondPermission, diff --git a/mobile/src/session/use-mobile-structured-prompt-responses.ts b/mobile/src/session/use-mobile-structured-prompt-responses.ts index 093029b2be52..eb2a0b38caa1 100644 --- a/mobile/src/session/use-mobile-structured-prompt-responses.ts +++ b/mobile/src/session/use-mobile-structured-prompt-responses.ts @@ -8,8 +8,10 @@ import { } from '../../../src/shared/agent-session-question-answer' import type { StructuredAgentSessionState } from '../../../src/shared/structured-agent-session-reducer' import { - pendingStructuredApproval, - pendingStructuredQuestion, + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' +import { structuredApprovalResponseTarget, structuredQuestionResponseTarget } from './mobile-structured-agent-prompts' @@ -54,7 +56,7 @@ export function useMobileStructuredPromptResponses(args: { async (optionId: string): Promise => { const target = structuredApprovalResponseTarget( optionId, - stateRef.current.items.find(pendingStructuredApproval) ?? null + actionableStructuredApproval(stateRef.current) ) if (!target) { return false @@ -107,7 +109,7 @@ export function useMobileStructuredPromptResponses(args: { const respondQuestion = useCallback( async (answer: string): Promise => { - const prompt = stateRef.current.items.find(pendingStructuredQuestion) ?? null + const prompt = actionableStructuredQuestion(stateRef.current) if (prompt?.body.questions) { const promptKey = groupedQuestionPromptKey(prompt.itemId, prompt.revision) const grouped = advanceGroupedQuestion({ diff --git a/mobile/src/session/use-mobile-structured-send-with-outcome.ts b/mobile/src/session/use-mobile-structured-send-with-outcome.ts index c6c8e9604e07..13caf44c1667 100644 --- a/mobile/src/session/use-mobile-structured-send-with-outcome.ts +++ b/mobile/src/session/use-mobile-structured-send-with-outcome.ts @@ -19,9 +19,9 @@ import type { MobileNativeChatSendErrorReporter } from './use-mobile-native-chat import { sendMobileStructuredAgentSessionMessage } from './mobile-structured-agent-session-send' import { timeoutForDeadline } from './mobile-structured-agent-session-rpc' import { - pendingStructuredApproval, - pendingStructuredQuestion -} from './mobile-structured-agent-prompts' + actionableStructuredApproval, + actionableStructuredQuestion +} from './mobile-structured-actionable-prompt' /** Whether a send made now asks the host to queue it. The host's queue waits on any pending prompt; * one this build cannot answer would hold the send forever, so it starts a turn instead. */ @@ -103,9 +103,8 @@ export function useMobileStructuredSendWithOutcome(args: { ...controller }, busy: () => - stateRef.current.items.some( - (item) => pendingStructuredApproval(item) || pendingStructuredQuestion(item) - ) + actionableStructuredApproval(stateRef.current) || + actionableStructuredQuestion(stateRef.current) ? 'prompt' : runningStructuredAgentSessionTurnId(stateRef.current) ? 'working' diff --git a/src/main/acp/acp-structured-host-crash.test.ts b/src/main/acp/acp-structured-host-crash.test.ts index 53465c822765..742e0de2b6eb 100644 --- a/src/main/acp/acp-structured-host-crash.test.ts +++ b/src/main/acp/acp-structured-host-crash.test.ts @@ -38,6 +38,18 @@ async function midReply() { batches.push(batch) return append(batch) }) + // A settlement plans its batch at its own turn in the write queue: recorded as it is planned. + const appendPlanned = journal.appendPlannedLifecycleBatch + vi.spyOn(journal, 'appendPlannedLifecycleBatch').mockImplementation((input) => + appendPlanned({ + ...input, + plan: () => { + const planned = input.plan() + batches.push({ ...input, mutations: planned.mutations }) + return planned + } + }) + ) child.stderr = LAST_WORDS return { ...rig, child, batches } } diff --git a/src/main/acp/acp-structured-host-restore-failed.test.ts b/src/main/acp/acp-structured-host-restore-failed.test.ts index bf6c7a363c8d..27d8b4eaa505 100644 --- a/src/main/acp/acp-structured-host-restore-failed.test.ts +++ b/src/main/acp/acp-structured-host-restore-failed.test.ts @@ -224,10 +224,11 @@ describe('the warning row of a replacement whose attach failed', () => { const { host, store, fence, warnings } = opened expect(await host.attach(CALLER, attachParams())).toMatchObject({ ok: true }) await host.close(SESSION, 'user-close') - // The journal's open fails after the fresh session's link is durable, before its row is written. - vi.spyOn(AgentSessionJournal.prototype, 'open').mockRejectedValueOnce( - new Error('journal path unavailable') - ) + // The journal's open fails after the fresh session's link is durable, before its row is written: + // the start's leftover settlement meets it first and goes on, then the attach's own open. + vi.spyOn(AgentSessionJournal.prototype, 'open') + .mockRejectedValueOnce(new Error('journal path unavailable')) + .mockRejectedValueOnce(new Error('journal path unavailable')) await expect(host.attach(CALLER, attachParams(fence()))).rejects.toThrow( 'journal path unavailable' ) diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index 0adc02d7eaa5..9f3674afc66f 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -104,12 +104,14 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { // A real journal's sequence only ever advances, so the feed's projection // cache must miss on every publish here: this test is about the rename. let sequence = 0 - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot() and the provenance reads of a journal. const journal = { snapshot: () => ({ items, submissions: [] }), stopMarks: { latest: () => null, revision: () => 0 }, lastActivityAt: () => 1, - cursor: () => ({ epoch: 1, sequence: (sequence += 1) }) + cursor: () => ({ epoch: 1, sequence: (sequence += 1) }), + itemFence: () => undefined, + wroteBeforeOpen: () => false } as unknown as AgentSessionJournal const pending: Promise[] = [] const observe = vi.fn((summary, options) => { @@ -193,7 +195,7 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { const { deps, setDisplayName, setRenameError } = makeDeps({ getRepo: () => ({ id: REPO_ID, kind: 'folder', path: '/workspace/platform' }) as Repo }) - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot() and the provenance reads of a journal. const journal = { lastActivityAt: () => 1, stopMarks: { latest: () => null, revision: () => 0 }, @@ -210,7 +212,9 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { } ], submissions: [] - }) + }), + itemFence: () => undefined, + wroteBeforeOpen: () => false } as unknown as AgentSessionJournal const location = { executionHostId: 'local' as const, diff --git a/src/main/claude/claude-background-stop-records.test.ts b/src/main/claude/claude-background-stop-records.test.ts index 0092d9e2f29d..59da29795077 100644 --- a/src/main/claude/claude-background-stop-records.test.ts +++ b/src/main/claude/claude-background-stop-records.test.ts @@ -33,7 +33,14 @@ async function droppedFromRoster() { { sessionId: 'session-1', fence: 7, - journal: { snapshot: () => ({ items: [] }), submissions: () => [] }, + journal: { + snapshot: () => ({ items: [] }), + submissions: () => [], + runningTurn: () => null, + itemFence: () => undefined, + visitItems: () => undefined, + wroteBeforeOpen: () => false + }, adapter: run.adapter }, RECORD, diff --git a/src/main/claude/claude-background-wake-retention.test.ts b/src/main/claude/claude-background-wake-retention.test.ts index 427e24e89609..7e3679d2e328 100644 --- a/src/main/claude/claude-background-wake-retention.test.ts +++ b/src/main/claude/claude-background-wake-retention.test.ts @@ -40,6 +40,9 @@ async function wiredSession() { }, // No Stop was ever pressed here. stopMarks: { latest: () => null, revision: () => 0 }, + // One generation produced everything here, and every send was accepted by this process. + itemFence: () => undefined, + wroteBeforeOpen: () => false, snapshot: () => ({ items: [...run.journalItems.values()] .sort((a, b) => a.sequence - b.sequence) @@ -53,7 +56,7 @@ async function wiredSession() { [ parent.sessionId, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies the feed's cursor, clock, Stop marks, snapshot, submissions and item reads. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the fixture supplies the feed's cursor, clock, Stop marks, snapshot, submissions, item and provenance reads. journal: journal as unknown as Journal, params: { location: { diff --git a/src/main/claude/claude-subagent-permission-request.test.ts b/src/main/claude/claude-subagent-permission-request.test.ts index c3db6a4011b6..95bfb84cb2b8 100644 --- a/src/main/claude/claude-subagent-permission-request.test.ts +++ b/src/main/claude/claude-subagent-permission-request.test.ts @@ -24,6 +24,7 @@ import { AgentHookServer } from '../agent-hooks/server' import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../native-chat/agent-session-journal/journal-host-database-test-support' import { settleStructuredAgentSessionDeadGeneration } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import { StructuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { createStructuredAgentSessionLogger } from '../native-chat/agent-session-wire/structured-agent-session-logger' @@ -243,7 +244,10 @@ async function pipeline() { await journal.appendItem( identity, { ...card.body, resolution: { ...resolution, resolvedBy: 'client-1', resolvedAt: now() } }, - { fence: FENCE, turnScope: journal.liveTurnScope() } + { + fence: FENCE, + turnScope: new StructuredAgentSessionCurrentWork(journal, FENCE).turnScope() + } ) } const cardId = (): string => { diff --git a/src/main/codex/codex-structured-journal-exit-convergence.test.ts b/src/main/codex/codex-structured-journal-exit-convergence.test.ts index 60c0eacd75ad..ab8e5a75ab36 100644 --- a/src/main/codex/codex-structured-journal-exit-convergence.test.ts +++ b/src/main/codex/codex-structured-journal-exit-convergence.test.ts @@ -5,7 +5,6 @@ import { afterEach, expect, it, vi } from 'vitest' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../shared/agent-session-journal-types' import { codexProviderHandle } from '../../shared/agent-session-provider-handle-encoding' import { readAgentJournalTurn } from '../../shared/agent-session-turn-record' -import { isStructuredAgentSessionMainAgentWorking } from '../../shared/structured-agent-session-main-agent-working' import type { AgentSessionRecord } from '../../shared/agent-session-record' import { agentSessionLeaseFixture, @@ -23,7 +22,7 @@ import { type StructuredAgentSessionChildExitSession } from '../native-chat/agent-session-wire/structured-agent-session-child-exit' import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' -import { pendingPromptExists } from '../native-chat/agent-session-wire/structured-agent-session-queued-messages' +import { StructuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' import { NO_STORE_RECEIPTS } from '../native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture' import { createCodexJournalTranslator } from './codex-structured-journal-translation' import { handleCodexSessionExit } from './codex-structured-session-close' @@ -47,16 +46,14 @@ afterEach(async () => { } }) -/** What the chat shows at the lease's `fence`: 7 while the child runs, 8 once its exit released it. */ -function userVisible(journal: AgentSessionJournal, fence: number) { +/** What the chat shows with the live generation at `liveFence`: 7 while the child runs, none once + * its exit released it (`structuredAgentSessionCurrentWork`). */ +function userVisible(journal: AgentSessionJournal, liveFence: number | null) { const items = journal.snapshot().items + const work = new StructuredAgentSessionCurrentWork(journal, liveFence) return { - working: isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(fence), - journal.submissions(), - fence - ), - prompt: pendingPromptExists(journal, fence), + working: work.working(), + prompt: work.hasActionablePrompt(), turnStates: items.flatMap((item) => readAgentJournalTurn(item.body)?.state ?? []), runningCalls: items.filter( (item) => item.body.kind === 'tool-call' && item.body.state === 'running' @@ -237,13 +234,13 @@ it('settles a refused Codex exit through the host in one commit, then permits a }, sessions: new Map([[SESSION, hostSession]]), flushLifecycle: () => deferred.lifecycleBarrier(), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId, task) => task(), now: () => 5_000 }, event ) - expect(userVisible(journal, 8)).toEqual({ + expect(userVisible(journal, null)).toEqual({ working: false, prompt: false, turnStates: ['interrupted'], diff --git a/src/main/native-chat/agent-session-journal/journal-context-rewind.ts b/src/main/native-chat/agent-session-journal/journal-context-rewind.ts index dbaf544e3ef7..bcfa773cbcf7 100644 --- a/src/main/native-chat/agent-session-journal/journal-context-rewind.ts +++ b/src/main/native-chat/agent-session-journal/journal-context-rewind.ts @@ -77,6 +77,7 @@ export async function rewindJournalContext(input: { seq, ts: item.observedAt ?? ts, fence: input.fence, + ownerFence: item.ownerFence, turnScope: item.turnScope ?? { kind: 'thread' } }), revision diff --git a/src/main/native-chat/agent-session-journal/journal-damage.test.ts b/src/main/native-chat/agent-session-journal/journal-damage.test.ts index 8b59940a6a19..168e74484b69 100644 --- a/src/main/native-chat/agent-session-journal/journal-damage.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-damage.test.ts @@ -206,8 +206,8 @@ describe('a row the reader would reject', () => { await expect( journal.replaceEpochItems('legacy_import', 1, [ - { identity: item(5), body: body('replacement') }, - { identity: item(6), body: blankCallId } + { identity: item(5), body: body('replacement'), ownerFence: 1 }, + { identity: item(6), body: blankCallId, ownerFence: 1 } ]) ).rejects.toMatchObject({ code: 'journal_row_rejected' }) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts index 36247e2571ff..c64027b94555 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.test.ts @@ -96,7 +96,7 @@ describe('journal epoch replacement', () => { const published: JournalLoad[] = [] replace({ - items: [{ identity: item(1), body: { kind: 'status', text: 'republished' } }], + items: [{ identity: item(1), body: { kind: 'status', text: 'republished' }, ownerFence: 1 }], onPublished: (loaded) => published.push(loaded) }) @@ -112,7 +112,7 @@ describe('journal epoch replacement', () => { 'a replace', (journal: AgentSessionJournal) => journal.replaceEpochItems('legacy_import', 1, [ - { identity: item(9), body: { kind: 'status', text: 'republished' } } + { identity: item(9), body: { kind: 'status', text: 'republished' }, ownerFence: 1 } ]) ], ['a rollover', (journal: AgentSessionJournal) => journal.rollEpoch('handle_forked', 1)] diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 67927b401588..17b9f5e3b956 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -38,6 +38,9 @@ export type JournalReplacementItem = AgentJournalProducerLinkage & { /** Absent for history rebuilt from a source that never stated one: derived from position, as * a legacy row's is, and then written down. */ turnScope?: AgentJournalTurnScope + /** Carried from the epoch it came from (`JournalItemRow.ownerFence`), never the replacing + * writer's fence: replacing history does not make it the current generation's work. */ + ownerFence: number } export function replaceJournalEpoch(input: { @@ -71,6 +74,7 @@ export function replaceJournalEpoch(input: { body: item.body, seq: state.lastSequence + 1, fence: input.fence, + ownerFence: item.ownerFence, ts: item.observedAt ?? input.now(), linkage: item, turnScope: item.turnScope ?? state.derivedTurnScope.scopeFor(item.body) diff --git a/src/main/native-chat/agent-session-journal/journal-item-fold.ts b/src/main/native-chat/agent-session-journal/journal-item-fold.ts index 10aee4a99c41..6085e6ab88d7 100644 --- a/src/main/native-chat/agent-session-journal/journal-item-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-item-fold.ts @@ -27,8 +27,8 @@ export function upsertJournalItem( itemId: string, revision: number, next: AgentJournalRenderItem, - /** The creating write's fence: the generation a running turn belongs to. */ - fence: number + /** The write's fence, and the provenance it states (`JournalItemRow.ownerFence`). */ + fences: { fence: number; ownerFence?: number } ): void { const tombstoned = state.tombstones.get(itemId) if (tombstoned !== undefined && revision <= tombstoned) { @@ -41,10 +41,14 @@ export function upsertJournalItem( state.derivedTurnScope.observe(itemId, isRootAgentJournalItem(next), existing?.body, next.body) if (!existing) { state.items.set(itemId, next) - state.itemFences.set(itemId, fence) + state.itemFences.set(itemId, fences.ownerFence ?? fences.fence) state.tombstones.delete(itemId) return } + // A revision keeps the provenance it found unless it states one: an owner's observation does. + if (fences.ownerFence !== undefined) { + state.itemFences.set(itemId, fences.ownerFence) + } // Creation sequence is the ordering key; a revision refreshes content only. // `observedAt` is pinned with it: clients sort the timeline by that timestamp, // so letting a revision advance it makes the row jump past everything that diff --git a/src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts b/src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts new file mode 100644 index 000000000000..e275e8102eda --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-item-provenance.test.ts @@ -0,0 +1,186 @@ +// Execution provenance (`JournalItemRow.ownerFence`): the generation whose execution produced an +// item, kept apart from each row's writer fence. Every writer states it; the fold prefers it. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { retainedRowReplacement } from '../agent-session-wire/structured-rewind-retained-host-rows' +import type { JournalHostDatabase } from './journal-host-database' +import { + createTrackedJournalOpener, + insertTestJournalRowJson, + liveTestJournalRows, + openTestJournalHostDatabase +} from './journal-host-database-test-support' +import type { AgentSessionJournal } from './journal-store' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: codexProviderHandle('thread-1') +} +const SCOPE = AGENT_JOURNAL_THREAD_SCOPE +const TURN: AgentJournalItemIdentity = { + provider: 'codex', + threadId: 'thread-1', + turnId: 'turn-1', + ordinal: 1 +} +const TURN_ID = agentJournalItemKey(TURN) +const RUNNING: AgentJournalItemBody = { kind: 'turn', turnId: 'turn-1', state: 'running' } +const SETTLED: AgentJournalItemBody = { kind: 'turn', turnId: 'turn-1', state: 'interrupted' } + +let root: string +let database: JournalHostDatabase +const journals = createTrackedJournalOpener() + +function item(ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal } +} + +type StoredItemWrite = { kind?: string; ownerFence?: number; mutations?: StoredItemWrite[] } + +/** Each stored item write's stated provenance, in write order, a lifecycle batch's included. */ +function storedOwnerFences(): (number | undefined)[] { + return liveTestJournalRows(database.db, IDENTITY.sessionId).flatMap((row) => { + const parsed: StoredItemWrite = JSON.parse(row.rowJson) + const writes = parsed.kind === 'lifecycle-batch' ? (parsed.mutations ?? []) : [parsed] + return writes.flatMap((write) => (write.kind === 'item' ? [write.ownerFence] : [])) + }) +} + +async function open(): Promise { + return journals.open({ identity: IDENTITY, stateDirectory: root }) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-journal-provenance-')) + database = openTestJournalHostDatabase(root) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +describe('execution provenance', () => { + it("records a provider observation's owner, and a host revision keeps it", async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 4, ownerFence: 3, turnScope: SCOPE }) + // Host bookkeeping under a later writer fence states no owner of its own. + await journal.appendItem(TURN, SETTLED, { fence: 5, turnScope: SCOPE }) + + expect(journal.itemFence(TURN_ID)).toBe(3) + // Every row states it: none leaves the reader to guess from the writer fence. + expect(storedOwnerFences()).toEqual([3, 3]) + }) + + it('a first write with no stated owner is its writer generation’s', async () => { + const journal = await open() + await journal.appendItem( + item(2), + { kind: 'status', text: 'note' }, + { fence: 6, turnScope: SCOPE } + ) + expect(journal.itemFence(agentJournalItemKey(item(2)))).toBe(6) + expect(storedOwnerFences()).toEqual([6]) + }) + + it('a lifecycle batch states it on each item: the batch owner, else the kept value', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 2, ownerFence: 2, turnScope: SCOPE }) + // A provider batch at the owner's fence creates one item. + await journal.appendLifecycleBatch({ + settlementId: 'provider', + fence: 3, + ownerFence: 3, + mutations: [{ kind: 'item', identity: item(2), body: RUNNING, turnScope: SCOPE }] + }) + // A settlement at a later fence revises the first: it keeps that item's owner. + await journal.appendLifecycleBatch({ + settlementId: 'settlement', + fence: 4, + recovered: true, + mutations: [{ kind: 'item', identity: TURN, body: SETTLED, turnScope: SCOPE }] + }) + + expect(journal.itemFence(TURN_ID)).toBe(2) + expect(journal.itemFence(agentJournalItemKey(item(2)))).toBe(3) + expect(storedOwnerFences()).toEqual([2, 3, 2]) + }) + + it('survives a reopen from storage', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 4, ownerFence: 3, turnScope: SCOPE }) + await journals.closeAll() + + expect((await open()).itemFence(TURN_ID)).toBe(3) + }) + + it('epoch replacement carries each item’s provenance', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 2, ownerFence: 2, turnScope: SCOPE }) + const before = journal.epoch + + await journal.replaceEpochItems('legacy_import', 5, [ + { identity: TURN, body: RUNNING, ownerFence: journal.itemFence(TURN_ID) ?? 0 }, + { identity: item(2), body: { kind: 'status', text: 'new' }, ownerFence: 5 } + ]) + + expect(journal.epoch).not.toBe(before) + // Written at fence 5, the dead generation's turn is still fence 2's work. + expect(journal.itemFence(TURN_ID)).toBe(2) + expect(journal.itemFence(agentJournalItemKey(item(2)))).toBe(5) + }) + + it("a rewind's retained row carries the old fold's provenance; an older record's is never current", () => { + const row = { + itemId: TURN_ID, + body: RUNNING, + observedAt: 1, + ownerFence: 2 + } + expect(retainedRowReplacement(row).ownerFence).toBe(2) + const { ownerFence: _stated, ...older } = row + expect(retainedRowReplacement(older).ownerFence).toBe(0) + }) + + it('a stored provenance that is not a fence is dropped, and the row reads as its writer’s', async () => { + const journal = await open() + await journal.appendItem(TURN, RUNNING, { fence: 2, ownerFence: 2, turnScope: SCOPE }) + const rows = liveTestJournalRows(database.db, IDENTITY.sessionId) + const stored = rows.find((row) => row.rowJson.includes('"kind":"item"')) + const last = rows.at(-1)?.seq + if (!stored || last === undefined) { + throw new Error('expected the item row') + } + await journals.closeAll() + database = openTestJournalHostDatabase(root) + const parsed: Record = JSON.parse(stored.rowJson) + insertTestJournalRowJson( + database.db, + IDENTITY.sessionId, + last + 1, + JSON.stringify({ + ...parsed, + seq: last + 1, + revision: Number(parsed.revision) + 1, + fence: 7, + ownerFence: -1 + }) + ) + + expect((await open()).itemFence(TURN_ID)).toBe(2) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts b/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts index 89e88b247ae1..308778fdb56f 100644 --- a/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-legacy-import.test.ts @@ -245,6 +245,10 @@ describe('claude import', () => { uuid: 'e3a5b7c9-2d13-4f88-94ce-6b0d8f2a4e31' }) ]) + // Imported at fence 1, yet no generation's work: provenance 0, so nothing in it is current. + expect(journal.snapshot().items.map((entry) => journal.itemFence(entry.itemId))).toEqual([ + 0, 0, 0, 0 + ]) }) it('stays aligned when the decoder drops lines the tracker still walks', async () => { diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-import.ts b/src/main/native-chat/agent-session-journal/journal-legacy-import.ts index bdb95a6fc8e7..7afdb6ffef7f 100644 --- a/src/main/native-chat/agent-session-journal/journal-legacy-import.ts +++ b/src/main/native-chat/agent-session-journal/journal-legacy-import.ts @@ -139,7 +139,9 @@ export async function prepareLegacyTranscriptImport(input: { replacement.push({ identity, body: legacyItemBody(message, limits), - observedAt: message.timestamp ?? undefined + observedAt: message.timestamp ?? undefined, + // Imported history is no generation's work: never current, whatever it says still runs. + ownerFence: 0 }) } return { ok: true, items: replacement } diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index c49d7fe4f67e..c40e673e948b 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -42,7 +42,8 @@ export class JournalLifecycleBatchAppender { const rejections = journalQueuedRejectionRowBuilders( this.deps.state, input.fence, - input.rejectsQueued + input.rejectsQueued, + input.rejectsQueuedOnly ) if (rejections.length > 0 && input.mutations.length === 0) { throw new Error('journal_lifecycle_batch_mutation_bound_exceeded') @@ -96,12 +97,22 @@ export class JournalLifecycleBatchAppender { } private planMutations( - input: Pick, - mutations: readonly JournalLifecycleMutationInput[] + input: Pick, + stated: readonly JournalLifecycleMutationInput[] ): ((seq: number, ts: number) => JournalRow)[] { if (this.wasApplied(input.settlementId)) { return [] } + // A provider's batch states its generation on every item it writes. + const { ownerFence } = input + const mutations = + ownerFence === undefined + ? stated + : stated.map((mutation) => + mutation.kind === 'item' && mutation.ownerFence === undefined + ? { ...mutation, ownerFence } + : mutation + ) const current = this.deps.state() const options = { ...input, epoch: current.epoch } // Every row in this transaction must advance past the rows planned before it. @@ -162,6 +173,7 @@ export class JournalLifecycleBatchAppender { ...only.linkage, turnScope: only.turnScope, fence: input.fence, + ownerFence: only.ownerFence, recovered: input.recovered }, revisions diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts index 6571efe14b9c..a0654a7ae374 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.test.ts @@ -57,6 +57,7 @@ function previousProbe(settlementId: string, mutations: readonly JournalLifecycl ? { ...mutation, body: { ...mutation.body, outcome: 'cancellation' } } : mutation, journalLifecycleMutationItemId(mutation), + Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER ) ) diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts index 4e01628c6d58..525605b9ec3b 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-partition.ts @@ -136,6 +136,8 @@ function toLifecycleMutationRow(mutation: JournalLifecycleMutationInput): Journa return journalLifecycleMutationRow( sizedAsStopped(mutation), journalLifecycleMutationItemId(mutation), + Number.MAX_SAFE_INTEGER, + // The widest provenance any row can state, so the chunk built from it still fits. Number.MAX_SAFE_INTEGER ) } diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts index 1340b99f3106..9b939c267436 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-mutation.ts @@ -26,6 +26,8 @@ export type JournalLifecycleMutationInput = JournalItemAddress & linkage?: AgentJournalProducerLinkage /** Which turn the row belongs to. Kept from the write that creates the row. */ turnScope: AgentJournalTurnScope + /** The writer's provenance claim (`journalItemOwnerFence`); absent for host bookkeeping. */ + ownerFence?: number } | { kind: 'tombstone' } ) @@ -61,7 +63,9 @@ export function journalLifecycleItemMutation( export function journalLifecycleMutationRow( mutation: JournalLifecycleMutationInput, itemId: string, - revision: number + revision: number, + /** What an item mutation states (`journalItemOwnerFence`). */ + ownerFence: number ): JournalLifecycleMutation { return mutation.kind === 'item' ? { @@ -70,6 +74,7 @@ export function journalLifecycleMutationRow( revision, body: mutation.body, turnScope: mutation.turnScope, + ownerFence, ...agentJournalLinkageFields(mutation.linkage) } : { kind: 'tombstone', itemId, revision } diff --git a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts index 3aacf477d790..6ab4a635de47 100644 --- a/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts +++ b/src/main/native-chat/agent-session-journal/journal-pending-submission-recovery.ts @@ -109,15 +109,18 @@ export async function rejectJournalQueuedSubmissions( export function journalQueuedRejectionRowBuilders( state: () => JournalReducerState, fence: number, - rejection: AgentJournalDispatchRejection + rejection: AgentJournalDispatchRejection, + which: (submission: AgentJournalSubmission) => boolean = () => true ): ((seq: number, ts: number) => JournalRow)[] { - return [...state().submissions.values()].filter(isQueuedAgentJournalSubmission).map((entry) => - journalDispatchRowBuilder(state, { - clientMessageId: entry.clientMessageId, - state: 'rejected', - ...rejection, - fence, - recovered: true - }) - ) + return [...state().submissions.values()] + .filter((entry) => isQueuedAgentJournalSubmission(entry) && which(entry)) + .map((entry) => + journalDispatchRowBuilder(state, { + clientMessageId: entry.clientMessageId, + state: 'rejected', + ...rejection, + fence, + recovered: true + }) + ) } diff --git a/src/main/native-chat/agent-session-journal/journal-reducer.ts b/src/main/native-chat/agent-session-journal/journal-reducer.ts index 3890b9a7f28f..281fd8a92e39 100644 --- a/src/main/native-chat/agent-session-journal/journal-reducer.ts +++ b/src/main/native-chat/agent-session-journal/journal-reducer.ts @@ -50,7 +50,8 @@ export type JournalReducerState = { oldestSequence: number highestFence: number items: Map - /** Fence of the writer that created each item: the generation a running turn belongs to. */ + /** Each item's execution provenance (`JournalItemRow.ownerFence`): the generation whose work it + * is, which decides whether it is current. */ itemFences: Map /** Revision of a removed item, so a late lower revision cannot resurrect it. */ tombstones: Map @@ -107,7 +108,7 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo itemId, row.revision, journalRenderItem(itemId, row.revision, row.body, row, statedOrDerivedTurnScope(state, row)), - row.fence + { fence: row.fence, ownerFence: row.ownerFence } ) return } @@ -137,7 +138,10 @@ export function applyJournalRow(state: JournalReducerState, row: JournalRow): vo const producer = journalBatchMutationProducer(row, mutation) const scope = statedOrDerivedTurnScope(state, mutation) const item = journalRenderItem(itemId, revision, body, row, scope, producer, sequenceIndex) - upsertJournalItem(state, itemId, revision, item, row.fence) + upsertJournalItem(state, itemId, revision, item, { + fence: row.fence, + ownerFence: mutation.ownerFence + }) } else { removeJournalItem(state, resolveItemId(state, mutation.itemId), mutation.revision) } diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index a6e3aa03d189..59a21943484d 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -51,7 +51,12 @@ export function journalItemRowBuilder( state: () => JournalReducerState, address: AgentJournalItemIdentity | string, body: AgentJournalItemBody, - options: AgentJournalRowAttribution & { fence: number; observedAt?: number; recovered?: true }, + options: AgentJournalRowAttribution & { + fence: number + ownerFence?: number + observedAt?: number + recovered?: true + }, revisions?: Map ): RowBuilder { return (seq, ts) => @@ -61,6 +66,7 @@ export function journalItemRowBuilder( body, seq, fence: options.fence, + ownerFence: options.ownerFence, ts: options.observedAt ?? ts, recovered: options.recovered, linkage: options, @@ -186,7 +192,11 @@ export function journalLifecycleBatchRowBuilder( ? { ...mutation, body: turnEndAfterStop(current, resolved, mutation.body) } : mutation, itemId, - revision + revision, + journalItemOwnerFence(current, resolved, { + fence: options.fence, + ownerFence: mutation.kind === 'item' ? mutation.ownerFence : undefined + }) ) }) const row: JournalLifecycleBatchRow = { @@ -219,12 +229,29 @@ export function journalRowBase( return { v: journalRowSchemaVersion(bodies), epoch, seq, fence, ts } } +/** The provenance a row states: the writer's own claim when it has one — a provider's observation, + * or history carried into a new epoch — else host bookkeeping's, which keeps the item's, and for + * an item it creates is the fence it writes at. Every item row states it. */ +export function journalItemOwnerFence( + state: Pick, + resolvedItemId: string, + write: { fence: number; ownerFence?: number } +): number { + if (write.ownerFence !== undefined) { + return write.ownerFence + } + const kept = state.items.has(resolvedItemId) ? state.itemFences.get(resolvedItemId) : undefined + return kept ?? write.fence +} + export function buildJournalItemRow( input: JournalItemAddress & { state: JournalReducerState body: AgentJournalItemBody seq: number fence: number + /** The writer's provenance claim (`journalItemOwnerFence`); absent for host bookkeeping. */ + ownerFence?: number ts: number recovered?: true linkage?: AgentJournalProducerLinkage @@ -244,6 +271,7 @@ export function buildJournalItemRow( revision, body, ...journalRowBase(input.state.epoch, input.seq, input.fence, input.ts, [body]), + ownerFence: journalItemOwnerFence(input.state, resolved, input), ...(input.recovered ? { recovered: input.recovered } : {}), turnScope: input.turnScope, ...agentJournalLinkageFields(input.linkage) diff --git a/src/main/native-chat/agent-session-journal/journal-row-schema.ts b/src/main/native-chat/agent-session-journal/journal-row-schema.ts index f018f6307f94..494495248ca2 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-schema.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-schema.ts @@ -80,12 +80,18 @@ export const AGENT_JOURNAL_EPOCH_REASONS = [ ] as const export type AgentJournalEpochReason = (typeof AGENT_JOURNAL_EPOCH_REASONS)[number] -export type JournalItemRow = JournalRowBase & { - kind: 'item' - itemId: string - revision: number - body: AgentJournalItemBody -} +/** The generation whose execution produced an item: execution provenance, distinct from the + * row's `fence`, which is writer authority. Rides the row like linkage, with no `v` bump: an older + * host ignores it. Absent on rows from before it: the item's creating fence stands in. */ +type JournalItemProvenance = { ownerFence?: number } + +export type JournalItemRow = JournalRowBase & + JournalItemProvenance & { + kind: 'item' + itemId: string + revision: number + body: AgentJournalItemBody + } export type JournalTombstoneRow = JournalRowBase & { kind: 'tombstone' @@ -196,13 +202,14 @@ export type JournalDispatchRow = JournalRowBase & { * Inline like the row base, and for the same reason no `v` bump: an older host * ignores the unknown keys and reads the mutation as root, as it always did. */ export type JournalLifecycleMutation = - | (AgentJournalProducerLinkage & { - kind: 'item' - itemId: string - revision: number - body: AgentJournalItemBody - turnScope?: AgentJournalTurnScope - }) + | (AgentJournalProducerLinkage & + JournalItemProvenance & { + kind: 'item' + itemId: string + revision: number + body: AgentJournalItemBody + turnScope?: AgentJournalTurnScope + }) | { kind: 'tombstone'; itemId: string; revision: number } /** One durable append whose nested mutations share the outer ordering facts. */ diff --git a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts index 6c8811eb4a5b..e6d25b21df9e 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-unusable-annotations.ts @@ -6,11 +6,13 @@ import { isAdmissibleAgentSessionContextUsage } from '../../../shared/agent-sess export function dropUnusableRowAnnotations(record: Record): void { dropUnusableProducerLinkage(record) dropUnusableTurnScope(record) + dropUnusableOwnerFence(record) if (record.kind === 'lifecycle-batch' && Array.isArray(record.mutations)) { for (const mutation of record.mutations) { if (isPlainObject(mutation)) { dropUnusableProducerLinkage(mutation) dropUnusableTurnScope(mutation) + dropUnusableOwnerFence(mutation) } } } @@ -56,6 +58,14 @@ function dropUnusableTurnScope(record: Record): void { } } +/** Provenance this build cannot read, removed like linkage: the item's creating fence stands in. */ +function dropUnusableOwnerFence(record: Record): void { + const value = record.ownerFence + if (value !== undefined && !(Number.isSafeInteger(value) && Number(value) >= 0)) { + delete record.ownerFence + } +} + /** Context facts this build cannot read, removed from the turn row that carries * them. Same reasoning as linkage: they are an annotation on the turn, and * rejecting the row for them would truncate the journal from that row on. */ diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 9d4b4473509f..c8f4d117edcc 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -8,6 +8,7 @@ import type { AgentJournalProducerLinkage, AgentJournalResetReason, AgentJournalRowAttribution, + AgentJournalSubmission, AgentJournalTurnScope, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' @@ -57,6 +58,9 @@ export type JournalAppendResult = { export type JournalItemAppendOptions = AgentJournalRowAttribution & { fence: number + /** The generation whose execution produced the item: set by a provider's observation, and by + * nothing else. Host bookkeeping leaves it out and keeps the item's (`journalItemOwnerFence`). */ + ownerFence?: number observedAt?: number recovered?: true } @@ -74,6 +78,9 @@ export type JournalLifecycleBatchInput = { settlementId: string mutations: readonly JournalLifecycleMutationInput[] fence: number + /** A provider's own batch: the generation that produced every item it writes + * (`JournalItemAppendOptions.ownerFence`). */ + ownerFence?: number recovered?: true /** Submission verdicts belonging to this settlement, committed before its item rows. */ dispatches?: readonly ResolveDispatchInput[] @@ -81,6 +88,8 @@ export type JournalLifecycleBatchInput = { * follows the messages it failed, and no reader meets one without the other. With none still * queued, the batch is not written either. */ rejectsQueued?: AgentJournalDispatchRejection + /** Narrows `rejectsQueued` to the queued sends this names. */ + rejectsQueuedOnly?: (submission: AgentJournalSubmission) => boolean } /** A lifecycle batch whose rows are chosen at its own turn in the write queue, so a write queued @@ -99,7 +108,7 @@ export type JournalPlannedLifecycleBatchInput = Pick< export type JournalResolvedLifecycleBatchInput = Omit< JournalLifecycleBatchInput, - 'mutations' | 'rejectsQueued' | 'dispatches' + 'mutations' | 'rejectsQueued' | 'rejectsQueuedOnly' | 'dispatches' > & { /** Read from the fold with every earlier write landed; may return none. */ resolve: () => readonly JournalLifecycleMutationInput[] diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 5b9d66cb9d69..f47415804f8a 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -13,7 +13,6 @@ import type { AgentJournalSubmission, AgentJournalThreadGoal, AgentJournalTurnLifecycle, - AgentJournalTurnScope, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' @@ -21,9 +20,8 @@ import { currentAgentSessionThreadGoalBySequence } from '../../../shared/agent-s import type { AgentSessionContextUsage } from '../../../shared/agent-session-context-usage' import { latestStructuredAgentContextFacts } from '../../../shared/structured-agent-session-context-usage' import { - currentStructuredAgentSessionTurnId, - liveStructuredAgentSessionTurnScope, - newestStructuredAgentSessionTurnBySequence + newestStructuredAgentSessionTurnBySequence, + runningStructuredAgentSessionTurnItemBySequence } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' import { readJournalSince } from './journal-cursor' @@ -223,14 +221,14 @@ export class AgentSessionJournal { } } - /** The turn this journal has published as running — the same read a client's snapshot gives, - * without materialising one. Given the lease's fence, only one a current generation opened. */ - activeTurnId = (currentFence?: number | null): string | null => - currentStructuredAgentSessionTurnId(this.state.items.values(), this.itemFence, currentFence) + /** The turn this journal records as running, whichever generation opened it — the same read a + * client's snapshot gives, without materialising one. Whether it is current work is the host + * projection's to say (`structuredAgentSessionCurrentWork`). */ + activeTurnId = (): string | null => this.runningTurn()?.turnId ?? null - /** Where a row written now belongs: the running turn, or the conversation. */ - liveTurnScope = (): AgentJournalTurnScope => - liveStructuredAgentSessionTurnScope(this.state.items.values()) + /** That turn with its record's item. */ + runningTurn = (): { item: AgentJournalRenderItem; turnId: string } | null => + runningStructuredAgentSessionTurnItemBySequence(this.state.items.values()) /** The newest turn record whatever state it settled in, for readers that need the outcome. */ newestTurn = (): AgentJournalTurnLifecycle | null => @@ -247,7 +245,7 @@ export class AgentSessionJournal { /** Includes revisions and completion tombstones, whose timestamps disappear from render items. */ lastActivityAt = (): number => this.state.lastActivityAt - /** Fence of the writer that created the item, while it is in the timeline. */ + /** The item's execution provenance (`JournalItemRow.ownerFence`), while it is in the timeline. */ itemFence = (itemId: string): number | undefined => this.state.itemFences.get(itemId) submissions = (): AgentJournalSubmission[] => [...this.state.submissions.values()] diff --git a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts index 7b03b4f457f8..1369f57892ee 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-fold.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-fold.ts @@ -46,7 +46,8 @@ export function applyJournalSubmission( itemId, 0, journalRenderItem(itemId, 0, row.body, row, turnScope), - row.fence + // A person's message is no generation's work: its provenance is the fence that accepted it. + { fence: row.fence } ) } diff --git a/src/main/native-chat/agent-session-wire/agent-session-history-page.ts b/src/main/native-chat/agent-session-wire/agent-session-history-page.ts index b1667daa512c..e99fe7af5c74 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-history-page.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-history-page.ts @@ -36,6 +36,15 @@ import { submissionBytesByItemId } from './agent-session-history-page-bounds' import { offPageSubagentRoster } from './agent-session-history-subagent-roster' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' + +/** What a page publishes of the host's current-work projection (`structuredAgentSessionCurrentWork`): + * the newest turn record as clients are told it, and the prompts still waiting on the person. + * Absent, a page states the raw record and no prompt ids, as an older host did. */ +export type AgentSessionPublishedWork = Pick< + StructuredAgentSessionCurrentWork, + 'publishedLatestTurn' | 'actionablePromptIds' +> export { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from './agent-session-history-page-bounds' @@ -80,19 +89,20 @@ export function readAgentSessionHistory( /** Reduced state to read against. A synchronous multi-page catch-up passes one * snapshot for the whole run so each page costs its own rows, not the timeline. */ snapshot: AgentJournalSnapshot = journal.snapshot(), - scope: AgentSessionHistoryScope = 'every-agent' + scope: AgentSessionHistoryScope = 'every-agent', + work?: AgentSessionPublishedWork ): AgentSessionHistoryResult { const limit = resolveHistoryLimit(request.limit) if (request.direction === 'after') { - return readForward(journal, snapshot, request.cursor, limit) + return readForward(journal, snapshot, request.cursor, limit, work) } const cursor = request.direction === 'before' ? request.cursor : undefined if (cursor) { if (cursor.epoch !== snapshot.cursor.epoch) { - return historyReset(snapshot, 'epoch_changed') + return historyReset(snapshot, 'epoch_changed', work) } if (cursor.sequence > snapshot.cursor.sequence) { - return historyReset(snapshot, 'cursor_ahead') + return historyReset(snapshot, 'cursor_ahead', work) } } const scoped = @@ -117,7 +127,8 @@ export function readAgentSessionHistory( nextCursor: items[0] ? { epoch: snapshot.cursor.epoch, sequence: items[0].sequence } : undefined, - subagentRoster: offPageSubagentRoster(scoped, items) + subagentRoster: offPageSubagentRoster(scoped, items), + work }) } } @@ -128,7 +139,8 @@ export function readAgentSessionHistory( * check re-reduces if anything did advance the journal between pages. */ export function createAgentSessionCatchUpReader( - journal: AgentSessionJournal + journal: AgentSessionJournal, + work?: AgentSessionPublishedWork ): (request: AgentSessionHistoryRequest) => AgentSessionHistoryResult { let snapshot = journal.snapshot() return (request) => { @@ -136,20 +148,33 @@ export function createAgentSessionCatchUpReader( if (live.epoch !== snapshot.cursor.epoch || live.sequence !== snapshot.cursor.sequence) { snapshot = journal.snapshot() } - return readAgentSessionHistory(journal, request, snapshot) + return readAgentSessionHistory(journal, request, snapshot, 'every-agent', work) } } export function readAgentSessionHydrationPage( journal: AgentSessionJournal, - fence?: number + fence?: number, + work?: AgentSessionPublishedWork ): AgentSessionHistoryPage { - return buildHydrationPage(journal.snapshot(), fence) + return buildHydrationPage(journal.snapshot(), fence, work) +} + +/** The current-work fields a page carries, for a frame that carries no page. */ +export function agentSessionPublishedWorkFields( + journal: AgentSessionJournal, + work: AgentSessionPublishedWork +): Pick { + return { + latestTurn: work.publishedLatestTurn(snapshotLatestTurn(journal.snapshot())), + actionablePromptIds: work.actionablePromptIds() + } } function buildHydrationPage( snapshot: AgentJournalSnapshot, - fence?: number + fence?: number, + work?: AgentSessionPublishedWork ): AgentSessionHistoryPage { const items = conversationWindow(snapshot.items, AGENT_SESSION_HISTORY_MAX_LIMIT) const bounded = boundHistoryItemsByBytes( @@ -169,18 +194,20 @@ function buildHydrationPage( ? { epoch: snapshot.cursor.epoch, sequence: bounded.items[0].sequence } : undefined, fence, - subagentRoster: offPageSubagentRoster(snapshot.items, bounded.items) + subagentRoster: offPageSubagentRoster(snapshot.items, bounded.items), + work }) } function historyReset( snapshot: AgentJournalSnapshot, - reset: Extract['reset'] + reset: Extract['reset'], + work: AgentSessionPublishedWork | undefined ): AgentSessionHistoryResult { return { ok: false, reset, - page: buildHydrationPage(snapshot) + page: buildHydrationPage(snapshot, undefined, work) } } @@ -188,18 +215,19 @@ function readForward( journal: AgentSessionJournal, snapshot: AgentJournalSnapshot, cursor: AgentJournalCursor | undefined, - limit: number + limit: number, + work: AgentSessionPublishedWork | undefined ): AgentSessionHistoryResult { if (!cursor) { // Why: forward paging replays rows after a position; without one there is // nothing to be after, and silently serving the tail would hand the client // a page it cannot place. - return historyReset(snapshot, 'cursor_ahead') + return historyReset(snapshot, 'cursor_ahead', work) } // One lookahead preserves hasNewer without rereading the entire remaining journal per page. const since = journal.readSince(cursor, limit + 1) if (!since.ok) { - return historyReset(snapshot, since.reset) + return historyReset(snapshot, since.reset, work) } const submissionBytes = submissionBytesByItemId(snapshot.submissions) // The page cost is EVERYTHING variable it carries: items with their @@ -225,7 +253,7 @@ function readForward( canonicalItemId: (itemId) => journal.canonicalItemId(itemId) }) if (!projected.ok) { - return historyReset(snapshot, projected.reset) + return historyReset(snapshot, projected.reset, work) } let contentBytes = pageContentBytes(projected.batch.items, projected.batch.removedItemIds) while (rows.length > 1 && contentBytes > HISTORY_PAGE_CONTENT_BUDGET_BYTES) { @@ -237,7 +265,7 @@ function readForward( canonicalItemId: (itemId) => journal.canonicalItemId(itemId) }) if (!shrunk.ok) { - return historyReset(snapshot, shrunk.reset) + return historyReset(snapshot, shrunk.reset, work) } projected = shrunk contentBytes = pageContentBytes(projected.batch.items, projected.batch.removedItemIds) @@ -257,7 +285,7 @@ function readForward( // break the client's keying. A bounded tail replaces the client's state // wholesale, which applies the removal without carrying the id, and the // client resumes from the live cursor past this row. - return historyReset(snapshot, 'cursor_compacted') + return historyReset(snapshot, 'cursor_compacted', work) } const lastSequence = rows.at(-1)?.seq ?? cursor.sequence return { @@ -271,7 +299,8 @@ function readForward( hasOlder: cursor.sequence > 0, hasNewer: since.rows.length > rows.length, fallbackCursor: cursor, - nextCursor: { epoch: cursor.epoch, sequence: lastSequence } + nextCursor: { epoch: cursor.epoch, sequence: lastSequence }, + work }) } } @@ -287,6 +316,7 @@ function buildPage(input: { nextCursor: AgentJournalCursor | undefined fence?: number subagentRoster?: AgentSessionSubagentRosterEntry[] + work?: AgentSessionPublishedWork }): AgentSessionHistoryPage { const epoch = input.snapshot.cursor.epoch const pageItemIds = new Set(input.items.map((item) => item.itemId)) @@ -311,8 +341,14 @@ function buildPage(input: { hasOlder: input.hasOlder, hasNewer: input.hasNewer, ...(input.subagentRoster === undefined ? {} : { subagentRoster: input.subagentRoster }), - // From the whole timeline, never the page: a long turn's record sits below any window. - latestTurn: snapshotLatestTurn(input.snapshot) + // From the whole timeline, never the page: a long turn's record sits below any window. Scoped + // by the host's projection where the reader has one: an ended generation's turn runs nowhere. + ...(input.work + ? { + latestTurn: input.work.publishedLatestTurn(snapshotLatestTurn(input.snapshot)), + actionablePromptIds: input.work.actionablePromptIds() + } + : { latestTurn: snapshotLatestTurn(input.snapshot) }) } } diff --git a/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts b/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts index df98d8ba902b..9f7c600c4fc6 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts @@ -12,7 +12,11 @@ import { import { sameJournalCursor } from '../agent-session-journal/journal-cursor' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { emptyAgentSessionBatch } from './agent-session-empty-batch' -import { createAgentSessionCatchUpReader } from './agent-session-history-page' +import { + agentSessionPublishedWorkFields, + createAgentSessionCatchUpReader +} from './agent-session-history-page' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { subscriberQueuedMessagesChanged, type SubscriberFieldHooks @@ -22,6 +26,11 @@ import type { Subscriber } from './structured-agent-session-subscribers' export type SubscriberDeliveryPort = { hooks: SubscriberFieldHooks & { readCommands?: (sessionId: string) => AgentSessionSlashCommand[] | undefined + /** The host's projection of current work, which every frame's turn and prompt fields state. */ + readCurrentWork?: ( + sessionId: string, + journal: AgentSessionJournal + ) => StructuredAgentSessionCurrentWork | undefined } emit: ( subscriber: Subscriber, @@ -49,12 +58,13 @@ export function deliverToSubscriber( hostNow, ...(publishedActivity !== undefined ? { activity: publishedActivity } : {}) } + const work = port.hooks.readCurrentWork?.(subscriber.sessionId, journal) // Caught up, so there are no rows to read: every publish behind a commit's own delivery. if (sameJournalCursor(subscriber.cursor, journal.cursor())) { - emitCaughtUp(port, subscriber, emitCheckpoint, shared) + emitCaughtUp(port, subscriber, emitCheckpoint, shared, { journal, work }) return } - const readPage = createAgentSessionCatchUpReader(journal) + const readPage = createAgentSessionCatchUpReader(journal, work) while (true) { const result = readPage({ sessionId: subscriber.sessionId, @@ -73,12 +83,13 @@ export function deliverToSubscriber( ...shared }) subscriber.cursor = page.liveCursor ?? page.window.nextCursor + subscriber.workLiveFence = work?.liveFence return } const page = result.page const advanced = page.window.nextCursor.sequence > subscriber.cursor.sequence if (!advanced) { - emitCaughtUp(port, subscriber, emitCheckpoint, shared) + emitCaughtUp(port, subscriber, emitCheckpoint, shared, { journal, work }) return } port.emit( @@ -94,6 +105,9 @@ export function deliverToSubscriber( }, fence: subscriber.fence, ...(page.latestTurn !== undefined ? { latestTurn: page.latestTurn } : {}), + ...(page.actionablePromptIds !== undefined + ? { actionablePromptIds: page.actionablePromptIds } + : {}), ...shared }, // On a multi-page catch-up the draft list rides only the final page, or a @@ -101,6 +115,7 @@ export function deliverToSubscriber( { withholdQueued: page.hasNewer } ) subscriber.cursor = page.window.nextCursor + subscriber.workLiveFence = work?.liveFence if (!page.hasNewer || !port.isActive(subscriber)) { return } @@ -114,19 +129,33 @@ function emitCaughtUp( shared: { hostNow: number activity?: AgentSessionTurnActivity | null - } + }, + current: { journal: AgentSessionJournal; work: StructuredAgentSessionCurrentWork | undefined } ): void { const commandsChanged = port.hooks.readCommands !== undefined && (port.hooks.readCommands(subscriber.sessionId) ?? null) !== subscriber.commands const queuedChanged = subscriberQueuedMessagesChanged(port.hooks, subscriber) - if (emitCheckpoint || shared.activity !== undefined || commandsChanged || queuedChanged) { + // A generation that ended (or began) with no row changes what is current: its frame says so. + const { work } = current + const workChanged = work !== undefined && work.liveFence !== subscriber.workLiveFence + if ( + emitCheckpoint || + shared.activity !== undefined || + commandsChanged || + queuedChanged || + workChanged + ) { port.emit(subscriber, { type: 'batch', sessionId: subscriber.sessionId, batch: emptyAgentSessionBatch(subscriber.cursor), fence: subscriber.fence, + ...(workChanged ? agentSessionPublishedWorkFields(current.journal, work) : {}), ...shared }) + if (workChanged) { + subscriber.workLiveFence = work.liveFence + } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts index 468a48dfcae1..c5c3c3edb168 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-context.ts @@ -33,6 +33,7 @@ export type StructuredAgentSessionAttachContext = { serialize: (sessionId: string, task: () => Promise) => Promise now: () => number publishStatus: (sessionId: string) => void + generationEnded?: StructuredAgentSessionLifetimeContext['generationEnded'] /** Joining a stop's close ends the child's record through the one exit handler. */ endExitedChild: StructuredAgentSessionLifetimeContext['endExitedChild'] wakeDelivery?: StructuredAgentSessionLifetimeContext['wakeDelivery'] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts index 06e325fefd37..a8368d067893 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-failure.ts @@ -84,4 +84,6 @@ export function endStructuredAgentSessionReleasedChild( context.runtimeState.currentEventSink(sessionId)?.close() context.runtimeState.discardEventSink(sessionId) context.publishStatus?.(sessionId) + // The released acquisition was the live generation: readers and the queued-card drain re-derive. + context.generationEnded?.(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index 1563146f7e4a..4ff230228538 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -36,6 +36,7 @@ import type { StructuredAgentRegistry } from './structured-agent-registry' import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' import { resolveAgentSessionReplayOutcome } from './structured-agent-session-replay-outcome' import { readAgentSessionHydrationPage } from './agent-session-history-page' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { acquireOwner } from './structured-agent-session-acquisition' import { importAdoptedTranscript, @@ -281,7 +282,12 @@ export async function performAttach( value: { sessionId, fence, - page: readAgentSessionHydrationPage(attached.journal, fence), + // The lease this attach just took is the live generation. + page: readAgentSessionHydrationPage( + attached.journal, + fence, + structuredAgentSessionCurrentWork(attached.journal, { record }) + ), unconfirmedClientMessageIds: attached.unconfirmedClientMessageIds, ...(tabId ? { tabId } : {}) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index cf44fc3d66b5..722450bbb320 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -7,6 +7,7 @@ import type { AgentChildWorkView } from '../../../shared/agent-status-child-work import { structuredChildWorkLegacyTasks } from '../../../shared/structured-agent-session-child-work-legacy' import { structuredRunningChildWork } from '../../../shared/agent-child-work-listing' import { readStructuredAgentSessionHistoryResult } from './structured-agent-session-history-result' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { structuredQueueSendGate, tryReadQueuePublication @@ -45,16 +46,24 @@ export class StructuredAgentSessionBackgroundTaskChannel { scope?: AgentSessionHistoryScope ): Promise { const journal = (await this.conversation(request.sessionId)).journal + const work = hostStructuredAgentSessionCurrentWork( + { store: this.deps.store, sessions: this.sessions }, + request.sessionId + ) const result = readStructuredAgentSessionHistoryResult({ journal, record: this.deps.store.getRecord(request.sessionId), request, - scope + scope, + ...(work ? { work } : {}) }) const backgroundTasks = this.read(request.sessionId) const queue = tryReadQueuePublication( journal, - structuredQueueSendGate(this.deps.store, request.sessionId) + structuredQueueSendGate( + { store: this.deps.store, sessions: this.sessions }, + request.sessionId + ) ) const hostNow = this.deps.now?.() ?? Date.now() return { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 24f144327305..553ea598107f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -53,14 +53,24 @@ export type StructuredAgentSessionChildExitSession = Pick< 'child' | 'lastEndedChild' > & { journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal } +/** `StructuredAgentSessionClientDelivery.publishGenerationEnded`: the one edge every way a + * generation ends goes through. */ +export type StructuredAgentSessionGenerationEnded = ( + sessionId: string, + options?: { restate?: boolean } +) => void + export type StructuredAgentSessionChildExitContext< TSession extends StructuredAgentSessionChildExitSession = StructuredAgentSessionHostSession > = { store: StructuredAgentSessionLeaseStore & StructuredAgentSessionLeftoverStore sessions: Map flushLifecycle: (sessionId: string) => Promise - publishFence: (sessionId: string, session: TSession) => void publishStatus?: (sessionId: string) => void + /** A generation ended: every reader re-derives current work and the queued-card drain is + * scheduled, with no journal write needed (`publishGenerationEnded`). `restate`: each chat + * re-baselines at the moved fence. */ + generationEnded: StructuredAgentSessionGenerationEnded /** The delivery loop hands over whatever is queued once the child is off the record. */ wakeDelivery?: (sessionId: string) => void /** An ended child's start, if still open, has nothing left to time. */ @@ -169,6 +179,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< if (!record || record.lease.handoffStage !== null) { // An acquisition or recovery already owns this lease's transition. endChild() + context.generationEnded(sessionId) context.wakeDelivery?.(sessionId) return } @@ -268,10 +279,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< } endChild() // A reader re-baselines on a death of the child's own; a close Orca asked for moves no fence a - // reader holds, as a client resends a message when its fence moves. - if (released && !expected) { - context.publishFence(sessionId, session) - } + // reader holds, as a client resends a message when its fence moves, and discards a reply (the + // Stop's own) issued at the fence before. Either way every reader learns the generation ended + // and the queued-card drain runs, whether or not the release or the settlement was written. + context.generationEnded(sessionId, { restate: released && !expected }) context.wakeDelivery?.(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts index 2ea3a16024be..35763ea56a30 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-root-exit.test.ts @@ -145,7 +145,7 @@ describe('Claude root-exit stop', () => { store, sessions, flushLifecycle: (id) => runtimeState.lifecycleBarrier(id), - publishFence: () => undefined, + generationEnded: () => undefined, publishStatus, serialize: (_sessionId, task) => task(), now: () => NOW + 30 * 60_000, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index c369ce1a5602..c4ebfa5a962c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -15,6 +15,8 @@ import type { } from './structured-agent-session-host-types' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import type { StructuredAgentSessionStatusSubscriber } from './structured-agent-session-status-feed' import { createStructuredAgentSessionHostStatusFeed } from './structured-agent-session-host-status-feed' import { @@ -35,7 +37,8 @@ export class StructuredAgentSessionClientDelivery { private readonly sessions: Map, now: () => number, private readonly deps: () => StructuredAgentSessionHostDeps, - private readonly onJournalActivity: (sessionId: string) => void, + /** `activity`: a journal write, which renews the idle clock; a generation's end is not one. */ + private readonly onJournalActivity: (sessionId: string, activity?: boolean) => void, onAgentStarted: (sessionId: string) => void, /** A session's child records changed; the chat strip republishes from them. */ onChildWorkChanged: (sessionId: string) => void, @@ -56,7 +59,7 @@ export class StructuredAgentSessionClientDelivery { this.statusFeed.journalProjection(sessionId, journal)?.state ?? null }) this.sendSettlement = new StructuredAgentSessionSendSettlement((sessionId) => - this.requireJournal(sessionId) + this.settlementReading(sessionId, this.requireJournal(sessionId)) ) this.waitForSendSettlement = this.sendSettlement.wait this.subscribers = new AgentSessionSubscribers({ @@ -64,10 +67,11 @@ export class StructuredAgentSessionClientDelivery { readQueuePublication: (sessionId) => tryReadQueuePublication( sessions.get(sessionId)?.journal, - structuredQueueSendGate(this.deps().store, sessionId) + structuredQueueSendGate({ store: this.deps().store, sessions }, sessionId) ), readBackgroundTasks, - onJournalPublished: (sessionId, journal) => this.publishJournal(sessionId, journal) + onJournalPublished: (sessionId, journal) => this.publishJournal(sessionId, journal), + readCurrentWork: (sessionId, journal) => this.currentWork(sessionId, journal) }) } @@ -110,10 +114,35 @@ export class StructuredAgentSessionClientDelivery { this.statusFeed.publish(sessionId) const journal = this.sessions.get(sessionId)?.journal if (journal) { - this.sendSettlement.publish(sessionId, journal) + this.sendSettlement.publish(sessionId, this.settlementReading(sessionId, journal)) } } + /** A generation ended — an exit, a release — which can change what is current with no journal + * row. The one edge for every way a generation ends: every reader re-derives current work (the + * status feed, settlement waits, each chat's frames) and the queued-card drain is scheduled, + * whatever became of the cleanup's own write. An end is no activity: it renews no idle clock. + * `restate`: each chat re-baselines at the fence, as a death of the child's own does. */ + publishGenerationEnded = (sessionId: string, options: { restate?: boolean } = {}): void => { + const journal = this.sessions.get(sessionId)?.journal + if (!journal) { + return + } + if (options.restate) { + const fence = structuredAgentSessionConversationFence(this.deps().store, sessionId) + this.subscribers.snapshot(sessionId, journal, fence) + return + } + this.subscribers.deliverFrames(sessionId, journal) + this.publishJournal(sessionId, journal, false) + } + + /** A held chat's current work (`structuredAgentSessionCurrentWork`); null when none is held. */ + readCurrentWork = (sessionId: string) => { + const journal = this.sessions.get(sessionId)?.journal + return journal ? this.currentWork(sessionId, journal) : null + } + publishRestored = (sessionId: string): void => { this.statusFeed.publish(sessionId, undefined, { replay: true }) this.turnCompletionFeed.observe(sessionId, undefined, { historical: true }) @@ -148,14 +177,31 @@ export class StructuredAgentSessionClientDelivery { this.sendSettlement.closeAll() } - private publishJournal(sessionId: string, journal: AgentSessionJournal): void { + private publishJournal(sessionId: string, journal: AgentSessionJournal, activity = true): void { this.statusFeed.publish(sessionId, journal) - this.sendSettlement.publish(sessionId, journal) + this.sendSettlement.publish(sessionId, this.settlementReading(sessionId, journal)) // Derived here rather than per-subscriber: this edge runs whether or not anyone is // subscribed, which is the whole reason a backgrounded chat can complete at all. After the // status publish, so it reads the projection that publish cached. this.turnCompletionFeed.observe(sessionId, journal) - this.onJournalActivity(sessionId) + this.onJournalActivity(sessionId, activity) + } + + /** The current-work projection over this journal (`structuredAgentSessionCurrentWork`). */ + private currentWork(sessionId: string, journal: AgentSessionJournal) { + const ended = this.sessions.get(sessionId)?.lastEndedChild + return structuredAgentSessionCurrentWork(journal, { + record: this.deps().store.getRecord(sessionId), + ...(ended ? { ended } : {}) + }) + } + + private settlementReading(sessionId: string, journal: AgentSessionJournal) { + return { + submissions: () => journal.submissions(), + cursor: () => journal.cursor(), + activeTurnId: () => this.currentWork(sessionId, journal).activeTurnId() + } } private requireJournal(sessionId: string): AgentSessionJournal { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts index 30810b9b9f3e..302bc99646af 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-command-turn.ts @@ -44,6 +44,10 @@ import type { StructuredAgentRegistry } from './structured-agent-registry' import { conversationCommandBlocked } from './structured-conversation-command-admission' import { structuredAgentSessionCommandTurn } from '../../../shared/structured-agent-session-command-turn-identity' import { refuseQueuedCommand } from './structured-agent-session-queued-command-refusal' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' export { structuredAgentSessionCommandTurn } from '../../../shared/structured-agent-session-command-turn-identity' @@ -90,11 +94,12 @@ export function structuredAgentSessionAwaitedCommand( : undefined } -/** Whether the journal's running turn is a command's, which takes no input while it runs. */ +/** Whether the live generation's running turn is a command's, which takes no input while it runs + * (`structuredAgentSessionCurrentWork`). */ export function structuredAgentSessionCommandRunning( - journal: Pick + work: Pick ): boolean { - const turnId = journal.activeTurnId() + const turnId = work.activeTurnId() return turnId !== null && isStructuredAgentSessionCommandTurnId(turnId) } @@ -220,7 +225,8 @@ async function openCommandTurn( clientMessageId, state: 'pending', fence: ctx.fence, - turnScope: ctx.journal.liveTurnScope() + // The child it goes to is live at `fence`: an ended generation's turn is not joined. + turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() }) const startedAt = ctx.now() const running = agentJournalTurnBody({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 431ca9e761de..4211929805cc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -84,6 +84,7 @@ export function createStructuredAgentSessionConversationLifetime(host: { isDisposed: () => disposed, deliveryActive: host.deliveryActive, childWork: host.readChildWork, + getRecord: (sessionId) => deps().store.getRecord(sessionId), hasOpenDispatch: (sessionId) => { const record = deps().store.getRecord(sessionId) return record !== null && deps().hasOpenDispatch?.(record) === true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts new file mode 100644 index 000000000000..d0ac1f96eb04 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts @@ -0,0 +1,299 @@ +// The host's one projection of current work (`structuredAgentSessionCurrentWork`), end to end +// against the real host, store and SQLite journal: what an ended generation left holds nothing and +// reads as nothing to every reader (the projection, the session list, a chat client, Stop, a prompt +// answer, the queue), whether or not its cleanup or its release landed, while a live owner's work +// holds exactly as before. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionExecutionLocation } from '../../../shared/agent-session-record' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { + isActionableStructuredAgentSessionPrompt, + runningStructuredAgentSessionTurnId +} from '../../../shared/structured-agent-session-live-turn' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession, + type StructuredAgentSessionState +} from '../../../shared/structured-agent-session-reducer' +import { + liveTestJournalRows, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + APPROVAL, + currentJournal, + currentWork, + exitWhileSettlementFails, + leaveUnfinishedWork, + REJECT_RECOVERED_ROWS, + sendText, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +const APPROVAL_ID = agentJournalItemKey(APPROVAL) +const FOLDER: AgentSessionExecutionLocation = { + executionHostId: 'local', + wslDistro: null, + workspaceId: 'folder-1', + workspaceKind: 'folder' +} + +/** A chat client and the session list, each fed by the host as a real one is. */ +function watch(current: QueuedMessageTestRig) { + let chat: StructuredAgentSessionState = EMPTY_STRUCTURED_AGENT_SESSION + let listed: AgentSessionStatusSummary | undefined + current.host.subscribe({ + id: 'chat-1', + sessionId: SESSION, + emit: (event) => { + chat = reduceStructuredAgentSession(chat, { type: 'event', event: structuredClone(event) }) + } + }) + current.host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + if (event.type === 'status' && event.session.sessionId === SESSION) { + listed = structuredClone(event.session) + } else if (event.type === 'snapshot') { + listed = event.sessions.find((session) => session.sessionId === SESSION) ?? listed + } + } + }) + return { + chat: () => chat, + listed: () => listed, + /** Whether the chat would let the person answer the prompt. */ + answerable: () => + chat.items.some( + (item) => + item.itemId === APPROVAL_ID && + item.body.kind === 'approval' && + item.body.resolution.state === 'pending' && + isActionableStructuredAgentSessionPrompt(item.itemId, chat.actionablePromptIds) + ) + } +} + +/** The tail page a client hydrates from. */ +async function hydration(current: QueuedMessageTestRig) { + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + if (!page.ok) { + throw new Error('history refused') + } + return page.page +} + +async function deadGenerationWithFailedSettlement(location?: typeof FOLDER) { + rig = await createQueuedMessageTestRig({ + restartable: true, + ...(location ? { location } : {}) + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const view = watch(current) + const exited = await exitWhileSettlementFails(current) + return { current, view, ...exited } +} + +describe("a dead generation's leftovers, its settlement refused and its lease released", () => { + it.each([ + ['a git worktree', undefined], + ['a folder workspace', FOLDER] + ])('in %s, read as nothing by every reader, and hold nothing', async (_name, location) => { + const { current, view } = await deadGenerationWithFailedSettlement(location) + // Still saved as running and pending: only a settlement would end them. + expect(turnState(current)).toBe('running') + const work = currentWork(current) + + expect(work.liveFence).toBeNull() + expect(work.working()).toBe(false) + expect(work.actionablePromptIds()).toEqual([]) + expect(view.listed()?.status).not.toBe('working') + expect(current.host.readStatusSummary(SESSION)?.status).not.toBe('working') + // A client hydrating now, and the one already watching, read the same. + const page = await hydration(current) + expect(page.latestTurn).toBeNull() + expect(page.actionablePromptIds).toEqual([]) + await vi.waitFor(() => expect(view.chat().actionablePromptIds).toEqual([])) + expect(runningStructuredAgentSessionTurnId(view.chat())).toBeNull() + expect(view.answerable()).toBe(false) + }) + + it('gives Stop nothing to stop: the dead turn is not touched', async () => { + const { current } = await deadGenerationWithFailedSettlement() + + await current.stop() + + expect(current.cancelTurn).not.toHaveBeenCalled() + expect(turnState(current)).toBe('running') + }) +}) + +describe('a close Orca asked for, its settlement refused', () => { + it('tells a watching client the turn and prompt are over, with no row to carry it', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const view = watch(current) + await vi.waitFor(() => expect(view.answerable()).toBe(true)) + const database = openTestJournalHostDatabase(current.root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + const before = currentJournal(current).cursor().sequence + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'requested-close', + reason: 'closed by Orca', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + + expect(currentJournal(current).cursor().sequence).toBe(before) + expect(turnState(current)).toBe('running') + await vi.waitFor(() => expect(view.answerable()).toBe(false)) + expect(runningStructuredAgentSessionTurnId(view.chat())).toBeNull() + expect(view.listed()?.status).not.toBe('working') + }) +}) + +describe('a waiting card when the generation ahead of it ends', () => { + it.each([ + ['an exit of its own, its settlement refused', 'unexpected-exit' as const], + ['a close Orca asked for', 'requested-close' as const] + ])('drains after %s, with no journal write in between', async (_name, cause) => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const queued = await sendText(current, 'behind the turn').result + expect(queued).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + const journal = currentJournal(current) + const [card] = journal.queuedMessages.list() + expect(card?.state).toBe('waiting') + const database = openTestJournalHostDatabase(current.root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + const before = journal.cursor().sequence + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause, + reason: 'ended', + fence: child.fence, + acquisitionGeneration: child.generation + }) + + await vi.waitFor(() => + expect(journal.queuedMessages.get(card?.messageId ?? '')?.state).toBe('dispatched') + ) + // The first row after the end is the drain's own hand-off: nothing was written to wake it. + const first = liveTestJournalRows(database.db, SESSION).find((row) => row.seq > before) + expect(JSON.parse(first?.rowJson ?? '{}')).toMatchObject({ + kind: 'submission', + queuedMessageId: card?.messageId + }) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('an observed exit whose release write failed', () => { + it('reads as not Working, and a send is delivered', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const transition = current.store.transitionHandoff.bind(current.store) + let refused = 0 + vi.spyOn(current.store, 'transitionHandoff').mockImplementation((sessionId, change) => + transition(sessionId, (record) => { + const next = change(record) + if (refused === 0 && next.lease.claimStatus === 'released') { + refused += 1 + throw new Error('disk full') + } + return next + }) + ) + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } + + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) + await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + + expect(refused).toBe(1) + // The release never landed: the lease still names the gone child. + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'live', + runtimeFence: child.fence + }) + const work = currentWork(current) + expect(work.liveFence).toBeNull() + expect(work.working()).toBe(false) + expect(work.actionablePromptIds()).toEqual([]) + expect(current.host.readStatusSummary(SESSION)?.status).not.toBe('working') + + expect((await sendText(current, 'go on').result).ok).toBe(true) + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + }) +}) + +describe('a live owner', () => { + it('publishes its running turn and its prompt exactly as before', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const view = watch(current) + + const work = currentWork(current) + expect(work.liveFence).toBe(current.store.getRecord(SESSION)?.lease.runtimeFence) + expect(work.activeTurnId()).toBe('unfinished') + expect(work.actionablePromptIds()).toEqual([APPROVAL_ID]) + const page = await hydration(current) + expect(page.latestTurn?.turn).toMatchObject({ turnId: 'unfinished', state: 'running' }) + expect(page.actionablePromptIds).toEqual([APPROVAL_ID]) + await vi.waitFor(() => expect(view.chat().actionablePromptIds).toEqual([APPROVAL_ID])) + expect(runningStructuredAgentSessionTurnId(view.chat())).toBe('unfinished') + expect(view.answerable()).toBe(true) + expect(view.listed()?.status).not.toBe('idle') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts new file mode 100644 index 000000000000..efca4ba2c835 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -0,0 +1,206 @@ +// The host's one operational projection of a chat's current work: the live generation's running +// turn, the prompts it raised that still wait on the person, the sends it owes, and the queued +// sends this process may hand over. Every host reader asks here; none keeps a rule of its own. +// +// Derived on every read, never stored, from what the host already holds: +// - the lease: the live or reserved generation's fence; none once it is released; +// - this host's own sight of that generation's root exit, which ends it even when the release +// write failed (`lastEndedChild`); +// - the journal's fold, where every item carries the generation that produced it (`ownerFence`). +// So work an ended generation left holds nothing and is not Working whether or not its cleanup +// landed: cleanup is bookkeeping and never gates what the person does next. Loss of contact is +// never an end: an unreconciled or conflicted lease keeps its fence, and its work stays current. + +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalRenderItem, + type AgentJournalSubmission, + type AgentJournalTurnScope +} from '../../../shared/agent-session-journal-types' +import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionLatestTurn } from '../../../shared/agent-session-wire' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' +import { isUnansweredStructuredAgentSessionDispatch } from '../../../shared/structured-agent-session-unanswered-dispatch' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionHostSession +} from './structured-agent-session-host-types' + +export type StructuredAgentSessionCurrentWorkJournal = Pick< + AgentSessionJournal, + 'runningTurn' | 'itemFence' | 'visitItems' | 'submissions' | 'wroteBeforeOpen' +> + +/** What the host knows of who runs the chat now. */ +export type StructuredAgentSessionWorkEvidence = { + record: Pick | null + /** The last child this host saw end (`StructuredAgentSessionHostSession.lastEndedChild`). */ + ended?: Pick +} + +/** The live or reserved generation's fence, or null when no generation is live. With no record + * nothing has ended: fence 0, under which every item is current, as before records scoped work. */ +export function structuredAgentSessionLiveFence( + evidence: StructuredAgentSessionWorkEvidence +): number | null { + const lease = evidence.record?.lease + if (!lease) { + return 0 + } + if (lease.claimStatus === 'released') { + return null + } + const { ended } = evidence + return ended?.rootGone === true && ended.fence === lease.runtimeFence ? null : lease.runtimeFence +} + +/** `StructuredAgentSessionCurrentWork.handsOver`, for a reader that needs no other answer. */ +export function structuredAgentSessionHandsOver( + journal: Pick, + submission: Pick< + AgentJournalSubmission, + 'handoverRecorded' | 'dispatchState' | 'handedOverAt' | 'acceptedSequence' + > +): boolean { + return ( + isQueuedAgentJournalSubmission(submission) && + !journal.wroteBeforeOpen(submission.acceptedSequence) + ) +} + +/** One read of a chat's current work; build a fresh one per decision, since nothing is cached. */ +export class StructuredAgentSessionCurrentWork { + /** The shared projections' form of this answer (`StructuredAgentSessionWorkScope`). */ + readonly scope: StructuredAgentSessionWorkScope = { + isCurrentItem: (itemId) => this.isCurrentItem(itemId), + owesSend: (submission) => this.owesSend(submission) + } + + constructor( + private readonly journal: StructuredAgentSessionCurrentWorkJournal, + /** `structuredAgentSessionLiveFence`. */ + readonly liveFence: number | null + ) {} + + /** Whether the live generation's execution produced this item. */ + isCurrentItem(itemId: string): boolean { + if (this.liveFence === null) { + return false + } + const provenance = this.journal.itemFence(itemId) + return provenance === undefined || provenance >= this.liveFence + } + + /** Whether this process may hand this send over: accepted here, never by an earlier host process. + * Derived from where the row sits, so nothing has to be written first; keeping an earlier + * process's send as a card is the startup share's bookkeeping (`holdUnsentSends`). */ + handsOver(submission: AgentJournalSubmission): boolean { + return structuredAgentSessionHandsOver(this.journal, submission) + } + + /** A send still owed: one this process may hand over, or one handed to the live generation that + * it has neither answered nor refused (`isUnansweredStructuredAgentSessionDispatch`). */ + owesSend(submission: AgentJournalSubmission): boolean { + if (isQueuedAgentJournalSubmission(submission)) { + return this.handsOver(submission) + } + return ( + this.liveFence !== null && + isUnansweredStructuredAgentSessionDispatch(submission, this.liveFence) + ) + } + + /** The live generation's running turn, with its record's item. */ + activeTurn(): { item: AgentJournalRenderItem; turnId: string } | null { + const running = this.journal.runningTurn() + return running && this.isCurrentItem(running.item.itemId) ? running : null + } + + activeTurnId(): string | null { + return this.activeTurn()?.turnId ?? null + } + + /** Where a row written now joins: the live generation's running turn, else the conversation. */ + turnScope(): AgentJournalTurnScope { + const running = this.activeTurn() + return running ? { kind: 'turn', turnItemId: running.item.itemId } : AGENT_JOURNAL_THREAD_SCOPE + } + + /** The approvals and questions the live generation raised that wait on the person, a + * subagent's included. */ + actionablePromptIds(): string[] { + const ids: string[] = [] + this.journal.visitItems((itemId, _sequence, body) => { + if ( + (body.kind === 'approval' || body.kind === 'question') && + body.resolution.state === 'pending' && + this.isCurrentItem(itemId) + ) { + ids.push(itemId) + } + }) + return ids + } + + hasActionablePrompt(): boolean { + return this.actionablePromptIds().length > 0 + } + + owesAnySend(): boolean { + return this.journal.submissions().some((submission) => this.owesSend(submission)) + } + + /** The session's own agent is working: its running turn, or a send it owes. */ + working(): boolean { + return this.activeTurnId() !== null || this.owesAnySend() + } + + /** The newest turn record as a client is told it: a running one an ended generation opened is + * none, so no client reads it as working or steers into it. */ + publishedLatestTurn(latest: AgentSessionLatestTurn | null): AgentSessionLatestTurn | null { + return latest?.turn.state === 'running' && !this.isCurrentItem(latest.itemId) ? null : latest + } +} + +export function structuredAgentSessionCurrentWork( + journal: StructuredAgentSessionCurrentWorkJournal, + evidence: StructuredAgentSessionWorkEvidence +): StructuredAgentSessionCurrentWork { + return new StructuredAgentSessionCurrentWork(journal, structuredAgentSessionLiveFence(evidence)) +} + +/** The host's sessions and lease store, as the projection reads them. */ +export type StructuredAgentSessionCurrentWorkHost = { + store: { getRecord: (sessionId: string) => AgentSessionRecord | null } + sessions: { + get( + sessionId: string + ): Pick | undefined + } +} + +/** A held chat's current work; null when this host holds no conversation for it. */ +export function hostStructuredAgentSessionCurrentWork( + host: StructuredAgentSessionCurrentWorkHost, + sessionId: string +): StructuredAgentSessionCurrentWork | null { + const session = host.sessions.get(sessionId) + return session + ? structuredAgentSessionCurrentWork(session.journal, { + record: host.store.getRecord(sessionId), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) + }) + : null +} + +/** An operation context's current work: the host projection it was built with, or, for one built + * without (a test's), the generation at its fence read as live. */ +export function contextStructuredAgentSessionCurrentWork(ctx: { + journal: StructuredAgentSessionCurrentWorkJournal + fence: number + currentWork?: () => StructuredAgentSessionCurrentWork +}): StructuredAgentSessionCurrentWork { + return ctx.currentWork?.() ?? new StructuredAgentSessionCurrentWork(ctx.journal, ctx.fence) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts index 1858eaeddf8d..85cd97d0d6eb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -9,8 +9,10 @@ // that arrives while a start holds the queue withdraw the queued messages before the handover that // would have written them. Stop and the conversation's close are the only other writers of a // queued message: a child's exit only ends the child, and this loop reads why. A message an -// earlier host process left queued is never handed over: the open, or this loop's first step, -// settles it first (`journal-unsent-send-hold.ts`). +// earlier host process left queued is never handed over: the loop hands over only what this +// process accepted (`structuredAgentSessionHandsOver`), read off the journal, so nothing has to be +// written first. Keeping that message as a card (`journal-unsent-send-hold.ts`) is the startup +// share's bookkeeping, and its failure holds up no one. import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import type { AgentSessionRecord } from '../../../shared/agent-session-record' @@ -37,7 +39,7 @@ import { handOverSubmission } from './structured-agent-session-turns' import { structuredAgentSessionNextHandover } from './structured-agent-session-opening-send' import { structuredAgentSessionCommandRunning } from './structured-agent-session-command-turn' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import { holdRestartedStructuredAgentSessionSends } from './structured-agent-session-host-lifetime' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { structuredAgentSessionEndedChildFailure } from './structured-agent-session-ended-child-failure' export type StructuredAgentSessionDeliveryLoopDeps = { @@ -167,21 +169,12 @@ export class StructuredAgentSessionDeliveryLoop { } } - /** Settles what an earlier host process left queued, then makes the session ready. */ + /** Makes the session ready for the oldest send this process may hand over. */ private async prepare(sessionId: string): Promise { const session = this.deps.sessions.get(sessionId) if (!session || this.disposed) { return this.stop(sessionId) } - // The open already did, unless its write failed: a row an earlier handle wrote is never handed - // over, whether it outlived a quit or a crash. A failure here throws, so none is: this run then - // fails, which rejects every queued send, this process's own too. - await holdRestartedStructuredAgentSessionSends( - this.deps.logger, - sessionId, - session.journal, - this.deps.conversationFence(sessionId) - ) if (!(await this.closeWhatTheUserClosed(sessionId, session))) { // Never start an agent for a message the user closed; the next wake re-derives and retries. return this.stop(sessionId) @@ -189,7 +182,16 @@ export class StructuredAgentSessionDeliveryLoop { const oldest = oldestQueuedSubmission(session) // A running command takes no input while its child carries it; its end is a commit, which // wakes the loop again. With no child it is a gone generation's, which the start below settles. - if (!oldest || (session.child && structuredAgentSessionCommandRunning(session.journal))) { + if ( + !oldest || + (session.child && + structuredAgentSessionCommandRunning( + structuredAgentSessionCurrentWork(session.journal, { + record: this.deps.record(sessionId), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) + }) + )) + ) { return this.stop(sessionId) } const failedStart = startThatFailedWhileQueued(session, oldest) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts index 51894a9ff8d7..a2b30d57c301 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-recovery.ts @@ -18,7 +18,8 @@ import { endExitedStructuredAgentSessionChildUnderSerialize, settleStructuredAgentSessionChildExit, type StructuredAgentSessionChildExit, - type StructuredAgentSessionChildExitContext + type StructuredAgentSessionChildExitContext, + type StructuredAgentSessionGenerationEnded } from './structured-agent-session-child-exit' import type { StructuredAgentSessionHostRuntimeState } from './structured-agent-session-host-runtime-state' @@ -31,8 +32,8 @@ export class StructuredAgentSessionEventRecovery { store: StructuredAgentSessionHostDeps['store'] sessions: Map flushLifecycle: (sessionId: string) => Promise - publishFence: (sessionId: string, session: StructuredAgentSessionHostSession) => void publishStatus?: (sessionId: string) => void + generationEnded: StructuredAgentSessionGenerationEnded serialize: (sessionId: string, task: () => Promise) => Promise now: () => number runtimeState: StructuredAgentSessionHostRuntimeState diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts index 476089292a93..3a443559e2e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.test.ts @@ -577,7 +577,7 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } + { fence: 5, ownerFence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } ]) deferred.close() } @@ -601,7 +601,7 @@ describe('producer linkage reaches the journal through every append path', () => await deferred.drained() expect(journalAppendOptions).toEqual([ - { fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } + { fence: 5, ownerFence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE, ...LINKAGE } ]) deferred.close() } @@ -637,7 +637,10 @@ describe('producer linkage reaches the journal through every append path', () => // A control, not a pin. Absence is the claim, so the keys must be missing // rather than present-and-undefined: a reader holding this options object // would read `agentId: undefined` as a key that exists. - expect(journalAppendOptions).toEqual([{ fence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE }]) + // A provider observation names the generation that produced it: the bound fence. + expect(journalAppendOptions).toEqual([ + { fence: 5, ownerFence: 5, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ]) deferred.close() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts index 1e2d2c64d70a..d375bc9cddd7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-event-sink.ts @@ -246,7 +246,9 @@ export function createDeferredStructuredAgentSessionEventSink(deps: { settlementId, mutations, // No row-level linkage: each mutation names its own (see the batch row builder). - fence: bound.fence + fence: bound.fence, + // The provider's own observation: its child's generation produced every item. + ownerFence: bound.fence }) }, { ...options, lifecycle: true } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts index 49fb06e83bad..882e7f940629 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-history-result.ts @@ -8,7 +8,8 @@ import type { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { readAgentSessionHistory, - type AgentSessionHistoryScope + type AgentSessionHistoryScope, + type AgentSessionPublishedWork } from './agent-session-history-page' export function structuredAgentSessionProviderSessionMetadata( @@ -23,8 +24,16 @@ export function readStructuredAgentSessionHistoryResult(input: { record: AgentSessionRecord | null request: AgentSessionHistoryRequest scope?: AgentSessionHistoryScope + /** The host's current-work projection, which the page's turn and prompt fields state. */ + work?: AgentSessionPublishedWork }): AgentSessionHistoryResult { - const result = readAgentSessionHistory(input.journal, input.request, undefined, input.scope) + const result = readAgentSessionHistory( + input.journal, + input.request, + undefined, + input.scope, + input.work + ) const fence = input.record?.lease.runtimeFence const providerSession = structuredAgentSessionProviderSessionMetadata(input.record) if (fence === undefined) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index dbc86bba0676..44abe4ae31af 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -2,7 +2,7 @@ // messages to a provider child. Bundled because they share one invariant — a conversation open // with a message queued has a delivery loop — and the open is where a loop for leftovers wakes. -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { holdClosedStructuredAgentSessionSends } from './structured-agent-session-host-lifetime' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { @@ -72,6 +72,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { : sessions.get(sessionId) await retireSignedOutStructuredAgentSessionChild(sessionId, session, { work: { + currentWork: () => + hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId), childWork: () => input.clientDelivery.readChildWork(sessionId), hasOpenDispatch: () => { const record = deps.store.getRecord(sessionId) @@ -118,10 +120,12 @@ export function createStructuredAgentSessionConversationDelivery(input: { } const wakesQueued = new Set() const afterCommit = (sessionId: string, journal: AgentSessionJournal): void => { + const work = hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId) if ( wakesQueued.has(sessionId) || - structuredAgentSessionCommandRunning(journal) || - !journal.submissions().some(isQueuedAgentJournalSubmission) + !work || + structuredAgentSessionCommandRunning(work) || + !journal.submissions().some((submission) => work.handsOver(submission)) ) { return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index ae1e1619380e..f2e6dff0bdd7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -49,6 +49,8 @@ export type StructuredAgentSessionLifetimeContext = { now: () => number /** Re-projects the session's status after its agent stopped and the chat stays. */ publishStatus?: (sessionId: string) => void + /** A generation ended (`StructuredAgentSessionClientDelivery.publishGenerationEnded`). */ + generationEnded?: (sessionId: string) => void /** Hands the delivery loop what is queued; for a caller inside the session's serialize. */ wakeDelivery?: (sessionId: string) => void /** The one exit handler (`structured-agent-session-child-exit`), for a caller inside the @@ -108,26 +110,6 @@ export async function holdClosedStructuredAgentSessionSends( return settled.ok } -/** What an earlier host process left queued and the open could not settle (its write failed): - * kept or rejected now, never handed over. A failure throws. A send that woke this came after the - * ones it settled, so the reopen mark starts where they did. */ -export async function holdRestartedStructuredAgentSessionSends( - logger: StructuredAgentSessionHostDeps['logger'], - sessionId: string, - journal: StructuredAgentSessionHostSession['journal'], - fence: number -): Promise { - const hostInstance = structuredAgentSessionHostInstance() - const settled = await holdUnsentSends(journal, { - fence, - hostInstance, - hold: { cause: 'hostRestarted' } - }) - if (settled !== null) { - await markStructuredQueueReopen(sessionId, journal, fence, logger, settled + 1) - } -} - /** * The agent goes to rest; the conversation stays. Begins the child's close, or joins the one * already begun, and waits for the exit's proof as long as a caller may. Still unproven, it throws diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts index 1d98c835ca50..2ad4dd4e1089 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts @@ -1,7 +1,10 @@ // The Stop event a host stop writes (`JournalStopEvent`): whether it ends work its event must // record, and the write itself, issued before the kill. -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { StructuredAgentSessionStopCause } from './structured-agent-session-adapter' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -62,22 +65,25 @@ export async function stopEndsWork( STOP_EVENT_DRAIN_TIMEOUT_MS, 'slow' as const ) + // The child is live: the work its stop ends is its generation's. + const work = structuredAgentSessionCurrentWork(journal, { + record: context.deps.store.getRecord(sessionId), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) + }) const working = - (drain === 'slow' && journal.stopMarks.latestAcceptedSendUnopened()) || - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(child.fence), - journal.submissions(), - child.fence - ) - return working && (ending.cause === 'user-close' || !defersToPersonsStop(session, child.fence)) + (drain === 'slow' && journal.stopMarks.latestAcceptedSendUnopened()) || work.working() + return working && (ending.cause === 'user-close' || !defersToPersonsStop(session, work)) } /** A host stop of work a person's Stop is already ending must not supersede that Stop's reason: * the turn it decides, or, with no turn running, the work under the queue pause it still holds, * with nothing sent since, which a host's event would lift. */ -function defersToPersonsStop(session: StructuredAgentSessionHostSession, fence: number): boolean { +function defersToPersonsStop( + session: StructuredAgentSessionHostSession, + work: StructuredAgentSessionCurrentWork +): boolean { const { journal } = session - const live = journal.activeTurnId(fence) + const live = work.activeTurnId() if (live !== null) { return journal.stopMarks.personStopDecides(live) } @@ -102,7 +108,10 @@ export function recordStopEvent( return Promise.resolve(null) } const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) - const turnId = session.journal.activeTurnId(fence) + const turnId = structuredAgentSessionCurrentWork(session.journal, { + record: context.deps.store.getRecord(sessionId), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) + }).activeTurnId() return session.journal .appendStopEvent({ reason: ending.cause, ...(turnId !== null ? { turnId } : {}) }, fence) .then( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 2bd92b56a60d..e32b2d8fd11d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -48,7 +48,6 @@ import { } from './structured-agent-session-restart-resume-host' import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' import * as conversation from './structured-agent-session-host-delivery' -import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import { wireStructuredAgentSessionQueuedMessages } from './structured-agent-session-queued-wiring' import * as sessionLogger from './structured-agent-session-logger' export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' @@ -76,7 +75,7 @@ export class StructuredAgentSessionHost { this.sessions, () => this.now(), () => this.deps, - (sessionId) => this.queued.onJournalActivity(sessionId), + (sessionId, activity) => this.queued.onJournalActivity(sessionId, activity), (sessionId) => this.restartResume.onAgentStarted(sessionId), (sessionId) => this.backgroundTasks.publish(sessionId), (sessionId) => this.backgroundTasks.read(sessionId) @@ -144,13 +143,8 @@ export class StructuredAgentSessionHost { store: deps.store, sessions: this.sessions, flushLifecycle: (sessionId) => this.runtimeState.lifecycleBarrier(sessionId), - publishFence: (sessionId, session) => - this.subscribers.snapshot( - sessionId, - session.journal, - structuredAgentSessionConversationFence(deps.store, sessionId) - ), publishStatus: this.clientDelivery.publishStatusAndSettlement, + generationEnded: this.clientDelivery.publishGenerationEnded, serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), now: () => this.now(), runtimeState: this.runtimeState, @@ -190,6 +184,7 @@ export class StructuredAgentSessionHost { sessions: this.sessions, now: () => this.now(), publishStatus: this.clientDelivery.publishStatus, + generationEnded: this.clientDelivery.publishGenerationEnded, wakeDelivery: (sessionId: string) => this.conversationDelivery.loop.wake(sessionId), endExitedChild: this.eventRecovery.endExitedChildUnderSerialize } satisfies StructuredAgentSessionLifetimeContext @@ -334,6 +329,9 @@ export class StructuredAgentSessionHost { journalSnapshot = async (sessionId: string): Promise => (await this.lifetime.conversation(sessionId)).journal.snapshot() + /** The host's projection of a held chat's current work (`structuredAgentSessionCurrentWork`). */ + currentWork = this.clientDelivery.readCurrentWork + subscribe = (input: AgentSessionSubscribeInput) => this.backgroundTasks.subscribe(input) settleLateDispatch = (input: Parameters[1]) => diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts index 7de8ad28581f..b96313dbcb38 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts @@ -320,6 +320,7 @@ describe('the idle sweep with no child running (P2-22 ii)', () => { now: () => IDLE_MS + 1, isDisposed: () => false, deliveryActive: () => false, + getRecord: () => null, childWork: () => undefined, hasOpenDispatch: () => false, providerHoldsDispatch: () => false, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts index 1fd7fc937568..2e9616ad5a92 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts @@ -9,11 +9,15 @@ // Owed work is derived on every tick, never stored, so there is nothing to disagree with it. import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + structuredAgentSessionCurrentWork, + structuredAgentSessionHandsOver, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' export const STRUCTURED_AGENT_SESSION_IDLE_SWEEP_INTERVAL_MS = 5 * 60_000 export const STRUCTURED_AGENT_SESSION_IDLE_MS = 30 * 60_000 @@ -33,6 +37,8 @@ export type StructuredAgentSessionIdleSweepDeps = { hasOpenDispatch: (sessionId: string) => boolean /** The provider holds a send it took and has not answered; see `holdsDispatch`. */ providerHoldsDispatch: (sessionId: string) => boolean + /** The lease the current-work projection reads (`structuredAgentSessionCurrentWork`). */ + getRecord: (sessionId: string) => AgentSessionRecord | null /** Each of these runs inside the session's serialize and never takes it again. */ stopAgent: (sessionId: string) => Promise stopStartingAgent: (sessionId: string) => Promise @@ -42,35 +48,34 @@ export type StructuredAgentSessionIdleSweepDeps = { idleMs?: number } -/** The host's reads of what a running child still serves besides its journal. */ +/** The host's reads of what a running child still serves. */ export type StructuredAgentSessionChildWorkReads = { + /** The host's projection of current work: an ended generation's turn or prompt holds nothing. + * Null when the host holds no conversation, which has none. */ + currentWork: () => Pick< + StructuredAgentSessionCurrentWork, + 'activeTurnId' | 'hasActionablePrompt' + > | null childWork: () => readonly AgentChildWorkView[] | undefined hasOpenDispatch: () => boolean providerHoldsDispatch: () => boolean } /** Work the running child owes apart from queued sends: a turn, a child agent, an open - * orchestration dispatch, a send the provider holds, or a prompt nobody answered. Whatever owes - * any of it keeps its child; it builds no snapshot. */ + * orchestration dispatch, a send the provider holds, or a prompt nobody answered (a subagent's + * too, which the lead turn cannot see). Whatever owes any of it keeps its child; it builds no + * snapshot. */ export function structuredAgentSessionChildHasOpenWork( - journal: Pick, reads: StructuredAgentSessionChildWorkReads ): boolean { - if ( - journal.activeTurnId() !== null || + const work = reads.currentWork() + return ( + (work !== null && work.activeTurnId() !== null) || agentChildWorkLiveness(reads.childWork()) !== null || reads.hasOpenDispatch() || - reads.providerHoldsDispatch() - ) { - return true - } - let pendingPrompt = false - journal.visitItems((_itemId, _sequence, body) => { - // A subagent can raise a prompt the lead turn cannot see. - pendingPrompt ||= - (body.kind === 'approval' || body.kind === 'question') && body.resolution.state === 'pending' - }) - return pendingPrompt + reads.providerHoldsDispatch() || + (work !== null && work.hasActionablePrompt()) + ) } export class StructuredAgentSessionIdleSweep { @@ -151,7 +156,10 @@ export class StructuredAgentSessionIdleSweep { private queuedOrDelivering(sessionId: string, session: StructuredAgentSessionHostSession) { return ( this.deps.deliveryActive(sessionId) || - session.journal.submissions().some(isQueuedAgentJournalSubmission) + // Only a send this process may hand over: an earlier one's is bookkeeping, owed to no child. + session.journal + .submissions() + .some((submission) => structuredAgentSessionHandsOver(session.journal, submission)) ) } @@ -160,7 +168,12 @@ export class StructuredAgentSessionIdleSweep { private owesWork(sessionId: string, session: StructuredAgentSessionHostSession): boolean { return ( this.queuedOrDelivering(sessionId, session) || - structuredAgentSessionChildHasOpenWork(session.journal, { + structuredAgentSessionChildHasOpenWork({ + currentWork: () => + structuredAgentSessionCurrentWork(session.journal, { + record: this.deps.getRecord(sessionId), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) + }), childWork: () => this.deps.childWork(sessionId), hasOpenDispatch: () => this.deps.hasOpenDispatch(sessionId), providerHoldsDispatch: () => this.deps.providerHoldsDispatch(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts index e7ce667ed2b6..4aa4d0f646da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-journal-append-options.ts @@ -3,6 +3,9 @@ // Every such path calls this, so a row-level field added to the sink's options // reaches the durable row through one edit rather than through whichever spread // the next change remembers. Lifecycle batches carry producers per mutation. +// +// The sink carries only the provider's own observations, bound at its child's fence: that fence is +// the generation whose execution produced each item (`JournalItemRow.ownerFence`). import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import type { JournalItemAppendOptions } from '../agent-session-journal/journal-store-contracts' @@ -14,6 +17,7 @@ export function structuredAgentSessionJournalAppendOptions( ): JournalItemAppendOptions { return { fence, + ownerFence: fence, ...(options.observedAt === undefined ? {} : { observedAt: options.observedAt }), turnScope: options.turnScope, ...agentJournalLinkageFields(options) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts index d8f224b080ba..04858d8c996e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -4,7 +4,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { HOST_TEST_NOW as NOW, @@ -19,6 +18,7 @@ import { structuredQueueHold } from './structured-agent-session-queued-messages' import { APPROVAL, currentJournal, + currentWork, exitWhileSettlementFails, leaveUnfinishedWork, logEveryWrite, @@ -43,18 +43,12 @@ function everyRow(db: ReturnType['db']): unk .map((table) => [table.name, db.prepare(`SELECT * FROM main."${String(table.name)}"`).all()]) } -/** Whether the chat reads Working and holds a queue-if-active send, by its current fence. */ +/** Whether the chat reads Working and holds a queue-if-active send, by the host's projection. */ function holds(current: QueuedMessageTestRig): { working: boolean; queueHeld: boolean } { - const journal = currentJournal(current) - const record = current.store.getRecord(SESSION) - const fence = record?.lease.runtimeFence ?? 0 + const work = currentWork(current) return { - working: isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(fence), - journal.submissions(), - fence - ), - queueHeld: structuredQueueHold({ journal, record, fence }) !== null + working: work.working(), + queueHeld: structuredQueueHold({ record: current.store.getRecord(SESSION), work }) !== null } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts index c9465ee4aed8..ce90d5547cdc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -4,8 +4,12 @@ import { agentJournalSubmissionKey } from '../../../shared/agent-session-journal-item-key' import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' +import { + agentSessionWriteNoticeEnglish, + agentSessionWriteNoticeParts +} from '../../../shared/agent-session-refusal-notice' +import { agentSessionRefusalFailure } from '../../../shared/agent-session-write-failure' import { isSubagentGroupBlock } from '../../../shared/native-chat-types' -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' import { codexSubagentGroupIdentity } from '../../codex/codex-subagent-roster' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' @@ -24,6 +28,7 @@ import { isMainAgentWorking } from './structured-agent-session-turns-cancel' import { APPROVAL, currentJournal, + currentWork, exitChild, exitWhileSettlementFails, leaveUnfinishedWork, @@ -156,22 +161,16 @@ describe('a settlement that cannot commit never gates the user', () => { const journal = currentJournal(current) const record = current.store.getRecord(SESSION) const fence = record?.lease.runtimeFence ?? 0 + const work = currentWork(current) // Still saved as running: only its settlement would end it. expect(turnState(current)).toBe('running') - expect( - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(fence), - journal.submissions(), - fence - ) - ).toBe(false) - expect(structuredQueueHold({ journal, record, fence })).toBeNull() - expect(isMainAgentWorking({ journal, fence })).toBe(false) + expect(work.working()).toBe(false) + expect(structuredQueueHold({ record, work })).toBeNull() + expect(isMainAgentWorking({ journal, fence, currentWork: () => work })).toBe(false) const { items, submissions } = journal.snapshot() expect( - projectStructuredAgentSessionStatusState(items, submissions, fence, journal.itemFence).summary - .status + projectStructuredAgentSessionStatusState(items, submissions, fence, work.scope).summary.status ).not.toBe('working') expect((await sendText(current, 'go on').result).ok).toBe(true) @@ -196,7 +195,7 @@ describe('a settlement that cannot commit never gates the user', () => { expect(leftoverSettledAt(current)).toBeNull() expect(turnState(current)).toBe('running') // The successor's turn is the one that counts. - expect(session.journal.activeTurnId(session.child?.fence)).toBeNull() + expect(currentWork(current).activeTurnId()).toBeNull() database.db.exec('DROP TRIGGER reject_recovered') const marks = watchMarks(database.db) @@ -242,10 +241,23 @@ describe('a stale prompt', () => { expect(session.journal.itemBody(prompt.itemId)).toMatchObject({ resolution: { state: 'pending' } }) - expect(await answer()).toMatchObject({ + const refused = await answer() + expect(refused).toMatchObject({ ok: false, - refusal: { details: { reason: 'promptGone' } } + refusal: { details: { reason: 'promptOwnerEnded' } } }) + if (refused.ok) { + throw new Error('expected the answer refused') + } + // Plain words: what did not happen, that the agent stopped, and how to go on. + const words = agentSessionWriteNoticeEnglish( + agentSessionWriteNoticeParts(agentSessionRefusalFailure(refused.refusal), 'answer', { + agentName: 'Codex' + }) + ) + expect(words).toBe( + 'Your answer was not sent. Codex stopped while this response was in progress. You can continue in this conversation.' + ) expect(current.answerPrompt).not.toHaveBeenCalled() database.db.exec('DROP TRIGGER reject_recovered') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts index 0d72e8b43527..3986f7d4bbbe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts @@ -20,6 +20,10 @@ import { import type Database from '../../sqlite/sync-database' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + hostStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' import { HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, @@ -203,6 +207,18 @@ export function currentJournal(current: QueuedMessageTestRig) { return journal } +/** The host's projection of the chat's current work, as every host reader asks it. */ +export function currentWork(current: QueuedMessageTestRig): StructuredAgentSessionCurrentWork { + const work = hostStructuredAgentSessionCurrentWork( + { store: current.store, sessions: current.host.collaboratorsForTests().sessions }, + SESSION + ) + if (!work) { + throw new Error('expected the open conversation') + } + return work +} + export function turnState(current: QueuedMessageTestRig): string | undefined { return readAgentJournalTurn(currentJournal(current).itemBody(TURN_KEY) ?? undefined)?.state } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index 3fac8f4d8709..ea8f4aa56979 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -43,6 +43,7 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { MutationPlan } from './structured-agent-session-mutation-plans' +import type { StructuredAgentSessionEndedChild } from './structured-agent-session-host-types' import { runSettledAgentSessionMutation } from './structured-agent-session-operation-settlement' import { agentSessionOperationOutcomeUnknown, @@ -82,6 +83,8 @@ export type AgentSessionMutationRequest = { plan: MutationPlan /** Journal of the attached session, read after `prepareSession`; absent when this host holds none. */ journal: () => AgentSessionJournal | undefined + /** The last child this host saw end, which the current-work projection reads. */ + endedChild?: () => StructuredAgentSessionEndedChild | undefined /** Between the ledger's answer and the lease check, for a call that may first have to make the * session ready for itself. Answers with the refusal that ends the call, if any. */ prepareSession?: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 1f543cbf19f0..e1f6f40f0b00 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -72,6 +72,7 @@ export function mutateStructuredAgentSession( envelope, plan, journal: () => context.sessions.get(envelope.sessionId)?.journal, + endedChild: () => context.sessions.get(envelope.sessionId)?.lastEndedChild, prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), now: () => context.now() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts index 776937662eb0..c97edadd6717 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts @@ -4,6 +4,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { AgentSessionMutationRequest } from './structured-agent-session-mutation-admission' import type { AgentSessionTurnContext } from './structured-agent-session-turns' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' export function mutationTurnContext( request: AgentSessionMutationRequest, @@ -11,11 +12,19 @@ export function mutationTurnContext( record: AgentSessionRecord ): AgentSessionTurnContext { const fence = record.lease.runtimeFence - const persistedOptions = request.store.getRecord(request.envelope.sessionId)?.options + const { sessionId } = request.envelope + const persistedOptions = request.store.getRecord(sessionId)?.options return { - sessionId: request.envelope.sessionId, + sessionId, journal, fence, + currentWork: () => { + const ended = request.endedChild?.() + return structuredAgentSessionCurrentWork(journal, { + record: request.store.getRecord(sessionId), + ...(ended ? { ended } : {}) + }) + }, adapter: request.adapter, agents: request.agents, agent: record.provider, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts index b4281859aba0..a92a9d036beb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts @@ -14,6 +14,7 @@ import { type PendingPromptValidation } from './structured-agent-session-prompt-state' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' type CancelOutcome = TurnOutcome type PendingPrompt = Extract @@ -64,7 +65,7 @@ export async function cancelStructuredAgentSessionPrompt( } function raisedByLiveTurn(ctx: AgentSessionTurnContext, pending: PendingPrompt): boolean { - const live = ctx.journal.liveTurnScope() + const live = contextStructuredAgentSessionCurrentWork(ctx).turnScope() const raised = pending.item.turnScope return live.kind === 'turn' && raised?.kind === 'turn' && raised.turnItemId === live.turnItemId } @@ -102,7 +103,7 @@ async function dismissPrompt( } }, // A revision: the prompt keeps the turn it was raised in. - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) committed = true } @@ -128,7 +129,7 @@ async function dismissPrompt( surface: 'row' }) }, - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts index 6c432486a19b..a95f066bad03 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts @@ -42,6 +42,7 @@ import { QUEUED_RIG_CALLER, type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' let rig: QueuedMessageTestRig @@ -551,6 +552,12 @@ describe('where the host would refuse the send', () => { if (!journal) { throw new Error('expected the conversation open') } + // The clear's generation end wakes the drain, which would send the card; holding the session's + // lane keeps it waiting while the test reads the gate (`release` lets the drain run). + let release = (): void => undefined + const held = rig.host + .collaboratorsForTests() + .serialize(HOST_TEST_SESSION, () => new Promise((resolve) => (release = resolve))) const card = 'left-behind' await journal.queuedMessages.insert({ messageId: card, @@ -562,20 +569,33 @@ describe('where the host would refuse the send', () => { if (!record) { throw new Error('expected the record') } - return { card, journal, record, fence: record.lease.runtimeFence } + const done = async (): Promise => { + release() + await held + } + return { card, journal, record, fence: record.lease.runtimeFence, done } } it('a card queued after clear is sendable with the completed clear record still present', async () => { - const { card, journal, record, fence } = await cardAddedAfterClear() - const gate = structuredQueueSendGate(rig.store, HOST_TEST_SESSION) + const { card, journal, record, fence, done } = await cardAddedAfterClear() + const gate = structuredQueueSendGate( + { store: rig.store, sessions: rig.host.collaboratorsForTests().sessions }, + HOST_TEST_SESSION + ) expect(readQueuePublication(journal, gate).nextQueuedMessageId).toBe(card) const { conversationCommand: _cleared, ...unblocked } = record - const next = readQueuePublication(journal, () => ({ record: unblocked, fence })) + const next = readQueuePublication(journal, () => ({ + record: unblocked, + work: new StructuredAgentSessionCurrentWork(journal, fence) + })) expect(next.nextQueuedMessageId).toBe(card) + await done() + // Released, the drain the clear's end woke sends it. + await vi.waitFor(() => expect(journal.queuedMessages.get(card)?.state).toBe('dispatched')) }) it('a rewind whose outcome is unknown names no next card', async () => { - const { journal, record, fence } = await cardAddedAfterClear() + const { journal, record, fence, done } = await cardAddedAfterClear() const { conversationCommand: _cleared, ...unblocked } = record const rewind = { operationId: hostTestOperationId(), @@ -586,9 +606,13 @@ describe('where the host would refuse the send', () => { retained: [] } const next = (gateRecord: typeof record) => - readQueuePublication(journal, () => ({ record: gateRecord, fence })).nextQueuedMessageId + readQueuePublication(journal, () => ({ + record: gateRecord, + work: new StructuredAgentSessionCurrentWork(journal, fence) + })).nextQueuedMessageId expect(next({ ...unblocked, rewind })).toBeNull() expect(next({ ...unblocked, rewind: { ...rewind, phase: 'provider-succeeded' } })).toBeNull() expect(next({ ...unblocked, rewind: { ...rewind, phase: 'completed' } })).not.toBeNull() + await done() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts index 09ba1797b29a..713b8a19d234 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-gate.test.ts @@ -17,6 +17,7 @@ import { hostTestMessage, hostTestOperationId } from './structured-agent-session-host-test-data' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' let rig: QueuedMessageTestRig let host: QueuedMessageTestRig['host'] @@ -86,9 +87,12 @@ describe('the one queue gate', () => { }) // Read in place: the gate runs on every admission and drain step. const snapshot = vi.spyOn(journal, 'snapshot') - expect(structuredQueueHold({ journal, record: store.getRecord(SESSION), fence: 1 })).toBe( - 'prompt' - ) + expect( + structuredQueueHold({ + record: store.getRecord(SESSION), + work: new StructuredAgentSessionCurrentWork(journal, 1) + }) + ).toBe('prompt') expect(snapshot).not.toHaveBeenCalled() snapshot.mockRestore() await journal.appendItem( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 631bd5d8e466..48a380142842 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -19,11 +19,11 @@ import { queuedSendAnswer } from './structured-agent-session-queued-send-answer' import { structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { queuedMessagesPublishedBytesRefusal } from './structured-agent-session-queued-published-bytes' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { - isStructuredAgentSessionEndedGenerationWork, - isStructuredAgentSessionMainAgentWorking -} from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' import type { AgentSessionTurnContext } from './structured-agent-session-turns' import { QueuedMessageNotConsumableError } from '../agent-session-journal/journal-queued-messages' import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table' @@ -42,27 +42,6 @@ export function queuedMessageBodyIsTextOnly(body: AgentJournalMessageItem): bool return body.blocks.every((block) => block.type === 'text') } -/** Walks the reduced items in place: the gate runs on every admission and - * drain step, so it must not render a snapshot of the whole journal. Only a prompt a generation - * at `fence` raised: an ended one's can take no answer (`isStructuredAgentSessionEndedGenerationWork`). */ -export function pendingPromptExists( - journal: Pick, - fence: number -): boolean { - let pending = false - journal.visitItems((itemId, _sequence, body) => { - if ( - !pending && - (body.kind === 'approval' || body.kind === 'question') && - body.resolution.state === 'pending' && - !isStructuredAgentSessionEndedGenerationWork(journal.itemFence(itemId), fence) - ) { - pending = true - } - }) - return pending -} - /** Waiting, not held on its own, and not positioned behind a returned card or a * card the queue's pause holds: the queue never reorders. The admission rule * (§accept) and the drain's selection both read it. */ @@ -97,54 +76,37 @@ function oldestActionableQueuedMessage( */ export type StructuredQueueHold = 'blocked' | 'working' | 'prompt' -export function structuredQueueHold(input: { - journal: AgentSessionJournal +/** What the queue's gate reads: the record, and the host's projection of current work + * (`structuredAgentSessionCurrentWork`), so an ended generation's leftovers hold nothing. */ +export type StructuredQueueGateInput = { record: AgentSessionRecord | null - fence: number -}): StructuredQueueHold | null { + work: StructuredAgentSessionCurrentWork +} + +export function structuredQueueHold(input: StructuredQueueGateInput): StructuredQueueHold | null { // Whatever refuses any send refuses the queue too: an uncertain rewind or a source a // clear superseded. One rule, the immediate path's own. if (structuredAgentSessionSendBlock(input.record)) { return 'blocked' } - const { journal } = input // `prompt` outranks `working`: it is the one wait Send-now may not override, // so a prompt raised mid-turn must not read as merely `working`. - if (pendingPromptExists(journal, input.fence)) { + if (input.work.hasActionablePrompt()) { return 'prompt' } - if ( - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(input.fence), - journal.submissions(), - input.fence - ) - ) { - return 'working' - } - return null + return input.work.working() ? 'working' : null } /** The card the drain sends next, or null while anything holds the queue: the drain's own pick * through the one gate, so a client told this reads what the drain acts on. Live facts only; the * backlog is never a gate, so a lone draft drains. */ -export function nextStructuredQueuedMessage(input: { - journal: AgentSessionJournal - record: AgentSessionRecord | null - fence: number -}): QueuedMessageRow | null { +export function nextStructuredQueuedMessage( + input: StructuredQueueGateInput & { journal: AgentSessionJournal } +): QueuedMessageRow | null { const next = oldestActionableQueuedMessage(input.journal) - const { journal, fence } = input // The gate's cheap `working` first: publication asks on every streamed frame, and the gate's // prompt check walks the whole fold. - if ( - next === null || - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(fence), - journal.submissions(), - fence - ) - ) { + if (next === null || input.work.working()) { return null } return structuredQueueHold(input) === null ? next : null @@ -158,11 +120,9 @@ export function nextStructuredQueuedMessage(input: { * turn starting is what lifts the pause — Orca's own queue policy, a stated * deviation from held-head backlog counting. */ -export function shouldQueueStructuredAgentSessionSend(input: { - journal: AgentSessionJournal - record: AgentSessionRecord | null - fence: number -}): boolean { +export function shouldQueueStructuredAgentSessionSend( + input: StructuredQueueGateInput & { journal: AgentSessionJournal } +): boolean { const hold = structuredQueueHold(input) if (hold === 'blocked') { // The immediate path's own refusal (`structuredAgentSessionSendBlock`) @@ -185,7 +145,10 @@ export async function maybeQueueStructuredAgentSessionSend( context: { deps: { store: { getRecord: (sessionId: string) => AgentSessionRecord | null } } }, - ctx: Pick, + ctx: Pick< + AgentSessionTurnContext, + 'sessionId' | 'journal' | 'fence' | 'operationReceipt' | 'currentWork' + >, params: { envelope: { clientOperationId: string } body: AgentJournalMessageItem @@ -218,7 +181,7 @@ export async function maybeQueueStructuredAgentSessionSend( !shouldQueueStructuredAgentSessionSend({ journal: ctx.journal, record: context.deps.store.getRecord(ctx.sessionId), - fence: ctx.fence + work: contextStructuredAgentSessionCurrentWork(ctx) }) ) { return null @@ -266,6 +229,8 @@ export type QueuedMessageDrainDeps = { getRecord: (sessionId: string) => AgentSessionRecord | null serialize: (sessionId: string, task: () => Promise) => Promise conversationFence: (sessionId: string) => number + /** The host's projection of the chat's current work (`structuredAgentSessionCurrentWork`). */ + currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void logger: StructuredAgentSessionLogger @@ -273,9 +238,11 @@ export type QueuedMessageDrainDeps = { /** * The serialized drain. Woken by every journal commit (turn, submission, prompt, - * command and Stop settlements are all commits), by draft mutations, and by the - * conversation opening; each step re-derives everything and consumes at most one - * draft — the consumed submission then owes work, which gates the next. + * command and Stop settlements are all commits), by draft mutations, by the + * conversation opening, and by a generation ending, which may write nothing + * (`StructuredAgentSessionClientDelivery.publishGenerationEnded`); each step + * re-derives everything and consumes at most one draft — the consumed submission + * then owes work, which gates the next. */ export class StructuredAgentSessionQueuedMessageDrain { private readonly scheduled = new Set() @@ -296,17 +263,13 @@ export class StructuredAgentSessionQueuedMessageDrain { return } // Cheap pre-check so token streams do not pay a serialized step per delta. - // Skipping while working is safe: whatever ends the work is itself a commit - // that schedules again, and the step re-reads every gate from the fold. + // Skipping while working is safe: whatever ends the work is a commit or a generation's end, + // and each schedules again; the step re-reads every gate. try { if ( !journal.queuedMessages.settlementOwed() && (oldestActionableQueuedMessage(journal) === null || - isStructuredAgentSessionMainAgentWorking( - journal.activeTurnId(this.deps.conversationFence(sessionId)), - journal.submissions(), - this.deps.conversationFence(sessionId) - )) + this.deps.currentWork(sessionId)?.working() !== false) ) { return } @@ -352,7 +315,8 @@ export class StructuredAgentSessionQueuedMessageDrain { const fence = this.deps.conversationFence(sessionId) // Whatever clears a hold publishes or commits, which re-derives this step. const record = this.deps.getRecord(sessionId) - const next = nextStructuredQueuedMessage({ journal, record, fence }) + const work = this.deps.currentWork(sessionId) + const next = work ? nextStructuredQueuedMessage({ journal, record, work }) : null if (this.disposed || !next) { return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts index 4c8c2ab2e6c8..838f8fbbab52 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts @@ -5,6 +5,7 @@ // records only that an operation happened. No mutation returns draft text: // the published list is the one authority a client renders. +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' import { agentSessionOperationKey } from '../../../shared/agent-session-operation-ledger' import type { @@ -129,7 +130,10 @@ export function sendQueuedStructuredAgentMessage( // The one queue gate; Send-now's override set is exactly `working` (plus // FIFO order and the stored hold, which the consume below clears). const record = context.deps.store.getRecord(ctx.sessionId) - const hold = structuredQueueHold({ journal: ctx.journal, record, fence: ctx.fence }) + const hold = structuredQueueHold({ + record, + work: contextStructuredAgentSessionCurrentWork(ctx) + }) if (hold === 'blocked') { return structuredAgentSessionSendBlock(record) ?? invalid('This conversation cannot send.') } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index d48d9bf27c39..f3c97dd2d713 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -15,10 +15,14 @@ import { resumableQueuePause } from '../agent-session-journal/queued-message-pause' import { structuredQueuePauses } from './structured-agent-session-queued-pause' -import { nextStructuredQueuedMessage } from './structured-agent-session-queued-messages' -import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + nextStructuredQueuedMessage, + type StructuredQueueGateInput +} from './structured-agent-session-queued-messages' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkHost +} from './structured-agent-session-current-work' export type QueuePublication = { queuedMessages: AgentSessionQueuedMessage[] @@ -29,16 +33,20 @@ export type QueuePublication = { } /** What the drain's gate reads beyond the journal; resolved per read. */ -export type QueueSendGate = () => { record: AgentSessionRecord | null; fence: number } +export type QueueSendGate = (journal: AgentSessionJournal) => StructuredQueueGateInput export function structuredQueueSendGate( - store: Pick, + host: StructuredAgentSessionCurrentWorkHost, sessionId: string ): QueueSendGate { - return () => ({ - record: store.getRecord(sessionId), - fence: structuredAgentSessionConversationFence(store, sessionId) - }) + return (journal) => { + const record = host.store.getRecord(sessionId) + const ended = host.sessions.get(sessionId)?.lastEndedChild + return { + record, + work: structuredAgentSessionCurrentWork(journal, { record, ...(ended ? { ended } : {}) }) + } + } } /** Waiting and returned rows only. `paused` is a per-card hold (a failed conversion); a person's @@ -135,7 +143,8 @@ export function readQueuePublication( resumable && !queuePauseLiftOnItsWay(resumable, journal.submissions()) ? resumable : null // The test narrows the type. const queuePause = pause?.reason === 'stopped' ? { reason: pause.reason } : null - const nextQueuedMessageId = nextStructuredQueuedMessage({ journal, ...gate() })?.messageId ?? null + const nextQueuedMessageId = + nextStructuredQueuedMessage({ journal, ...gate(journal) })?.messageId ?? null const previous = publications.get(journal) if ( previous && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts index a351a73793e8..75f533340284 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-stop.ts @@ -14,6 +14,7 @@ import { } from '../agent-session-journal/queued-message-table' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' import { isMainAgentWorking } from './structured-agent-session-turns-cancel' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { structuredAgentSessionStopNamesTurnNotLive, structuredAgentSessionStoppedTurnId @@ -45,7 +46,11 @@ export async function runRecordedStop( const skipped = (error: unknown): void => report(ctx, 'event row', error) return stop(() => { try { - const turnId = structuredAgentSessionStoppedTurnId(ctx.journal, event.turnId) ?? undefined + const turnId = + structuredAgentSessionStoppedTurnId( + contextStructuredAgentSessionCurrentWork(ctx), + event.turnId + ) ?? undefined return ctx.journal .appendStopEvent({ ...event, ...(turnId ? { turnId } : {}) }, ctx.fence) .then(() => undefined, skipped) @@ -65,10 +70,10 @@ export async function runRecordedStop( * already over, as a late Stop from a phone can. */ export function stopReachesUnrecordedWork( - ctx: Pick, + ctx: Pick, namedTurnId: string | undefined ): 'unrecorded' | 'repeat' | 'late' { - const live = ctx.journal.activeTurnId(ctx.fence) + const live = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() // No turn published yet while the agent works: the named one may still be opening. if ( structuredAgentSessionStopNamesTurnNotLive(namedTurnId, live) && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index a5ddfeb64748..783da04803c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -8,6 +8,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { StructuredAgentSessionQueuedMessageDrain } from './structured-agent-session-queued-messages' import { deleteQueuedStructuredAgentMessage, @@ -31,6 +32,8 @@ export function wireStructuredAgentSessionQueuedMessages( serialize: (sessionId, task) => context().serialize(sessionId, task), conversationFence: (sessionId) => structuredAgentSessionConversationFence(context().deps.store, sessionId), + currentWork: (sessionId) => + hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) @@ -39,8 +42,11 @@ export function wireStructuredAgentSessionQueuedMessages( drain, /** Every journal publish: turn, submission, prompt, command and Stop * settlements are all commits, and each re-derives the drain's gates. */ - onJournalActivity: (sessionId: string) => { - sessions.touch(sessionId) + /** `activity`: false for a generation's end, which wakes the drain but is no activity. */ + onJournalActivity: (sessionId: string, activity = true) => { + if (activity) { + sessions.touch(sessionId) + } drain.schedule(sessionId) }, queuedMessageSend: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts index 09e48d3a3ef2..3027ffae3792 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-context-clear.test.ts @@ -75,11 +75,16 @@ describe('restart offers after context clear', () => { ...value(), providerHandleChain: [proved] } + // Teardown runs while the child at fence 2 still holds the lease; the restart finds it released. + const heldRecord = { + ...currentRecord, + lease: { ...currentRecord.lease, claimStatus: 'live' as const, runtimeFence: 2 } + } const turn = { ...turnItem('new-turn', 'running'), sequence: 6 } const offer = structuredAgentSessionWorkingAtStop({ sessionId: SESSION, session: { journal: journal([clear, turn]), child: { fence: 2 } }, - getRecord: () => currentRecord, + getRecord: () => heldRecord, childWork: () => undefined, trigger: 'quit', teardownId: TEARDOWN_CURRENT, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts index e5fa9576f6ae..d2fcee941a45 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume-test-harness.ts @@ -155,11 +155,14 @@ export function journal( items: AgentJournalRenderItem[], submissions: AgentJournalSubmission[] = [] ) { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test calls only snapshot(), submissions() and appendItem(); a real AgentSessionJournal needs an on-disk SQLite store. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the code under test calls only snapshot(), submissions(), appendItem() and the provenance reads; a real AgentSessionJournal needs an on-disk SQLite store. return { snapshot: () => ({ items, submissions, cursor: { epoch: EPOCH, sequence: items.length } }), submissions: () => submissions, - appendItem: async () => undefined + appendItem: async () => undefined, + // Created by the generation the record names: current while its lease is held. + itemFence: () => undefined, + wroteBeforeOpen: () => false } as never } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts index bc88fe1c4152..25b7cd85a6f0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts @@ -42,7 +42,17 @@ function markersAtTeardown( } ) { return [...input.sessions].flatMap(([sessionId, session]) => { - const recorded = structuredAgentSessionWorkingAtStop({ ...input, sessionId, session }) + // Teardown runs while this host's child still holds the lease. + const getRecord = (id: string) => { + const held = input.getRecord(id) + return held && { ...held, lease: { ...held.lease, claimStatus: 'live' as const } } + } + const recorded = structuredAgentSessionWorkingAtStop({ + ...input, + getRecord, + sessionId, + session + }) return recorded ? [recorded] : [] }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts new file mode 100644 index 000000000000..6ed2a1d09b77 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind-provenance.test.ts @@ -0,0 +1,154 @@ +// A rewind after a failed cleanup: the dead generation's turn and prompt are still saved as +// running and pending, and the rewind rebuilds the epoch at the live fence. Through the rewind's +// own row builders and a real SQLite journal, each carries the old fold's provenance, so the host +// projection never promotes that work to the live generation's. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody, + type AgentJournalItemIdentity, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { + mergeRetainedHostLifecycleRows, + retainedRowReplacement, + rewindProviderRows +} from './structured-rewind-retained-host-rows' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'ws-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: codexProviderHandle('thread-1') +} +const DEAD = 1 +const LIVE = 3 + +function codexItem(turnId: string, ordinal: number): AgentJournalItemIdentity { + return { provider: 'codex', threadId: 'thread-1', turnId, ordinal } +} +const TURN = codexItem('turn-dead', 0) +const PROMPT = codexItem('turn-dead', 1) +const REPLY = codexItem('turn-dead', 2) +const LATER = codexItem('turn-live', 0) +const PENDING = { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null +} +const REPLY_BODY: AgentJournalItemBody = { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: 'partial' }] +} + +const LATER_BODY: AgentJournalItemBody = { + kind: 'message', + role: 'assistant', + blocks: [{ type: 'text', text: 'mine' }] +} + +let root: string +const journals = createTrackedJournalOpener() + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-rewind-provenance-')) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +/** The dead generation's work as its cleanup left it: nothing settled. */ +async function leftByDeadGeneration(): Promise { + const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) + const scope = { fence: DEAD, ownerFence: DEAD, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + await journal.appendItem(TURN, { kind: 'turn', turnId: 'turn-dead', state: 'running' }, scope) + const inTurn = { + ...scope, + turnScope: { kind: 'turn' as const, turnItemId: agentJournalItemKey(TURN) } + } + await journal.appendItem( + PROMPT, + { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: PENDING + }, + inTurn + ) + await journal.appendItem(REPLY, REPLY_BODY, inTurn) + return journal +} + +/** The rewind's rebuild, as `rewindStructuredAgentSession` and its recovery write it. */ +async function rewindAtLiveFence(journal: AgentSessionJournal): Promise { + const retained = journal.snapshot().items.map(({ itemId, observedAt, turnScope }) => { + const ownerFence = journal.itemFence(itemId) + return { + itemId, + body: journal.itemBody(itemId)!, + observedAt, + ...(turnScope ? { turnScope } : {}), + ...(ownerFence === undefined ? {} : { ownerFence }) + } + }) + // The live agent's history names the dead turn's prompt and reply it still holds, and one of + // its own. + const provider = rewindProviderRows( + [ + { identity: PROMPT, body: journal.itemBody(agentJournalItemKey(PROMPT))! }, + { identity: REPLY, body: REPLY_BODY }, + { identity: LATER, body: LATER_BODY } + ], + 2_000, + LIVE + ) + const merged = mergeRetainedHostLifecycleRows(retained, provider) + await journal.replaceEpochItems('handle_forked', LIVE, merged.map(retainedRowReplacement)) +} + +describe('a rewind after a failed cleanup', () => { + it("does not promote the dead generation's turn, prompt or reply to the live one", async () => { + const journal = await leftByDeadGeneration() + const before = journal.epoch + + await rewindAtLiveFence(journal) + + expect(journal.epoch).not.toBe(before) + const work = new StructuredAgentSessionCurrentWork(journal, LIVE) + // Rebuilt at the live fence, still the dead generation's work. + expect(journal.itemFence(agentJournalItemKey(TURN))).toBe(DEAD) + expect(journal.itemFence(agentJournalItemKey(PROMPT))).toBe(DEAD) + expect(journal.itemFence(agentJournalItemKey(REPLY))).toBe(DEAD) + expect(work.activeTurnId()).toBeNull() + expect(work.actionablePromptIds()).toEqual([]) + expect(work.working()).toBe(false) + // What the live agent itself reported is its own. + expect(work.isCurrentItem(agentJournalItemKey(LATER))).toBe(true) + }) + + it('leaves a generation that is still live holding its own work', async () => { + const journal = await leftByDeadGeneration() + + await rewindAtLiveFence(journal) + + const work = new StructuredAgentSessionCurrentWork(journal, DEAD) + expect(work.activeTurnId()).toBe('turn-dead') + expect(work.actionablePromptIds()).toEqual([agentJournalItemKey(PROMPT)]) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index bd819d09f2ca..7886149e0356 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -2,7 +2,6 @@ import { agentSessionProviderHandleChainHead } from '../../../shared/agent-sessi import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import { - agentJournalItemKey, agentJournalSubmissionKey, parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' @@ -23,7 +22,10 @@ import { conversationCommandBlocked } from './structured-conversation-command-ad import { rewindRefusal } from './structured-rewind-refusal' import { renameRewindTurnOpener } from './structured-rewind-journal-body' import { persistRewindRecord, recoverStructuredRewind } from './structured-rewind-recovery' -import { mergeRetainedHostLifecycleRows } from './structured-rewind-retained-host-rows' +import { + mergeRetainedHostLifecycleRows, + rewindProviderRows +} from './structured-rewind-retained-host-rows' export async function rewindStructuredAgentSession( context: StructuredAgentSessionMutationContext, @@ -58,6 +60,7 @@ export async function rewindStructuredAgentSession( : { ok: true } }), journal: () => context.sessions.get(sessionId)?.journal, + endedChild: () => context.sessions.get(sessionId)?.lastEndedChild, publish: (journal) => context.publish(sessionId, journal), now: context.now, plan: { @@ -165,11 +168,13 @@ export async function rewindStructuredAgentSession( if (!body) { throw new Error('agent_session_rewind:missing-retained-item') } + const ownerFence = ctx.journal.itemFence(itemId) return { itemId: providerKey(itemId), body: renameRewindTurnOpener(body, providerKey), observedAt, ...(turnScope ? { turnScope } : {}), + ...(ownerFence === undefined ? {} : { ownerFence }), ...agentJournalLinkageFields(linkage) } }) @@ -204,11 +209,7 @@ export async function rewindStructuredAgentSession( onPrepared: async (items) => { const retained = mergeRetainedHostLifecycleRows( prepared.retained, - items.map(({ identity, body }) => ({ - itemId: agentJournalItemKey(identity), - body, - observedAt: ctx.now() - })) + rewindProviderRows(items, ctx.now(), ctx.fence) ) if ( retained.length > 10_000 || @@ -241,11 +242,7 @@ export async function rewindStructuredAgentSession( const confirmed = provider.items ? mergeRetainedHostLifecycleRows( prepared.retained, - provider.items.map(({ identity, body }) => ({ - itemId: agentJournalItemKey(identity), - body, - observedAt: ctx.now() - })) + rewindProviderRows(provider.items, ctx.now(), ctx.fence) ) : prepared.retained if ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts index a9815b953f8f..b89d36ffe3b4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.ts @@ -7,8 +7,12 @@ import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-qu import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { isStructuredAgentSessionCommandTurnId } from './structured-agent-session-command-turn' -/** What a settlement read needs of a journal. */ -type SendSettlementJournal = Pick +/** What a settlement read needs: the journal's sends and position, and the live generation's + * running turn as the host's projection reads it (`structuredAgentSessionCurrentWork`), so a send + * never reads as waiting behind a command an ended generation left. */ +export type SendSettlementJournal = Pick & { + activeTurnId: () => string | null +} type SettledSend = { cursor: AgentJournalCursor diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts index 5fe3bc74da53..7f97faa864f4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-shown-work.ts @@ -15,7 +15,7 @@ import { type StructuredAgentSessionAgentStatus } from '../../../shared/structured-agent-session-agent-status' import { projectStructuredAgentSessionStatus } from '../../../shared/structured-agent-session-projection' -import type { StructuredAgentSessionItemFence } from '../../../shared/structured-agent-session-main-agent-working' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' /** The full row the sidebar's fold would show, for callers that need the lead's own state beside * the working answer — the teardown snapshot records both from this one computation. */ @@ -27,14 +27,15 @@ export function structuredAgentSessionShownStatus( childWork: readonly AgentChildWorkView[] | undefined, /** The session's lease fence, as the status feed passes it: a send from an older one is not work. */ fence: number | undefined, - /** The host's item fences, as the status feed passes them: an ended generation's turn is not work. */ - itemFence?: StructuredAgentSessionItemFence + /** The host's projection of current work (`structuredAgentSessionCurrentWork`), as the status + * feed passes it: an ended generation's turn is not work. */ + scope?: StructuredAgentSessionWorkScope ): StructuredAgentSessionAgentStatus { const status = projectStructuredAgentSessionStatus( journal.items, journal.submissions, fence, - itemFence + scope ) return structuredAgentSessionAgentStatus({ status, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts index ae09e13d8782..02a4e11d91d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.test.ts @@ -126,7 +126,6 @@ const conversation = ( child?: StructuredAgentSessionProviderChild | null items?: Row[] submissions?: ReturnType[] - activeTurnId?: string newestTurnEndedAt?: number } = {} ) => { @@ -142,7 +141,6 @@ const conversation = ( visit(item.itemId, index, item.body, item.agentId ? { agentId: item.agentId } : {}) ), itemFence: (itemId: string) => items.find((item) => item.itemId === itemId)?.fence, - activeTurnId: () => input.activeTurnId ?? null, newestTurn: (): AgentJournalTurnLifecycle | null => input.newestTurnEndedAt === undefined ? null @@ -197,6 +195,7 @@ describe('retireSignedOutStructuredAgentSessionChild', () => { invocation: { invocationId: 'spawn-subagent-1', generation: 1 } } const idle: StructuredAgentSessionChildWorkReads = { + currentWork: () => ({ activeTurnId: () => null, hasActionablePrompt: () => false }), childWork: () => [], hasOpenDispatch: () => false, providerHoldsDispatch: () => false @@ -223,7 +222,9 @@ describe('retireSignedOutStructuredAgentSessionChild', () => { it('keeps it while it owes work the idle sweep also protects', async () => { // A pending prompt is the shared check's too; the idle sweep's tests cover that case. for (const kept of [ - await retire({ ...signedOut, activeTurnId: 'turn-1' }), + await retire(signedOut, { + currentWork: () => ({ activeTurnId: () => 'turn-1', hasActionablePrompt: () => false }) + }), await retire(signedOut, { childWork: () => [liveChildWork] }), await retire(signedOut, { hasOpenDispatch: () => true }), await retire(signedOut, { providerHoldsDispatch: () => true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts index 40ea59a69eb8..a4315eeb1ca0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-signed-out-child.ts @@ -16,10 +16,7 @@ import type { StructuredAgentSessionLogger } from './structured-agent-session-lo /** What the check reads of a conversation; every read skips building a snapshot. */ type SignedOutReading = Pick & { - journal: Pick< - AgentSessionJournal, - 'visitItems' | 'visitItemsWithLinkage' | 'itemFence' | 'activeTurnId' | 'newestTurn' - > & { + journal: Pick & { submissions(): readonly Pick[] } } @@ -94,7 +91,7 @@ export async function retireSignedOutStructuredAgentSessionChild( if ( !session || !structuredAgentSessionChildReportedSignedOut(session, deps.startUnavailable?.()) || - structuredAgentSessionChildHasOpenWork(session.journal, deps.work) || + structuredAgentSessionChildHasOpenWork(deps.work) || owesWakeUp(session.journal, deps.work.childWork()) ) { return diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts index 9206f95fde0b..6674d5345a2a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -1,10 +1,10 @@ import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' -import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { structuredAgentSessionStartFailureRowIdentity } from '../../../shared/structured-agent-session-start-failure-row-key' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionStartFailureWords } from './structured-agent-session-failure-text' +import { structuredAgentSessionHandsOver } from './structured-agent-session-current-work' /** A start that failed, keyed like its row, in the words `structuredAgentSessionStartFailure` gave. */ export type StructuredAgentSessionStartFailure = StructuredAgentSessionStartFailureWords & { @@ -55,22 +55,27 @@ export async function recordStructuredAgentSessionStartFailure( return } const startKey = failure.startKey ?? oldest.clientMessageId - await session.journal.appendLifecycleBatch({ + const { journal } = session + await journal.appendLifecycleBatch({ settlementId: `start-failure:${startKey}`, fence: session.fence, recovered: true, mutations: [structuredAgentSessionStartFailureRow(startKey, failure)], - rejectsQueued: { reason: failure.reason, rejection: failure.rejection } + rejectsQueued: { reason: failure.reason, rejection: failure.rejection }, + // The start was for this process's sends; an earlier process's is the startup share's to keep. + rejectsQueuedOnly: (submission) => structuredAgentSessionHandsOver(journal, submission) }) } +/** The oldest send this process may hand over (`structuredAgentSessionHandsOver`): an earlier host + * process's unhanded send is never handed over, whether or not its bookkeeping landed. */ export function oldestQueuedSubmission( session: Pick ): ReturnType[number] | undefined { let oldest: ReturnType for (const submission of session.journal.submissions()) { if ( - isQueuedAgentJournalSubmission(submission) && + structuredAgentSessionHandsOver(session.journal, submission) && (oldest === undefined || (submission.acceptedSequence ?? 0) < (oldest.acceptedSequence ?? 0)) ) { oldest = submission diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index d192a0d01b14..36a4ac14a219 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -65,7 +65,7 @@ function contextFor(session: StructuredAgentSessionChildExitSession) { }, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index 2a26af994dcb..5aeb9f3755d2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -19,7 +19,10 @@ import type { import type { AgentChildWorkEvidence } from '../../../shared/agent-status-child-work-evidence' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' +import type { + StructuredAgentSessionEndedChild, + StructuredAgentSessionProviderChild +} from './structured-agent-session-host-types' import { structuredAgentSessionStatusSummary } from './structured-agent-session-status-summary' import { structuredStatusChildWork } from './structured-agent-session-status-child-work' import { @@ -46,6 +49,8 @@ type StatusFeedSession = { journal: AgentSessionJournal params: { location: AgentSessionRecord['location']; provider: AgentSessionRecord['provider'] } child?: Pick | null + /** What the current-work projection reads of the last child this host saw end. */ + lastEndedChild?: Pick restartResume?: AgentSessionStatusSummary['restartResume'] } @@ -208,8 +213,11 @@ export class StructuredAgentSessionStatusFeed { sessionId: string, journal?: AgentSessionJournal ): StructuredAgentSessionJournalProjection | null { - const source = journal ?? this.deps.sessions.get(sessionId)?.journal - return source ? this.projections.read(source, this.deps.getRecord(sessionId)) : null + const session = this.deps.sessions.get(sessionId) + const source = journal ?? session?.journal + return source + ? this.projections.read(source, this.deps.getRecord(sessionId), session?.lastEndedChild) + : null } /** Re-projects one session after its journal changed; equal projections are not re-sent. */ @@ -220,7 +228,7 @@ export class StructuredAgentSessionStatusFeed { } const source = journal ?? session.journal const record = this.deps.getRecord(sessionId) - const projection = this.projections.read(source, record) + const projection = this.projections.read(source, record, session.lastEndedChild) this.retireSettledChildrenOnNewTurn(sessionId, session, projection.acceptedSendKey) const summary = structuredAgentSessionStatusSummary({ sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index e672be5a673e..1053b4c993c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -7,6 +7,10 @@ import { isStructuredAgentSessionCommandEntry } from '../../../shared/structured import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { projectStructuredAgentSessionStatusState } from '../../../shared/structured-agent-session-projection' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionWorkEvidence +} from './structured-agent-session-current-work' import { newestAcceptedSendKey } from './structured-agent-session-status-child-work' import { structuredAgentSessionStopping } from './structured-agent-session-stopping' @@ -18,6 +22,9 @@ export type StructuredAgentSessionJournalProjection = { epoch: string sequence: number fence: number | undefined + /** The live generation's fence (`structuredAgentSessionLiveFence`): an exit this host saw, or a + * release, changes what is current without a journal row. */ + liveFence: number | null /** The Stop marks' settle revision: a settle edge writes no row, so it is a key of its own. */ stopRevision: number state: StructuredAgentSessionStatusState @@ -37,11 +44,13 @@ export class StructuredAgentSessionJournalProjections { read( journal: AgentSessionJournal, - record: AgentSessionRecord | null + record: AgentSessionRecord | null, + ended?: StructuredAgentSessionWorkEvidence['ended'] ): StructuredAgentSessionJournalProjection { const cursor = journal.cursor() // The conversation's fence, which a child's end moves: its unanswered sends stop counting. const fence = record?.lease.runtimeFence + const work = structuredAgentSessionCurrentWork(journal, { record, ...(ended ? { ended } : {}) }) const stopRevision = journal.stopMarks.revision() let projection = this.byJournal.get(journal) if ( @@ -49,6 +58,7 @@ export class StructuredAgentSessionJournalProjections { projection.epoch !== cursor.epoch || projection.sequence !== cursor.sequence || projection.fence !== fence || + projection.liveFence !== work.liveFence || projection.stopRevision !== stopRevision ) { // A journalled submission bumps `lastSequence`, so the send-time working @@ -81,16 +91,22 @@ export class StructuredAgentSessionJournalProjections { firstInputSubmissionKey, submissionCount: snapshot.submissions.length, fence, + liveFence: work.liveFence, stopRevision, - // The host knows who wrote each item: an ended generation's work is not Working. + // The host's projection of current work: an ended generation's is not Working. state: projectStructuredAgentSessionStatusState( snapshot.items, snapshot.submissions, fence, - journal.itemFence + work.scope ), acceptedSendKey: newestAcceptedSendKey(cursor.epoch, snapshot.submissions), - stopping: structuredAgentSessionStopping(journal, snapshot.items, snapshot.submissions) + stopping: structuredAgentSessionStopping( + journal, + snapshot.items, + snapshot.submissions, + work.scope + ) } this.byJournal.set(journal, projection) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts index 03e9108f2e5d..2a91867ae446 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts @@ -443,7 +443,8 @@ describe('a rewind that restates a turnless Stop', () => { await journal().replaceEpochItems('handle_forked', 1, [ { identity: stoppedTurn, - body: { ...ended, completedAt: Date.now() + 1, outcome: 'cancellation' } + body: { ...ended, completedAt: Date.now() + 1, outcome: 'cancellation' }, + ownerFence: 1 } ]) const mail = rig.send('mail after the rewind') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts index f2001d8da1b4..a6dc5d103b06 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-wind-down.ts @@ -11,6 +11,7 @@ import { type AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { isMainAgentWorking } from './structured-agent-session-turns-cancel' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { AgentSessionTurnContext } from './structured-agent-session-turns' import type { JournalStopFailedOn } from '../agent-session-journal/queued-message-pause' import { structuredAgentSessionFailedStopMark } from './structured-agent-session-stopping' @@ -38,7 +39,7 @@ export type StructuredAgentSessionStopWindDown = { * keeps the child on record, and the next operation that reaches the agent joins that close. */ export async function endStoppedStructuredAgentSession( - ctx: Pick, + ctx: Pick, windDown: StructuredAgentSessionStopWindDown, stopChild: () => Promise, onError: (error: unknown) => void @@ -51,7 +52,10 @@ export async function endStoppedStructuredAgentSession( await stopChild() } catch (error) { onError(error) - failedOn = structuredAgentSessionFailedStopMark(ctx.journal, ctx.fence) + failedOn = structuredAgentSessionFailedStopMark( + contextStructuredAgentSessionCurrentWork(ctx), + ctx.journal + ) await reviseStopNoteUnconfirmed(ctx, windDown.stopNote).catch(onError) } finally { windDown.settled?.(failedOn) @@ -61,7 +65,7 @@ export async function endStoppedStructuredAgentSession( /** While the work it stopped runs on, the Stop's own note, if it wrote one, says what a lost * interrupt's says. Work that ended took the Stop, whatever became of the child. */ async function reviseStopNoteUnconfirmed( - ctx: Pick, + ctx: Pick, identity: AgentJournalItemIdentity ): Promise { if (!isMainAgentWorking(ctx)) { @@ -85,7 +89,7 @@ async function reviseStopNoteUnconfirmed( // Stop of that turn writes it: its proven end finds it there. A row keeps the scope it was created // with, so a note a Stop wrote before the turn showed is re-keyed, in one batch. Known limit: with // no turn open yet, the note keeps its key and scope, and no turn's end revises it. - const running = ctx.journal.activeTurnId(ctx.fence) + const running = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() const turnScope = running !== null ? structuredAgentSessionNamedTurnScope(ctx.journal, running) : null const onTurn = running !== null ? structuredAgentSessionStopNoteIdentity(running) : identity diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts index 0cf15693b3ce..c491814f542b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.ts @@ -10,6 +10,8 @@ import type { } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-live-turn' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { stopIsAPersons } from '../agent-session-journal/journal-stop-turn-end' import type { JournalStopFailedOn } from '../agent-session-journal/queued-message-pause' @@ -19,7 +21,9 @@ import type { JournalStopFailedOn } from '../agent-session-journal/queued-messag export function structuredAgentSessionStopping( journal: Pick, items: readonly AgentJournalRenderItem[], - submissions: readonly AgentJournalSubmission[] = [] + submissions: readonly AgentJournalSubmission[] = [], + /** The host's projection of current work: a Stop bound to an ended generation's turn is over. */ + scope?: StructuredAgentSessionWorkScope ): boolean { const stop = journal.stopMarks.latest() if (stop === null || !stopIsAPersons(stop.event.reason)) { @@ -33,7 +37,10 @@ export function structuredAgentSessionStopping( stop.event.turnId ?? stop.settle?.turnId ?? (failed && 'turnId' in failed ? failed.turnId : undefined) - const live = activeStructuredAgentSessionTurnId(items) + const live = activeStructuredAgentSessionTurnId( + items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ) if (bound !== undefined) { return bound === live } @@ -64,10 +71,10 @@ function firstSendHandedOverAfter( /** Where a Stop that failed leaves "Stopping…" (display only): the turn running now, or, with * none, the first that opens after this position. */ export function structuredAgentSessionFailedStopMark( - journal: Pick, - fence: number + work: Pick, + journal: Pick ): JournalStopFailedOn { - const live = journal.activeTurnId(fence) + const live = work.activeTurnId() return live !== null ? { turnId: live } : { openedAfter: journal.cursor().sequence } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index dcd3e07c4f2a..683d0667c7a8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -22,6 +22,7 @@ import { emptyAgentSessionBatch } from './agent-session-empty-batch' import { readAgentSessionHydrationPage } from './agent-session-history-page' import { rememberSessionActivity } from './structured-agent-session-activity-retention' import { refreshDerivedStopNotes } from './agent-session-stop-note-refresh' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' export type AgentSessionSubscriberEmit = (event: AgentSessionSubscribeEvent) => void export type AgentSessionSubscribeInput = { @@ -43,6 +44,9 @@ export type Subscriber = { queuePublication?: QueuePublication /** Fingerprint of the background-task roster last SENT. */ backgroundTasks?: string + /** The live generation the last frame's current-work fields were read under; a change with no + * row still owes this subscriber a frame. */ + workLiveFence?: number | null } export type AgentSessionSubscribersHooks = { @@ -53,6 +57,12 @@ export type AgentSessionSubscribersHooks = { readBackgroundTasks?: SubscriberFieldHooks['readBackgroundTasks'] /** Fires after publications that can change journal content. */ onJournalPublished?: (sessionId: string, journal: AgentSessionJournal) => void + /** The host's projection of current work (`structuredAgentSessionCurrentWork`), which every + * page and frame states as its turn and prompt fields. */ + readCurrentWork?: ( + sessionId: string, + journal: AgentSessionJournal + ) => StructuredAgentSessionCurrentWork | undefined now?: () => number } @@ -103,7 +113,9 @@ export class AgentSessionSubscribers { hostNow ) } else { - const page = readAgentSessionHydrationPage(input.journal, input.fence) + const work = this.hooks.readCurrentWork?.(input.sessionId, input.journal) + const page = readAgentSessionHydrationPage(input.journal, input.fence, work) + subscriber.workLiveFence = work?.liveFence this.emit(subscriber, { type: 'snapshot', sessionId: input.sessionId, @@ -144,20 +156,31 @@ export class AgentSessionSubscribers { this.activityBySession.delete(sessionId) } } + this.deliverFrames(sessionId, journal, activity) + if (activity === undefined) { + this.hooks.onJournalPublished?.(sessionId, journal) + } + } + + /** Each subscriber's frame for what changed, with none of a journal write's other edges. */ + deliverFrames( + sessionId: string, + journal: AgentSessionJournal, + activity?: AgentSessionTurnActivity | null + ): void { const hostNow = this.now() for (const subscriber of this.subscribers(sessionId)) { this.deliver(subscriber, journal, hostNow, false, activity) } - if (activity === undefined) { - this.hooks.onJournalPublished?.(sessionId, journal) - } } /** Every subscriber back to a bounded tail page. */ snapshot(sessionId: string, journal: AgentSessionJournal, fence: number): void { - const page = readAgentSessionHydrationPage(journal, fence) + const work = this.hooks.readCurrentWork?.(sessionId, journal) + const page = readAgentSessionHydrationPage(journal, fence, work) const hostNow = this.now() for (const subscriber of this.subscribers(sessionId)) { + subscriber.workLiveFence = work?.liveFence this.emit(subscriber, { type: 'snapshot', sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts index 4af804263aec..1310663c9989 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-thread-goal.ts @@ -13,6 +13,7 @@ import type { } from '../../../shared/agent-session-wire' import type { MutationPlan } from './structured-agent-session-mutation-plans' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' function refused( reason: AgentSessionRefusalReason<'agent_session_operation_invalid'>, @@ -70,7 +71,7 @@ export async function performThreadGoalChange( sentAs: 'goal' }, // Accepting a goal is delivering it, so it joins whatever turn runs now. - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) } const withdrawObjective = async (): Promise => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts index 18805b0b7404..5b7409e85cfa 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-transition.ts @@ -118,6 +118,7 @@ function transitionAppend( batch: { settlementId: step.settlementId, fence, + ownerFence: fence, resolve: () => step.resolve(journal) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts index 0f86c053ad66..4a954f7e8fa1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turn-stop-notes.ts @@ -5,15 +5,16 @@ import type { AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' export const STOP_NOTE_CANCELLATION_REQUESTED = 'Cancellation requested.' -/** The turn a Stop is about: the one it named, else the one running when it is read. */ +/** The turn a Stop is about: the one it named, else the live generation's running turn. */ export function structuredAgentSessionStoppedTurnId( - journal: Pick, + work: Pick, namedTurnId: string | undefined ): string | null { - return namedTurnId ?? journal.activeTurnId() + return namedTurnId ?? work.activeTurnId() } /** A Stop names a turn the journal does not show running, as a phone does once that turn ended. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index 568922e9cd4f..df94fa7a34f3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -24,7 +24,7 @@ import { structuredAgentSessionStopNamesTurnNotLive, structuredAgentSessionStoppedTurnId } from './structured-agent-session-turn-stop-notes' -import { isStructuredAgentSessionMainAgentWorking } from '../../../shared/structured-agent-session-main-agent-working' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { StructuredAgentSessionStopWindDown } from './structured-agent-session-stop-wind-down' import type { JournalStopFailedOn } from '../agent-session-journal/queued-message-pause' import { structuredAgentSessionFailedStopMark } from './structured-agent-session-stopping' @@ -32,16 +32,12 @@ import { sendStopCanTakeBack } from './structured-agent-session-unopened-send-wi import { settleTakenStop } from './structured-agent-session-taken-stop-settle' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' -/** Whether the fold reads working. Every write has landed by its call's return, so a Stop reads it - * without waiting on the write queue. */ +/** Whether the host's projection reads working (`structuredAgentSessionCurrentWork`). Every write + * has landed by its call's return, so a Stop reads it without waiting on the write queue. */ export function isMainAgentWorking( - ctx: Pick + ctx: Pick ): boolean { - return isStructuredAgentSessionMainAgentWorking( - ctx.journal.activeTurnId(ctx.fence), - ctx.journal.submissions(), - ctx.fence - ) + return contextStructuredAgentSessionCurrentWork(ctx).working() } /** @@ -50,14 +46,15 @@ export function isMainAgentWorking( * its exit ends its turn, and the host's own exit handling waits behind this step. */ function stillRunsStoppedTurn( - ctx: Pick, + ctx: Pick, stoppedTurnId: string | null ): boolean { - if (!isMainAgentWorking(ctx)) { + const work = contextStructuredAgentSessionCurrentWork(ctx) + if (!work.working()) { return false } // Working with no turn open after the Stop's turn is a later send whose turn has not opened. - return stoppedTurnId === null || ctx.journal.activeTurnId(ctx.fence) === stoppedTurnId + return stoppedTurnId === null || work.activeTurnId() === stoppedTurnId } /** Whether the Stop took back every send it found in flight, none of them having run @@ -162,17 +159,20 @@ async function cancelAndNote( kind: 'status', text: STOP_NOTE_CANCELLATION_REQUESTED } + // Every read of what runs goes through the host's projection: an ended generation's turn is no + // Stop's target, whether or not its cleanup landed. + const work = () => contextStructuredAgentSessionCurrentWork(ctx) // The turn the Stop names, read before the cancel settles it: the note reports on that turn. const turnScope = (input.turnId !== undefined && !input.scope ? structuredAgentSessionNamedTurnScope(ctx.journal, input.turnId) - : null) ?? ctx.journal.liveTurnScope() + : null) ?? work().turnScope() // Only the provider's end or the child's ends a command. A command the provider has not opened a // turn for, would not interrupt, or was already asked to stop, ends with its child; that child's // dead-generation settlement writes the command's verdict. - const liveTurnId = ctx.journal.activeTurnId(ctx.fence) + const liveTurnId = work().activeTurnId() // Read with the live turn, before the cancel settles it: the note is keyed by this turn. - const stoppedTurnId = structuredAgentSessionStoppedTurnId(ctx.journal, input.turnId) + const stoppedTurnId = structuredAgentSessionStoppedTurnId(work(), input.turnId) const namesTurnNotLive = structuredAgentSessionStopNamesTurnNotLive(input.turnId, liveTurnId) const runningCommand = input.stopChild !== undefined && @@ -214,7 +214,7 @@ async function cancelAndNote( ...(input.turnId !== undefined ? { turnId: input.turnId } : {}), fence: ctx.fence, // The journal is what the client read to name a turn, so it is what judges the request. - resolveLiveTurnId: () => ctx.journal.activeTurnId(ctx.fence), + resolveLiveTurnId: () => work().activeTurnId(), ...(dispatchStatus ? { dispatchStatus } : {}), ...(input.prompt ? { prompt: { itemId: input.prompt.itemId } } : {}) }) @@ -311,7 +311,7 @@ async function cancelAndNote( binding.turnId = stoppedTurn ?? stoppedTurnId ?? undefined } else if (interruptFailed) { // A Stop that failed reads "Stopping…" through the turn it could not stop; its end stays its own. - binding.failedOn = structuredAgentSessionFailedStopMark(ctx.journal, ctx.fence) + binding.failedOn = structuredAgentSessionFailedStopMark(work(), ctx.journal) } if (taken === true && stoppedTurn !== undefined && !binding.closedByWindDown && !input.scope) { const opened = await settleTakenStop(ctx, stoppedTurn) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts index 301022c42c6a..5eba02a11c62 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-prompt.ts @@ -24,7 +24,7 @@ import { AgentSessionPromptUnavailableError } from './structured-agent-session-adapter' import { settledPrompt, validatePendingPrompt } from './structured-agent-session-prompt-state' -import { isStructuredAgentSessionEndedGenerationWork } from '../../../shared/structured-agent-session-main-agent-working' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' export type AgentSessionPromptRequest = { @@ -98,9 +98,13 @@ export async function performPrompt( return held ? { ok: true, value: held } : validated } const { prompt } = validated - if (isStructuredAgentSessionEndedGenerationWork(ctx.journal.itemFence(input.itemId), ctx.fence)) { - // The agent that asked has ended: an answer would reach whichever agent runs now. - return invalid('promptGone', `Item ${input.itemId} was raised by an agent that has ended.`) + const work = contextStructuredAgentSessionCurrentWork(ctx) + if (!work.isCurrentItem(input.itemId)) { + // The agent that asked has ended: an answer would reach whichever agent runs now, or none. + return invalid( + 'promptOwnerEnded', + `Item ${input.itemId} was raised by an agent that has ended.` + ) } const choice = readPromptChoice(prompt, input) if (!choice) { @@ -137,7 +141,7 @@ export async function performPrompt( identity, { ...prompt, resolution }, // A revision: the prompt keeps the turn it was raised in. - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: work.turnScope() } ) } }) @@ -160,7 +164,7 @@ export async function performPrompt( surface: 'row' }) }, - { fence: ctx.fence, turnScope: ctx.journal.liveTurnScope() } + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } ) } const appended = committed.item diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 17ccba7938ef..48a88f5ed0df 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -24,6 +24,10 @@ import { } from '../../../shared/agent-session-wire' import { isAgentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { DISPATCH_DOUBT_PERSISTENCE_FAILED } from '../agent-session-journal/journal-dispatch-doubt-reasons' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { @@ -52,6 +56,9 @@ export type AgentSessionTurnContext = { sessionId: string journal: AgentSessionJournal fence: number + /** The host's projection of what work is current (`structuredAgentSessionCurrentWork`), read + * fresh per call: a child's end can move it mid-operation. */ + currentWork?: () => StructuredAgentSessionCurrentWork adapter: StructuredAgentSessionAdapter /** What each agent declares; the session's own answer is `agents.capabilities(agent)`. */ agents: StructuredAgentRegistry @@ -210,12 +217,13 @@ export async function handOverSubmission( await handOverStructuredAgentSessionCommand(ctx, submission, body) return } - // The message joins the turn running at handover, a steer, or opens its own. + // The message joins the turn the child it goes to is running, a steer, or opens its own; never a + // turn an ended generation left. await ctx.journal.resolveDispatch({ clientMessageId, state: 'pending', fence: ctx.fence, - turnScope: ctx.journal.liveTurnScope() + turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() }) // The handover row's instant on the host clock; the turn this dispatch opens records it so the // live counter never re-anchors at turn-open. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index aef5e47314dc..fc3a9ed9bd9d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -123,7 +123,7 @@ describe('provider-exit settlement', () => { now = 60_000 return { ok: false, error: new Error('sink unavailable') } }, - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId, task) => task(), now: () => now } as never, @@ -201,11 +201,12 @@ describe('provider-exit settlement', () => { } await settleStructuredAgentSessionChildExit( + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the exit settlement reads only these members of its context; the store double is partial. { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId, task) => task(), now: () => 1_234 } as never, @@ -312,7 +313,7 @@ describe('provider-exit settlement', () => { ] return { ok: true } }, - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1_234 } @@ -360,7 +361,7 @@ describe('provider-exit settlement', () => { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), - publishFence: vi.fn(), + generationEnded: vi.fn(), serialize: async (_sessionId: string, task: () => Promise) => task(), now: () => 1 } @@ -411,7 +412,7 @@ describe('provider-exit settlement', () => { }) } const log = recordingStructuredAgentSessionLogger() - const publishFence = vi.fn() + const generationEnded = vi.fn() const event = { type: 'ended' as const, sessionId: SESSION, @@ -425,7 +426,7 @@ describe('provider-exit settlement', () => { store, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: false, error: new Error('sink failed') }), - publishFence, + generationEnded, serialize: async (_sessionId, task) => task(), now: () => 1, logger: log.logger @@ -433,7 +434,8 @@ describe('provider-exit settlement', () => { await settleStructuredAgentSessionChildExit(context, event) expect(session.child).toBeNull() - expect(publishFence).toHaveBeenCalledTimes(1) + expect(generationEnded).toHaveBeenCalledTimes(1) + expect(generationEnded).toHaveBeenCalledWith(SESSION, { restate: true }) expect(log.scopes()).toEqual(['exit-lifecycle-barrier', 'exit-settlement']) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts index b80c5cd51878..d174065cc133 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts @@ -599,7 +599,7 @@ describe('what is not kept', () => { ) }) - it('the delivery step keeps one the open could not settle, and hands nothing over', async () => { + it('a failed keep rejects nothing: a fresh send is delivered, the earlier one is never handed over, and the next open keeps it', async () => { const id = await acceptWhileStarting(sendRequest('open write failed')) const resolve = AgentSessionJournal.prototype.resolveDispatch // The open's keep and its plain-rejection fallback both fail. @@ -616,12 +616,20 @@ describe('what is not kept', () => { return resolve.apply(this, args) }) await quitRestart() - // The open's write failed; a new send wakes the delivery loop, whose first step settles it. + // The open's keep failed. Delivery does not wait on that bookkeeping: what may be handed over + // is derived from where each send sits, so only this process's own send goes. const next = rig.send('wakes the loop') - await next.result - await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) + expect(await next.result).toMatchObject({ ok: true }) await eventually(() => expect(dispatchedTexts()).toEqual(['wakes the loop'])) + const earlier = await rig.submission(id) + expect(earlier).toMatchObject({ dispatchState: 'pending', handoverRecorded: true }) + expect(earlier).not.toHaveProperty('handedOverAt') + expect(await rig.drafts()).toEqual([]) + // The next open's bookkeeping lands, and keeps it as a card. + await quitRestart() + await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', ...HOST_RESTARTED }) + expect(dispatchedTexts()).toEqual(['wakes the loop']) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index cd901310d8dc..ba16a34b58ca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -52,6 +52,11 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { isUnansweredHandedOverSubmission } from '../agent-session-journal/journal-unsent-send-hold' import { structuredAgentSessionShownStatus } from './structured-agent-session-shown-work' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionWorkEvidence +} from './structured-agent-session-current-work' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' /** A send Orca journaled that the provider has neither opened a turn for nor refused. Mirrors the * projection's own unanswered-dispatch rule, which is what makes that window read as `working`. */ @@ -71,13 +76,18 @@ function pendingSubmissionInFlight( return null } -/** The work identity to record: a running turn if one exists, else the send still awaiting one. */ +/** The work identity to record: a running turn if one exists, else the send still awaiting one. + * `scope`: the host's (`structuredAgentSessionCurrentWork`), so an ended generation's turn is none. */ export function structuredAgentSessionWorkInFlight( items: readonly AgentJournalRenderItem[], - submissions: readonly AgentJournalSubmission[] + submissions: readonly AgentJournalSubmission[], + scope?: StructuredAgentSessionWorkScope ): AgentSessionResumeWork | null { const current = agentSessionCurrentContextRows(items, submissions) - const turnId = activeStructuredAgentSessionTurnId(current.items) + const turnId = activeStructuredAgentSessionTurnId( + current.items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ) if (turnId) { return { kind: 'turn', id: turnId } } @@ -91,7 +101,8 @@ export function structuredAgentSessionWorkInFlight( */ function structuredAgentSessionResumeWork( items: readonly AgentJournalRenderItem[], - submissions: readonly AgentJournalSubmission[] + submissions: readonly AgentJournalSubmission[], + scope: StructuredAgentSessionWorkScope ): AgentSessionResumeWork | null { const bodies = new Map(items.map((item) => [item.itemId, item.body])) const bodyOf = (itemId: string) => bodies.get(itemId) @@ -107,7 +118,7 @@ function structuredAgentSessionResumeWork( ) { return null } - const inFlight = structuredAgentSessionWorkInFlight(items, submissions) + const inFlight = structuredAgentSessionWorkInFlight(items, submissions, scope) if (inFlight) { return inFlight } @@ -129,13 +140,17 @@ function boundedLabel(text: string | undefined): string { /** The prompts the session is blocked on, from the same items the projection called `attention` * over — one read, so the recorded state and the prompts it stands for cannot disagree. */ -function pendingPrompts(items: readonly AgentJournalRenderItem[]): AgentSessionRestartPrompt[] { +function pendingPrompts( + items: readonly AgentJournalRenderItem[], + scope: StructuredAgentSessionWorkScope +): AgentSessionRestartPrompt[] { const prompts: AgentSessionRestartPrompt[] = [] for (const item of items) { const body = item.body if ( (body.kind !== 'approval' && body.kind !== 'question') || - body.resolution.state !== 'pending' + body.resolution.state !== 'pending' || + !scope.isCurrentItem(item.itemId) ) { continue } @@ -165,6 +180,7 @@ type WorkingCandidateSession = { journal: AgentSessionJournal /** Only this host generation's own child counts. A restored-for-reading journal has none. */ child: { fence: number } | null + lastEndedChild?: StructuredAgentSessionWorkEvidence['ended'] } /** The offer one session is owed, taken right before teardown stops its provider child; null when @@ -197,17 +213,21 @@ export function structuredAgentSessionWorkingAtStop(input: { !(unanswered && isUnansweredHandedOverSubmission(submission)) ) const childWork = input.childWork(sessionId) + const record = input.getRecord(sessionId) + const { scope } = structuredAgentSessionCurrentWork(session.journal, { + record, + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) + }) const status = structuredAgentSessionShownStatus( { items: current.items, submissions: handedOver }, childWork, session.child.fence, - session.journal.itemFence + scope ) if (status.state === 'done') { return null } - const work = structuredAgentSessionResumeWork(current.items, handedOver) - const record = input.getRecord(sessionId) + const work = structuredAgentSessionResumeWork(current.items, handedOver, scope) const head = record ? agentSessionProviderHandleChainHead(record.providerHandleChain) : null if (!work || !head) { return null @@ -216,7 +236,7 @@ export function structuredAgentSessionWorkingAtStop(input: { // The lead's OWN state, not the fold: a settled lead with running children reads `done` here // and carries them in `tasks`, which is how the dialog tells the two apart. state: status.mainAgent.state, - prompts: pendingPrompts(current.items), + prompts: pendingPrompts(current.items, scope), tasks: liveTasks(childWork) } return { diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts index 50ddcbc08e9e..9a0d655c771a 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.test.ts @@ -27,7 +27,14 @@ function contextWith( return { sessionId: 'session-1', fence: 1, - journal: { snapshot: () => ({ items: [] }), submissions: () => [] }, + journal: { + snapshot: () => ({ items: [] }), + submissions: () => [], + runningTurn: () => null, + itemFence: () => undefined, + visitItems: () => undefined, + wroteBeforeOpen: () => false + }, adapter } } @@ -59,6 +66,8 @@ const SETTLED = child({ settledAt: 300 }) const RECORD = { lease: {} } as unknown as AgentSessionRecord +// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: admission reads only the running turn's id and item id. +const RUNNING_TURN = { turnId: 'turn-1', item: { itemId: 'turn-1' } } as never const STOP = 'Stop background tasks before using this command.' const WAIT = 'Wait for background tasks to finish before using this command.' @@ -120,6 +129,7 @@ describe('conversationCommandBlocked background work', () => { } ] }) as unknown as ReturnType + ctx.journal.runningTurn = () => RUNNING_TURN expect(conversationCommandBlocked(ctx, RECORD, [child()])?.message).toBe( 'Wait for the current turn to finish before using this command.' ) @@ -135,6 +145,7 @@ describe('conversationCommandBlocked for a command sent at rest (C6, B3)', () => const ctx = contextWith(undefined) // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the admission reads only each item's body. ctx.journal.snapshot = () => staleTurn as never + ctx.journal.runningTurn = () => RUNNING_TURN expect(conversationCommandBlocked(ctx, RECORD, [], 'at-rest')).toBeNull() expect(conversationCommandBlocked(ctx, RECORD, [])).toMatchObject({ details: { reason: 'turnActive' }, diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 7d81e4a77180..7fe5b1fce056 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -6,10 +6,13 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work-liveness' import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' -import { activeStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-projection' import { agentSessionCurrentContextRows } from '../../../shared/agent-session-context-clear' -import { isUnansweredStructuredAgentSessionDispatch } from '../../../shared/structured-agent-session-unanswered-dispatch' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + contextStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkJournal +} from './structured-agent-session-current-work' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { refuse, @@ -21,10 +24,11 @@ import { export type ConversationCommandAdmissionContext = { sessionId: string fence: number - journal: { + journal: StructuredAgentSessionCurrentWorkJournal & { snapshot(): Pick, 'items'> - submissions: AgentSessionJournal['submissions'] } + /** The host's projection of current work; without one, the generation at `fence` is live. */ + currentWork?: () => StructuredAgentSessionCurrentWork adapter: Pick } @@ -64,14 +68,18 @@ export function conversationCommandBlocked( if (record.lease.handoffStage || record.lease.handoffOperationId) { return blocked('handoffInFlight', 'Wait for the session handoff to finish.') } - if (admission !== 'at-rest' && activeStructuredAgentSessionTurnId(items)) { + // What is current is the host projection's to say: an ended generation's turn, prompt or send + // refuses nothing. + const work = contextStructuredAgentSessionCurrentWork(ctx) + if (admission !== 'at-rest' && work.activeTurnId()) { return blocked('turnActive', 'Wait for the current turn to finish before using this command.') } if ( items.some( (item) => (item.body.kind === 'approval' || item.body.kind === 'question') && - item.body.resolution.state === 'pending' + item.body.resolution.state === 'pending' && + work.isCurrentItem(item.itemId) ) ) { return blocked( @@ -93,8 +101,7 @@ export function conversationCommandBlocked( if ( submissions.some( (entry) => - !(admission === 'handover' && isQueuedAgentJournalSubmission(entry)) && - isUnansweredStructuredAgentSessionDispatch(entry, ctx.fence) + !(admission === 'handover' && isQueuedAgentJournalSubmission(entry)) && work.owesSend(entry) ) ) { return blocked( diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index ab6df634d13f..9d36e1b43d03 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -69,6 +69,7 @@ export function runStructuredConversationCommand( // Starts the agent only to settle a rewind in doubt, as a send does; a /clear itself starts nothing. prepareSession: sendPreparation(context, envelope, { refusesInRun: true }), journal: () => context.sessions.get(sessionId)?.journal, + endedChild: () => context.sessions.get(sessionId)?.lastEndedChild, publish: (journal) => context.publish(sessionId, journal), now: context.now, plan: { diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts b/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts index ffaf2104b410..894b8b819b55 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-compaction.test.ts @@ -248,15 +248,16 @@ it('hands over a message held behind the command when the command ends just as t await state.host.conversationCommand(CALLER, compactParams()) await vi.waitFor(() => expect(compact).toHaveBeenCalledOnce(), { interval: 1 }) const { journal: live } = state.host['sessions'].get(SESSION)! - const activeTurnId = live.activeTurnId + const runningTurn = live.runningTurn let ended = false - vi.spyOn(live, 'activeTurnId').mockImplementation(() => { - const read = activeTurnId() + // The loop reads the command through the host's current-work projection, which asks this. + vi.spyOn(live, 'runningTurn').mockImplementation(() => { + const read = runningTurn() // The provider's end lands the moment the loop's own step reads the command as running and // stops; no other reader's view of it matters here. if ( !ended && - read?.startsWith('compact:') && + read?.turnId.startsWith('compact:') && /at (?:StructuredAgentSessionDeliveryLoop\.)?prepare \(.*structured-agent-session-delivery-loop/.test( new Error('who reads').stack ?? '' ) diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts b/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts index f41e83522d2c..5fc0f7f184a7 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-retained-host-rows.ts @@ -5,7 +5,10 @@ // each keeps the ones its retained row held. import { parseCodexGoalJournalItemId } from '../../codex/codex-goal-journal-identity' -import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + agentJournalItemKey, + parseAgentJournalItemKey +} from '../../../shared/agent-session-journal-item-key' import { AGENT_JOURNAL_ITEM_BODY_KINDS, isAdmissibleAgentJournalMessageBody @@ -64,12 +67,28 @@ export function mergeRetainedHostLifecycleRows( ) } +/** The live agent's rewound history as retained rows: its own observation, at the fence it + * answered at. A row the old epoch held keeps that row's provenance when merged. */ +export function rewindProviderRows( + items: readonly { identity: AgentJournalItemIdentity; body: AgentJournalItemBody }[], + observedAt: number, + ownerFence: number +): RetainedRow[] { + return items.map(({ identity, body }) => ({ + itemId: agentJournalItemKey(identity), + body, + observedAt, + ownerFence + })) +} + /** The rebuilt epoch's item for one retained row, with the scope and producer it was written with. */ export function retainedRowReplacement(row: RetainedRow): AgentJournalProducerLinkage & { identity: AgentJournalItemIdentity body: AgentJournalItemBody observedAt: number turnScope?: AgentJournalTurnScope + ownerFence: number } { const identity = parseAgentJournalItemKey(row.itemId) if (!identity) { @@ -87,6 +106,9 @@ export function retainedRowReplacement(row: RetainedRow): AgentJournalProducerLi body: restoreRewindJournalBody(row.body), observedAt: row.observedAt, ...(scope ? { turnScope: scope } : {}), + // The old fold's provenance, so a rewind never makes a dead generation's work current. A record + // from before rows carried it states none: 0, which is never current. + ownerFence: row.ownerFence ?? 0, ...(agentId === undefined ? {} : { agentId }), ...(parentAgentId === undefined ? {} : { parentAgentId }), ...(providerParentRef === undefined ? {} : { providerParentRef }), diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index d3bf795a19c6..05683fcc505c 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -73,7 +73,10 @@ describe('structured mailbox pointer host', () => { // far from the tail, so any page-sized read reports a busy worker as idle — and `@idle` then // wakes it mid-turn. const items = [runningTurn(), ...transcript(500)] - hostRef.current = { journalSnapshot: () => ({ items, submissions: [] }) } + hostRef.current = { + journalSnapshot: () => ({ items, submissions: [] }), + currentWork: () => null + } expect(await readStructuredSessionGateFacts('s1')).toEqual({ turnRunning: true, awaitingHuman: false diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index e51734b5b85e..92b1e0594823 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -47,7 +47,12 @@ export async function readStructuredSessionGateFacts( sessionId: string ): Promise { const snapshot = await readSessionJournal(sessionId) - return snapshot ? structuredSessionGateFacts(snapshot.items) : null + if (!snapshot) { + return null + } + // The host's projection of current work: an agent that ended is idle, whatever its rows say. + const work = getStructuredAgentSessionHost()?.currentWork(sessionId) + return structuredSessionGateFacts(snapshot.items, work?.scope) } /** What each recorded send settled as. */ diff --git a/src/main/runtime/orchestration/structured-session-pointer-delivery.ts b/src/main/runtime/orchestration/structured-session-pointer-delivery.ts index fc7758e07a19..8a99deb39950 100644 --- a/src/main/runtime/orchestration/structured-session-pointer-delivery.ts +++ b/src/main/runtime/orchestration/structured-session-pointer-delivery.ts @@ -8,6 +8,7 @@ */ import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' import { activeStructuredAgentSessionTurnId, projectStructuredAgentSessionStatus @@ -46,11 +47,17 @@ export type StructuredSessionGateFacts = { * and idle-with-history is the normal steady state of a working agent. */ export function structuredSessionGateFacts( - items: readonly AgentJournalRenderItem[] + items: readonly AgentJournalRenderItem[], + /** The host's projection of current work (`structuredAgentSessionCurrentWork`). */ + scope?: StructuredAgentSessionWorkScope ): StructuredSessionGateFacts { return { - turnRunning: activeStructuredAgentSessionTurnId(items) !== null, - awaitingHuman: projectStructuredAgentSessionStatus(items) === 'attention' + turnRunning: + activeStructuredAgentSessionTurnId( + items, + scope && ((item) => scope.isCurrentItem(item.itemId)) + ) !== null, + awaitingHuman: projectStructuredAgentSessionStatus(items, [], undefined, scope) === 'attention' } } diff --git a/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts b/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts index e255f5eefc80..dd1a7ec78dc9 100644 --- a/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts +++ b/src/main/runtime/orchestration/structured-worker-group-addressing.test.ts @@ -83,7 +83,8 @@ function installHost(options: { present: true, sessionIds: options.tabListed ? [SESSION_ID] : [] }), - journalSnapshot: async () => ({ items: options.items ?? [idleTurn()] }) + journalSnapshot: async () => ({ items: options.items ?? [idleTurn()] }), + currentWork: () => null } } diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index a24d21e69382..3ffb9943b329 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -101,12 +101,13 @@ function statusFeed(): StructuredAgentSessionStatusFeed { [ STATUS_SESSION, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks and snapshot() of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot() and itemFence() of a journal with no sends. journal: { cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), lastActivityAt: () => 2, stopMarks: { latest: () => null, revision: () => 0 }, - snapshot: () => ({ items: STATUS_ITEMS, submissions: [] }) + snapshot: () => ({ items: STATUS_ITEMS, submissions: [] }), + itemFence: () => undefined } as unknown as AgentSessionJournal, params: { location: { diff --git a/src/main/runtime/structured-worker-cleared-session.test.ts b/src/main/runtime/structured-worker-cleared-session.test.ts index c4d07df01fb8..037ec7beb14c 100644 --- a/src/main/runtime/structured-worker-cleared-session.test.ts +++ b/src/main/runtime/structured-worker-cleared-session.test.ts @@ -207,6 +207,8 @@ function installHost(): void { open.add(id) return { items: [message(`${id}-1`, 'idle')], submissions: [] } }, + // No projection held: the gate reads the timeline as it stands. + currentWork: () => null, setSessionTabVisibility: async (id: string, visible: boolean) => { if (id === failHideOf) { throw new Error('the durable tab index is wedged') diff --git a/src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts b/src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts new file mode 100644 index 000000000000..530ef37163e0 --- /dev/null +++ b/src/renderer/src/components/native-chat/structured-agent-session-actionable-prompts.test.ts @@ -0,0 +1,57 @@ +// The desktop chat answers only the prompts the host says still wait on the person: one an agent +// that has ended raised gets no card, and an older host, which names none, keeps today's rule. + +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' +import { EMPTY_STRUCTURED_AGENT_SESSION } from '../../../../shared/structured-agent-session-reducer' +import { pendingStructuredSessionPrompts } from './structured-agent-session-message-projection' +import { structuredAttentionReadObservation } from './structured-attention-read-observation' + +function approval(itemId: string, sequence: number): AgentJournalRenderItem { + return { + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } + } +} + +const ITEMS = [approval('live-prompt', 1), approval('dead-prompt', 2)] +const ids = (items: AgentJournalRenderItem[]) => items.map((item) => item.itemId) + +describe('the prompt cards a desktop chat offers', () => { + it('are the ones the host names', () => { + expect(ids(pendingStructuredSessionPrompts(ITEMS, ['live-prompt']))).toEqual(['live-prompt']) + expect(pendingStructuredSessionPrompts(ITEMS, [])).toEqual([]) + }) + + it('are every pending prompt when an older host names none', () => { + expect(ids(pendingStructuredSessionPrompts(ITEMS))).toEqual(['live-prompt', 'dead-prompt']) + }) +}) + +describe('what marks a chat read', () => { + it('changes when the host stops naming a prompt, with no row changing', () => { + const ask: AgentJournalRenderItem = { + itemId: 'ask', + revision: 1, + sequence: 0, + observedAt: 0, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'go' }] } + } + const state = { ...EMPTY_STRUCTURED_AGENT_SESSION, items: [ask, ...ITEMS] } + const named = structuredAttentionReadObservation({ + ...state, + actionablePromptIds: ['live-prompt'] + }) + const ended = structuredAttentionReadObservation({ ...state, actionablePromptIds: [] }) + expect(ended).not.toBe(named) + }) +}) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts b/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts index 25ff332ceb87..fa9297874932 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-message-projection.ts @@ -6,6 +6,7 @@ import type { import type { StructuredAgentSessionOptimisticMessage } from '../../../../shared/structured-agent-session-message-projection' import { projectStructuredAgentSessionMessages as projectMessages } from '../../../../shared/structured-agent-session-message-projection' import { projectStructuredQuestionMessages } from './structured-agent-question-projection' +import { isActionableStructuredAgentSessionPrompt } from '../../../../shared/structured-agent-session-live-turn' /** The desktop's transcript: a message the host accepted and then rejected stays where it was * sent, as not sent, unless the queue holds it as a card. */ @@ -27,12 +28,16 @@ export type StructuredPromptItem = AgentJournalRenderItem & { body: Extract } +/** The prompt cards a person can answer: pending, and named by the host as waiting + * (`isActionableStructuredAgentSessionPrompt`) when the host names them. */ export function pendingStructuredSessionPrompts( - items: AgentJournalRenderItem[] + items: AgentJournalRenderItem[], + actionablePromptIds?: readonly string[] ): StructuredPromptItem[] { return agentSessionCurrentContextRows(items).items.filter( (item): item is StructuredPromptItem => (item.body.kind === 'approval' || item.body.kind === 'question') && - item.body.resolution.state === 'pending' + item.body.resolution.state === 'pending' && + isActionableStructuredAgentSessionPrompt(item.itemId, actionablePromptIds) ) } diff --git a/src/renderer/src/components/native-chat/structured-attention-read-observation.ts b/src/renderer/src/components/native-chat/structured-attention-read-observation.ts index a614438deb57..8a365d4fc652 100644 --- a/src/renderer/src/components/native-chat/structured-attention-read-observation.ts +++ b/src/renderer/src/components/native-chat/structured-attention-read-observation.ts @@ -1,5 +1,6 @@ import type { StructuredAgentSessionState } from '../../../../shared/structured-agent-session-reducer' import { projectStructuredAgentSessionStatusState } from '../../../../shared/structured-agent-session-projection' +import { isActionableStructuredAgentSessionPrompt } from '../../../../shared/structured-agent-session-live-turn' const observations = new WeakMap() @@ -16,7 +17,10 @@ export function structuredAttentionReadObservation(state: StructuredAgentSession const request = projection.latestRequest const key = JSON.stringify([ state.cursor?.epoch, - projection.pendingPromptIds, + // The host's answer where it gives one: a prompt an agent that ended raised waits on no one. + projection.pendingPromptIds.filter((itemId) => + isActionableStructuredAgentSessionPrompt(itemId, state.actionablePromptIds) + ), request?.turnState !== 'running' ? [request?.kind, request?.id, request?.outcome] : null ]) observations.set(state, key) diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts index ef730a1781df..2f430e053ada 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts @@ -22,6 +22,8 @@ export function useStructuredAgentSessionTransportState( const subagentRoster = (enabled ? state.subagentRoster : undefined) ?? NO_SUBAGENT_ROSTER const fence = enabled ? state.fence : null const latestTurn = enabled ? state.latestTurn : undefined + // The host's answer to which pending prompts still wait on the person; absent from an older host. + const actionablePromptIds = enabled ? state.actionablePromptIds : undefined const current = useMemo( () => agentSessionCurrentContextRows(journalItems, submissions), [journalItems, submissions] @@ -58,6 +60,7 @@ export function useStructuredAgentSessionTransportState( return { journalItems, latestTurn, + actionablePromptIds, subagentRoster, submissions, fence, diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session.ts b/src/renderer/src/components/native-chat/use-structured-agent-session.ts index 5c5f4615057e..3a0e144138ac 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session.ts @@ -130,7 +130,10 @@ export function useStructuredAgentSession(args: { hostStopping, mutate }) - const prompts = pendingStructuredSessionPrompts(transportState.journalItems) + const prompts = pendingStructuredSessionPrompts( + transportState.journalItems, + transportState.actionablePromptIds + ) const promptsUnanswerableHere = pendingPromptsAllUnanswerableHere(prompts) // A send after a command the queue will run goes behind it, even with follow-ups off. // A host's queue waits on any pending prompt, and nothing here can settle one this build cannot diff --git a/src/shared/agent-session-refusal-details.ts b/src/shared/agent-session-refusal-details.ts index 3c1fde945665..46d3141639b6 100644 --- a/src/shared/agent-session-refusal-details.ts +++ b/src/shared/agent-session-refusal-details.ts @@ -47,6 +47,8 @@ export const AGENT_SESSION_REFUSAL_REASONS = { 'rewindUnconfirmed', // A prompt card or an option 'promptGone', + /** The agent that raised the prompt has ended, so nothing can take its answer. */ + 'promptOwnerEnded', 'optionRejected', // The provider 'providerStarting', diff --git a/src/shared/agent-session-refusal-notice.test.ts b/src/shared/agent-session-refusal-notice.test.ts index aadf9e26fa49..4063a95aaea5 100644 --- a/src/shared/agent-session-refusal-notice.test.ts +++ b/src/shared/agent-session-refusal-notice.test.ts @@ -526,6 +526,12 @@ describe('the notice for every reason a host names', () => { 'answer', 'This question was already answered or has changed.' ], + [ + 'agent_session_operation_invalid', + 'promptOwnerEnded', + 'answer', + 'Your answer was not sent. The agent stopped while this response was in progress. You can continue in this conversation.' + ], [ 'agent_session_checkpoint_stale', 'fenceStale', diff --git a/src/shared/agent-session-refusal-notice.ts b/src/shared/agent-session-refusal-notice.ts index f81e1b51a901..67c686f9b720 100644 --- a/src/shared/agent-session-refusal-notice.ts +++ b/src/shared/agent-session-refusal-notice.ts @@ -125,6 +125,10 @@ function reasonParts( return undefined } if ('fact' in words) { + if (write === 'answer') { + // An answer to an agent that stopped: its row's sentence, which names it and says go on. + return [NOT_DONE[write], { failure: { kind: words.fact }, surface: 'row', context }] + } return write === 'send' || write === 'composer-send' ? [ NOT_DONE[write], diff --git a/src/shared/agent-session-refusal-reason-words.ts b/src/shared/agent-session-refusal-reason-words.ts index 0f947a850917..bbf30cd0d27f 100644 --- a/src/shared/agent-session-refusal-reason-words.ts +++ b/src/shared/agent-session-refusal-reason-words.ts @@ -88,6 +88,8 @@ const REASON_WORDS = { rewindRefused: codeWords('hostFinding'), rewindUnconfirmed: codeWords('hostFinding'), promptGone: causeWords('questionChanged', 'nothingLeft'), + // Said as that agent's own stop, naming it, with the step on from there. + promptOwnerEnded: { fact: 'providerExited', action: 'nothingLeft' }, optionRejected: causeWords('optionRejected', 'retry'), providerStarting: AGENT_STARTING, goalsUnsupported: causeWords('goalsUnsupported', 'hostFinding'), diff --git a/src/shared/agent-session-rewind.ts b/src/shared/agent-session-rewind.ts index e7d864390cca..36e30f9b79a4 100644 --- a/src/shared/agent-session-rewind.ts +++ b/src/shared/agent-session-rewind.ts @@ -55,6 +55,9 @@ export const AgentSessionRewindRecordSchema = z.object({ observedAt: z.number().finite(), // Absent on records written before rows stated them: the rebuild then places the row by position. turnScope: AgentJournalTurnScopeSchema.optional(), + // The row's execution provenance, carried into the rebuilt epoch. Absent on records written + // before rows carried it. + ownerFence: z.number().int().nonnegative().optional(), ...AgentJournalProducerLinkageFields }) ) diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index 21aec935cdad..e06444e6d75f 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -148,6 +148,11 @@ export type AgentSessionHistoryPage = { subagentRoster?: AgentSessionSubagentRosterEntry[] /** As of the page's read; a client applies it only from a page that replaces its state. */ latestTurn?: AgentSessionLatestTurn | null + /** Rides with `latestTurn`: the approvals and questions still waiting on the person that the + * agent running now raised, over the whole journal. One an agent that has ended raised is not + * listed, since nothing can take its answer. Absent from an older host, whose clients read every + * pending prompt they hold as waiting. */ + actionablePromptIds?: string[] } export type AgentSessionHistoryResult = @@ -207,6 +212,8 @@ export type AgentSessionSubscribeEvent = /** Rides every batch that carries rows, removals or submissions, so absent there means an * older host; absent on one that carries none, which changes no turn. */ latestTurn?: AgentSessionLatestTurn | null + /** Rides with `latestTurn` (`AgentSessionHistoryPage.actionablePromptIds`). */ + actionablePromptIds?: string[] } & AgentSessionFrameFields) | ({ type: 'reset' diff --git a/src/shared/structured-agent-session-coalescer.ts b/src/shared/structured-agent-session-coalescer.ts index f80658838dc1..21e11ad4b6ce 100644 --- a/src/shared/structured-agent-session-coalescer.ts +++ b/src/shared/structured-agent-session-coalescer.ts @@ -1,5 +1,8 @@ import type { AgentSessionSubscribeEvent } from './agent-session-wire' -import { latestTurnAfterStructuredAgentSessionBatch } from './structured-agent-session-live-turn' +import { + actionablePromptIdsAfterStructuredAgentSessionBatch, + latestTurnAfterStructuredAgentSessionBatch +} from './structured-agent-session-live-turn' export const STRUCTURED_AGENT_SESSION_CLIENT_COALESCE_MS = 48 @@ -29,6 +32,10 @@ function mergeBatch( } // As applying both in turn would leave it, so an older host's rows still drop a stale claim. const latestTurn = latestTurnAfterStructuredAgentSessionBatch(left.latestTurn, right) + const actionablePromptIds = actionablePromptIdsAfterStructuredAgentSessionBatch( + left.actionablePromptIds, + right + ) return { type: 'batch', ...(right.commands !== undefined || left.commands !== undefined @@ -55,7 +62,8 @@ function mergeBatch( ...(right.activity !== undefined || left.activity !== undefined ? { activity: right.activity !== undefined ? right.activity : (left.activity ?? null) } : {}), - ...(latestTurn !== undefined ? { latestTurn } : {}) + ...(latestTurn !== undefined ? { latestTurn } : {}), + ...(actionablePromptIds !== undefined ? { actionablePromptIds } : {}) } } diff --git a/src/shared/structured-agent-session-live-turn.ts b/src/shared/structured-agent-session-live-turn.ts index 5049e55409d0..91f4bda9f21a 100644 --- a/src/shared/structured-agent-session-live-turn.ts +++ b/src/shared/structured-agent-session-live-turn.ts @@ -27,10 +27,6 @@ import { isRootAgentJournalItem } from './agent-session-journal-producer' import type { AgentSessionLatestTurn, AgentSessionSubscribeEvent } from './agent-session-wire' import { readAgentJournalTurn } from './agent-session-turn-record' import { isAgentSessionContextClear } from './agent-session-context-clear' -import { - isStructuredAgentSessionEndedGenerationWork, - type StructuredAgentSessionItemFence -} from './structured-agent-session-main-agent-working' import type { NativeChatToolCallBlock } from './native-chat-types' import { isRunningStructuredAgentSessionToolAction, @@ -39,8 +35,8 @@ import { type StructuredAgentSessionToolAction } from './structured-agent-session-tool-call-block' -/** `current`: whether the running turn's item is current work; one an ended generation opened is - * none (`isStructuredAgentSessionEndedGenerationWork`). */ +/** `current`: a host's answer to whether the running turn's item is current work + * (`StructuredAgentSessionWorkScope`); one an ended generation opened is none. */ export function activeStructuredAgentSessionTurnId( items: readonly AgentJournalRenderItem[], current?: (item: AgentJournalRenderItem) => boolean @@ -110,20 +106,6 @@ export function runningStructuredAgentSessionTurnItemBySequence( return newest && turn?.state === 'running' ? { item: newest, turnId: turn.turnId } : null } -/** The running turn by sequence, given the lease's fence: one an ended generation opened is none - * (`isStructuredAgentSessionEndedGenerationWork`). The host's one read of its active turn. */ -export function currentStructuredAgentSessionTurnId( - items: Iterable, - itemFence: StructuredAgentSessionItemFence, - currentFence?: number | null -): string | null { - const running = runningStructuredAgentSessionTurnItemBySequence(items) - return running && - !isStructuredAgentSessionEndedGenerationWork(itemFence(running.item.itemId), currentFence) - ? running.turnId - : null -} - /** The newest turn record whatever state it ended in, STATE INCLUDED. Restart resume compares both * halves against the teardown marker: the id alone cannot tell a turn that was interrupted from * one that finished, and offering a finished chat is the failure this feature exists to avoid. @@ -193,6 +175,45 @@ export function latestTurnAfterStructuredAgentSessionBatch( return carriesRows || event.latestTurn !== undefined ? event.latestTurn : previous } +/** The host's actionable prompts once `event` applies, under `latestTurn`'s rule: a batch carrying + * rows restates them, so one without them came from an older host and drops the claim. */ +export function actionablePromptIdsAfterStructuredAgentSessionBatch( + previous: string[] | undefined, + event: Extract +): string[] | undefined { + const { items, removedItemIds, submissions } = event.batch + const carriesRows = items.length > 0 || removedItemIds.length > 0 || submissions.length > 0 + return carriesRows || event.actionablePromptIds !== undefined + ? event.actionablePromptIds + : previous +} + +/** Whether a frame's turn and prompts, when it names them, are what the state already holds: a + * generation's end can change them with no row, so a frame that does is never a no-op. */ +export function restatesStructuredAgentSessionHostWork( + event: Pick< + Extract, + 'latestTurn' | 'actionablePromptIds' + >, + state: { latestTurn?: AgentSessionLatestTurn | null; actionablePromptIds?: string[] } +): boolean { + return ( + (event.latestTurn === undefined || + JSON.stringify(event.latestTurn) === JSON.stringify(state.latestTurn)) && + (event.actionablePromptIds === undefined || + event.actionablePromptIds.join('\n') === state.actionablePromptIds?.join('\n')) + ) +} + +/** Whether a pending prompt waits on the person: the host names the ones that do, and a prompt an + * agent that has ended raised is not among them. An older host names none: every pending one. */ +export function isActionableStructuredAgentSessionPrompt( + itemId: string, + actionablePromptIds: readonly string[] | undefined +): boolean { + return actionablePromptIds === undefined || actionablePromptIds.includes(itemId) +} + /** * Whether the newest thing the active turn produced is the model's own reasoning. * diff --git a/src/shared/structured-agent-session-main-agent-working.ts b/src/shared/structured-agent-session-main-agent-working.ts index 0b03765cded8..ea8af7771282 100644 --- a/src/shared/structured-agent-session-main-agent-working.ts +++ b/src/shared/structured-agent-session-main-agent-working.ts @@ -1,29 +1,30 @@ import type { AgentJournalSubmission } from './agent-session-journal-types' import { hasUnansweredStructuredAgentSessionDispatch } from './structured-agent-session-unanswered-dispatch' +/** The host's answer to which work is current, built only by its operational projection + * (`structuredAgentSessionCurrentWork`). A client has none, so every running turn, pending prompt + * and unanswered send it holds reads as current, as it always has. */ +export type StructuredAgentSessionWorkScope = { + /** Whether the live generation's execution produced this item. */ + isCurrentItem: (itemId: string) => boolean + /** Whether this send is still owed: handed to the live generation and unanswered, or accepted by + * this host and not yet handed over. */ + owesSend: (submission: AgentJournalSubmission) => boolean +} + /** Whether the session's own agent is working: a running turn, or a send it has not answered. - * The one rule behind every session list's Working and the chat's own Stop. Callers pass the turn - * scoped to `currentFence` (`isStructuredAgentSessionEndedGenerationWork`). */ + * The one rule behind every session list's Working and the chat's own Stop. A host passes the turn + * its projection scoped, and that projection's `scope`. */ export function isStructuredAgentSessionMainAgentWorking( activeTurnId: string | null, submissions: readonly AgentJournalSubmission[], - currentFence?: number | null + currentFence?: number | null, + scope?: StructuredAgentSessionWorkScope ): boolean { return ( - activeTurnId !== null || hasUnansweredStructuredAgentSessionDispatch(submissions, currentFence) + activeTurnId !== null || + (scope + ? submissions.some(scope.owesSend) + : hasUnansweredStructuredAgentSessionDispatch(submissions, currentFence)) ) } - -/** The fence of the writer that created a journal item, where the reader knows it: the host does, - * a client does not. */ -export type StructuredAgentSessionItemFence = (itemId: string) => number | undefined - -/** Work a generation below the lease's fence wrote — a running turn, a pending prompt. Every release - * moves the fence, so that generation can no longer finish it: it holds nothing and is not - * Working, whether or not its settlement has landed yet. */ -export function isStructuredAgentSessionEndedGenerationWork( - itemFence: number | undefined, - currentFence: number | null | undefined -): boolean { - return currentFence != null && itemFence !== undefined && itemFence < currentFence -} diff --git a/src/shared/structured-agent-session-owed-work.ts b/src/shared/structured-agent-session-owed-work.ts index 3024e2af867b..259bbfe6877c 100644 --- a/src/shared/structured-agent-session-owed-work.ts +++ b/src/shared/structured-agent-session-owed-work.ts @@ -1,9 +1,8 @@ import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' import { activeStructuredAgentSessionTurnId } from './structured-agent-session-live-turn' import { - isStructuredAgentSessionEndedGenerationWork, isStructuredAgentSessionMainAgentWorking, - type StructuredAgentSessionItemFence + type StructuredAgentSessionWorkScope } from './structured-agent-session-main-agent-working' import { agentSessionCurrentContextRows } from './agent-session-context-clear' @@ -13,17 +12,16 @@ export function owesStructuredAgentSessionWork( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[], currentFence?: number | null, - itemFence?: StructuredAgentSessionItemFence + scope?: StructuredAgentSessionWorkScope ): boolean { const current = agentSessionCurrentContextRows(items, submissions) return isStructuredAgentSessionMainAgentWorking( activeStructuredAgentSessionTurnId( current.items, - itemFence && - ((item) => - !isStructuredAgentSessionEndedGenerationWork(itemFence(item.itemId), currentFence)) + scope && ((item) => scope.isCurrentItem(item.itemId)) ), current.submissions, - currentFence + currentFence, + scope ) } diff --git a/src/shared/structured-agent-session-projection.ts b/src/shared/structured-agent-session-projection.ts index 7c9b1a4076d7..f9fa35155adf 100644 --- a/src/shared/structured-agent-session-projection.ts +++ b/src/shared/structured-agent-session-projection.ts @@ -39,10 +39,7 @@ import { sha256 } from './sha256' import { readAgentMessageSource } from './agent-session-message-source' import { structuredAgentSessionStatusStartedAt } from './structured-agent-session-status-started-at' import { owesStructuredAgentSessionWork } from './structured-agent-session-owed-work' -import { - isStructuredAgentSessionEndedGenerationWork, - type StructuredAgentSessionItemFence -} from './structured-agent-session-main-agent-working' +import type { StructuredAgentSessionWorkScope } from './structured-agent-session-main-agent-working' import { agentSessionCurrentContextRows } from './agent-session-context-clear' // Re-exported so the live-turn readers' and the unanswered-send rule's existing consumers keep one @@ -215,31 +212,30 @@ export function structuredAgentSessionTabId(sessionId: string): string { return `structured-agent-session-${sessionId}` } -/** A prompt waiting on the user, unless an ended generation raised it: nothing can take its answer. */ +/** A prompt waiting on the user, unless the host's scope says an ended generation raised it: + * nothing can take its answer. */ function pendingStructuredAgentSessionPrompt( - currentFence?: number | null, - itemFence?: StructuredAgentSessionItemFence + scope?: StructuredAgentSessionWorkScope ): (item: AgentJournalRenderItem) => boolean { return (item) => (item.body.kind === 'approval' || item.body.kind === 'question') && item.body.resolution.state === 'pending' && - !( - itemFence && isStructuredAgentSessionEndedGenerationWork(itemFence(item.itemId), currentFence) - ) + (!scope || scope.isCurrentItem(item.itemId)) } -/** `itemFence`: a host's, so work an ended generation left neither holds nor reads as Working. */ +/** `scope`: a host's (`StructuredAgentSessionWorkScope`), so work an ended generation left neither + * holds nor reads as Working. */ export function projectStructuredAgentSessionStatus( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[] = [], currentFence?: number | null, - itemFence?: StructuredAgentSessionItemFence + scope?: StructuredAgentSessionWorkScope ): StructuredAgentSessionProjectedStatus { ;({ items, submissions } = agentSessionCurrentContextRows(items, submissions)) - if (items.some(pendingStructuredAgentSessionPrompt(currentFence, itemFence))) { + if (items.some(pendingStructuredAgentSessionPrompt(scope))) { return 'attention' } - return owesStructuredAgentSessionWork(items, submissions, currentFence, itemFence) + return owesStructuredAgentSessionWork(items, submissions, currentFence, scope) ? 'working' : 'idle' } @@ -281,7 +277,7 @@ export function projectStructuredAgentSessionStatusState( items: readonly AgentJournalRenderItem[], submissions: readonly AgentJournalSubmission[] = [], currentFence?: number | null, - itemFence?: StructuredAgentSessionItemFence + scope?: StructuredAgentSessionWorkScope ): { summary: StructuredAgentSessionStatusProjection latestRequest: StructuredAgentSessionLatestRequest | null @@ -300,7 +296,7 @@ export function projectStructuredAgentSessionStatusState( } } ;({ items, submissions } = agentSessionCurrentContextRows(items, submissions)) - const status = projectStructuredAgentSessionStatus(items, submissions, currentFence, itemFence) + const status = projectStructuredAgentSessionStatus(items, submissions, currentFence, scope) const statusToolCall = status === 'working' ? statusStructuredAgentSessionToolCall(items) : null const toolName = statusToolCall ? normalizeOptionalField(statusToolCall.name, AGENT_STATUS_TOOL_NAME_MAX_LENGTH) @@ -331,13 +327,10 @@ export function projectStructuredAgentSessionStatusState( return { latestRequest, owesWork: - status !== 'idle' && - owesStructuredAgentSessionWork(items, submissions, currentFence, itemFence), + status !== 'idle' && owesStructuredAgentSessionWork(items, submissions, currentFence, scope), pendingPromptIds: status === 'attention' - ? items - .filter(pendingStructuredAgentSessionPrompt(currentFence, itemFence)) - .map((item) => item.itemId) + ? items.filter(pendingStructuredAgentSessionPrompt(scope)).map((item) => item.itemId) : [], summary: { status, diff --git a/src/shared/structured-agent-session-reducer.test.ts b/src/shared/structured-agent-session-reducer.test.ts index 1592e949b1b2..1838740ae486 100644 --- a/src/shared/structured-agent-session-reducer.test.ts +++ b/src/shared/structured-agent-session-reducer.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from 'vitest' import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' import type { AgentSessionHistoryPage, AgentSessionLatestTurn } from './agent-session-wire' -import { runningStructuredAgentSessionTurnId } from './structured-agent-session-live-turn' +import { + isActionableStructuredAgentSessionPrompt, + runningStructuredAgentSessionTurnId +} from './structured-agent-session-live-turn' import { EMPTY_STRUCTURED_AGENT_SESSION, reduceStructuredAgentSession @@ -595,6 +598,75 @@ describe('structured agent session reducer', () => { }) }) + describe("the host's actionable prompts", () => { + const prompt = (itemId: string, sequence: number): AgentJournalRenderItem => ({ + itemId, + revision: 1, + sequence, + observedAt: sequence, + body: { + kind: 'approval', + title: 'Run command?', + detail: null, + options: [{ id: 'yes', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } + }) + const PROMPTS = [prompt('live-prompt', 1), prompt('dead-prompt', 2)] + const opened = (page: AgentSessionHistoryPage) => + reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'event', + event: { type: 'snapshot', sessionId: 'session-a', fence: 1, page } + }) + const batch = ( + state: ReturnType, + items: AgentJournalRenderItem[], + actionablePromptIds?: string[] + ) => + reduceStructuredAgentSession(state, { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + cursor: { epoch: 'epoch-a', sequence: state.cursor!.sequence + items.length }, + items, + removedItemIds: [], + submissions: [] + }, + ...(actionablePromptIds !== undefined ? { actionablePromptIds } : {}) + } + }) + const actionable = (state: ReturnType) => + state.items + .filter((entry) => + isActionableStructuredAgentSessionPrompt(entry.itemId, state.actionablePromptIds) + ) + .map((entry) => entry.itemId) + + it('reads them from the snapshot, keeps them across a frame without rows, and takes the next', () => { + const state = opened({ ...hydrationPage(PROMPTS), actionablePromptIds: ['live-prompt'] }) + expect(actionable(state)).toEqual(['live-prompt']) + const quiet = batch(state, []) + expect(quiet.actionablePromptIds).toEqual(['live-prompt']) + // The generation ended with no row written: the host's frame names none. + const ended = batch(quiet, [], []) + expect(actionable(ended)).toEqual([]) + }) + + it('an older host names none: its page reduces, and every pending prompt stays answerable', () => { + // As an older host serialized it: no such field at all. + const older: AgentSessionHistoryPage = JSON.parse(JSON.stringify(hydrationPage(PROMPTS))) + const state = opened(older) + expect(state.actionablePromptIds).toBeUndefined() + expect(actionable(state)).toEqual(['live-prompt', 'dead-prompt']) + // Rows from a downgraded host restate nothing: back to every pending prompt. + const named = opened({ ...hydrationPage(PROMPTS), actionablePromptIds: ['live-prompt'] }) + const downgraded = batch(named, [prompt('another', 3)]) + expect(downgraded.actionablePromptIds).toBeUndefined() + }) + }) + it('records the host clock from frames that carry it and keeps it otherwise', () => { const snapshot = reduceStructuredAgentSession( EMPTY_STRUCTURED_AGENT_SESSION, diff --git a/src/shared/structured-agent-session-reducer.ts b/src/shared/structured-agent-session-reducer.ts index 6ff6124feb38..d82950eaf1d7 100644 --- a/src/shared/structured-agent-session-reducer.ts +++ b/src/shared/structured-agent-session-reducer.ts @@ -26,7 +26,11 @@ import { import { compareAgentJournalItems } from './agent-session-journal-position' import { readAgentJournalTurn } from './agent-session-turn-record' import { queuePublicationField } from './structured-agent-session-queue-publication-field' -import { latestTurnAfterStructuredAgentSessionBatch } from './structured-agent-session-live-turn' +import { + actionablePromptIdsAfterStructuredAgentSessionBatch, + latestTurnAfterStructuredAgentSessionBatch, + restatesStructuredAgentSessionHostWork +} from './structured-agent-session-live-turn' import { foldStructuredAgentSubagentRoster, foldStructuredAgentSubagentRosterPage, @@ -75,6 +79,9 @@ export type StructuredAgentSessionState = { /** The host's newest turn record over the whole journal, which says whether a turn runs; absent * from an older host, whose answer is read off `items` instead. */ latestTurn?: AgentSessionLatestTurn | null + /** The prompts the host says still wait on the person (`AgentSessionHistoryPage`); absent from + * an older host, whose every pending prompt in `items` is read as waiting. */ + actionablePromptIds?: string[] /** Bumped per live batch that leaves a turn row's newest revision outside the window * (dropped or trimmed), so a whole-journal answer derived from turn rows is asked for again. */ unloadedTurnRevisions?: number @@ -130,6 +137,9 @@ function replacePage( subagentRoster: foldStructuredAgentSubagentRosterPage(undefined, page), activity: activity ?? null, ...(page.latestTurn !== undefined ? { latestTurn: page.latestTurn } : {}), + ...(page.actionablePromptIds !== undefined + ? { actionablePromptIds: page.actionablePromptIds } + : {}), ...(backgroundTasks !== undefined ? { backgroundTasks } : page.backgroundTasks !== undefined @@ -275,6 +285,7 @@ export function reduceStructuredAgentSession( (event.nextQueuedMessageId === undefined || event.nextQueuedMessageId === state.nextQueuedMessageId) && backgroundTaskStatesEqual(backgroundTasks, state.backgroundTasks) && + restatesStructuredAgentSessionHostWork(event, state) && activity?.turnId === state.activity?.turnId && activity?.text === state.activity?.text && state.status === 'ready' && @@ -310,6 +321,10 @@ export function reduceStructuredAgentSession( readRefusal: undefined, commands: event.commands !== undefined ? event.commands : state.commands, latestTurn: latestTurnAfterStructuredAgentSessionBatch(state.latestTurn, event), + actionablePromptIds: actionablePromptIdsAfterStructuredAgentSessionBatch( + state.actionablePromptIds, + event + ), ...queuePublicationField(event, state), backgroundTasks, ...(activity !== undefined ? { activity } : {}), From f122acb91ba96aab45e0e6e18a1b0cecb42ba913 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:37:48 -0700 Subject: [PATCH 03/22] feat(native-chat): settle an ended agent generation's work in the background, once per chat Every lease write that ends a generation (a release of any kind, an eviction, a failed start, a restart's adjudication, a new owner over a dead one) signals one in-memory reconciliation worker per chat after commit, as do an exit whose release could not be written and a late write from an ended generation. The worker re-derives what is owed each attempt, repairs the release and settles the work in one transaction, backs off up to 30 s on failure, and retires when nothing is owed. Startup scans every known chat in the background. No user action waits on it, opening a chat writes nothing, and the stored settled-at marker is gone. --- .../scripts/vitest-sqlite-runtime-files.mjs | 4 + ...obile-structured-actionable-prompt.test.ts | 2 +- ...tured-agent-session-prompt-cancel.test.tsx | 16 + .../use-mobile-structured-agent-session.ts | 10 +- ...aude-subagent-resume-after-restart.test.ts | 35 +- ...tructured-journal-exit-convergence.test.ts | 4 +- .../journal-item-provenance.ts | 137 +++++++ .../journal-lifecycle-batch-appender.ts | 6 +- .../journal-row-builders.ts | 33 +- .../journal-row-writer.ts | 15 +- .../journal-store-collaborators.ts | 2 +- .../journal-store-contracts.ts | 3 - .../agent-session-journal/journal-store.ts | 11 +- .../journal-subagent-liveness.ts | 63 +++- .../agent-session-record-test-fixture.ts | 3 +- .../agent-session-history-page.ts | 10 +- .../agent-session-subscriber-catch-up.ts | 12 +- ...-agent-session-accept-then-deliver.test.ts | 8 +- ...ed-agent-session-acquisition-settlement.ts | 72 ---- ...ured-agent-session-attach-orchestration.ts | 4 - ...d-agent-session-background-task-channel.ts | 2 +- .../structured-agent-session-child-exit.ts | 33 +- ...ssion-claude-stop-exit-ends-record.test.ts | 10 +- ...tructured-agent-session-client-delivery.ts | 6 +- ...red-agent-session-conversation-lifetime.ts | 9 + ...ctured-agent-session-conversations.test.ts | 8 +- .../structured-agent-session-conversations.ts | 17 +- ...tured-agent-session-crash-turn-end.test.ts | 57 ++- ...uctured-agent-session-current-work.test.ts | 6 +- .../structured-agent-session-current-work.ts | 49 +-- ...gent-session-dead-generation-settlement.ts | 5 - ...ctured-agent-session-forget-status.test.ts | 3 +- ...uctured-agent-session-host-current-work.ts | 40 ++ .../structured-agent-session-host-delivery.ts | 2 +- ...d-agent-session-host-runtime-state.test.ts | 3 +- .../structured-agent-session-host-types.ts | 3 + .../structured-agent-session-host.ts | 27 +- ...-session-leftover-settlement-scope.test.ts | 15 +- ...ssion-leftover-settlement-triggers.test.ts | 286 ++++++++------- ...ession-leftover-settlement.test-fixture.ts | 78 ++-- ...tured-agent-session-leftover-settlement.ts | 132 ++++--- ...ructured-agent-session-mutation-context.ts | 3 + ...gent-session-provider-child-record.test.ts | 12 +- ...ed-agent-session-queued-card-holds.test.ts | 7 +- ...tructured-agent-session-queued-messages.ts | 7 +- ...ctured-agent-session-queued-publication.ts | 6 +- .../structured-agent-session-queued-wiring.ts | 3 +- ...tured-agent-session-reconciliation-pass.ts | 162 +++++++++ ...gent-session-reconciliation-worker.test.ts | 163 +++++++++ ...red-agent-session-reconciliation-worker.ts | 341 ++++++++++++++++++ ...gent-session-resend-ledger-refusal.test.ts | 3 + .../structured-agent-session-rest-test-rig.ts | 8 +- ...tructured-agent-session-restart-restore.ts | 21 +- .../structured-agent-session-reveal.ts | 49 ++- ...ed-agent-session-roster-provenance.test.ts | 277 ++++++++++++++ ...agent-session-send-open-stale-turn.test.ts | 5 +- ...-session-settlement-double.test-fixture.ts | 11 +- ...ed-agent-session-stale-state-settlement.ts | 54 ++- ...ured-agent-session-stale-turn-open.test.ts | 37 +- ...agent-session-startup-failure-exit.test.ts | 8 +- ...uctured-agent-session-startup-scan.test.ts | 279 ++++++++++++++ ...ctured-agent-session-startup-settlement.ts | 195 ++-------- .../structured-agent-session-status-feed.ts | 7 +- ...agent-session-status-journal-projection.ts | 18 +- .../structured-agent-session-subscribers.ts | 10 +- ...ured-agent-session-unexpected-exit.test.ts | 16 +- ...t-session-wedged-profile-migration.test.ts | 42 ++- ...ructured-conversation-command-admission.ts | 4 +- .../agent-session-conversation-receipts.ts | 11 +- ...session-death-evidence-persistence.test.ts | 16 +- .../agent-session-generation-end.test.ts | 271 ++++++++++++++ .../runtime/agent-session-generation-end.ts | 93 +++++ .../agent-session-lease-transitions.ts | 22 +- .../runtime/agent-session-record-founding.ts | 4 +- .../runtime/agent-session-record-store.ts | 44 +-- .../agent-session-reservation-admission.ts | 4 +- ...tured-agent-session-mobile-restore.test.ts | 10 +- .../structured-worker-identity.test.ts | 3 +- ...tructured-agent-session-transport-state.ts | 13 +- ...t-session-transport-state.working.test.tsx | 147 ++++++++ src/shared/agent-session-journal-schemas.ts | 6 +- .../agent-session-lease-adjudication.test.ts | 1 - .../agent-session-lease-adjudication.ts | 5 +- .../agent-session-legacy-handoff-lease.ts | 11 +- .../agent-session-record.test-fixture.ts | 1 - src/shared/agent-session-record.ts | 6 +- src/shared/agent-session-wire.ts | 7 + src/shared/native-chat-types.ts | 5 + .../structured-agent-session-coalescer.ts | 5 +- .../structured-agent-session-live-turn.ts | 35 +- .../structured-agent-session-reducer.ts | 6 + 91 files changed, 2824 insertions(+), 891 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/journal-item-provenance.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts create mode 100644 src/main/runtime/agent-session-generation-end.test.ts create mode 100644 src/main/runtime/agent-session-generation-end.ts create mode 100644 src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 0d359f63de59..fe4ab072e38a 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -111,6 +111,9 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-lease-renewer.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-main-agent-working-agreement.test.ts', @@ -188,6 +191,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/runtime/agent-session-conversation-clear-commit.test.ts', 'src/main/runtime/agent-session-conversation-name-store.test.ts', 'src/main/runtime/agent-session-death-evidence-persistence.test.ts', + 'src/main/runtime/agent-session-generation-end.test.ts', 'src/main/runtime/agent-session-launch-directory.test.ts', 'src/main/runtime/agent-session-launch-env-admission.test.ts', 'src/main/runtime/agent-session-lease-renewal.test.ts', diff --git a/mobile/src/session/mobile-structured-actionable-prompt.test.ts b/mobile/src/session/mobile-structured-actionable-prompt.test.ts index 159418b1889e..c170bc0e50ca 100644 --- a/mobile/src/session/mobile-structured-actionable-prompt.test.ts +++ b/mobile/src/session/mobile-structured-actionable-prompt.test.ts @@ -39,7 +39,7 @@ function question(itemId: string, sequence: number): AgentJournalRenderItem { observedAt: sequence, body: { kind: 'question', - title: 'Which one?', + question: 'Which one?', options: [{ id: 'a', label: 'A' }], resolution: PENDING } diff --git a/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx b/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx index 7a14895fd66e..7b26f7003784 100644 --- a/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx +++ b/mobile/src/session/use-mobile-structured-agent-session-prompt-cancel.test.tsx @@ -266,6 +266,22 @@ describe('mobile structured prompt cancellation', () => { expect(hook.turnId).toBeNull() }) + it("reads Working from the host's answer, and from its rows only when an older host gives none", () => { + // A turn row an agent that has since ended left running: the host says it is not working. + state = { ...state, items: [runningTurn()], working: false } + act(() => { + renderer = create(createElement(Harness, { promptCancelSupported: true })) + }) + expect(hook.isWorking).toBe(false) + act(() => renderer?.unmount()) + const { working: _answered, ...olderHost } = state + state = olderHost + act(() => { + renderer = create(createElement(Harness, { promptCancelSupported: true })) + }) + expect(hook.isWorking).toBe(true) + }) + it('uses the rendered prompt identity when the journal changes before tap', async () => { act(() => { renderer = create(createElement(Harness, { promptCancelSupported: true })) diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index b9871df085a5..4633519adef8 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -7,7 +7,8 @@ import { isStructuredAgentSessionMainAgentWorking } from '../../../src/shared/st import { isFinalAgentSessionReadRefusal } from '../../../src/shared/structured-agent-session-read-refusal' import { isStructuredAgentSessionThinking, - runningStructuredAgentSessionTurnId + runningStructuredAgentSessionTurnId, + structuredAgentSessionHostSaysWorking } from '../../../src/shared/structured-agent-session-live-turn' import { selectStructuredAgentTurnActivity } from '../../../src/shared/native-chat-turn-activity' import { @@ -159,7 +160,10 @@ export function useMobileStructuredAgentSession(args: { const activityText = selectStructuredAgentTurnActivity(state.items, turnId, state.activity)?.text ?? null const thinking = isStructuredAgentSessionThinking(state) - const isWorking = isStructuredAgentSessionMainAgentWorking(turnId, state.submissions, state.fence) + // The host's answer counts only the agent running now; an older host gives none. + const isWorking = structuredAgentSessionHostSaysWorking(state.working, () => + isStructuredAgentSessionMainAgentWorking(turnId, state.submissions, state.fence) + ) const backgroundTasks = useMobileStructuredBackgroundTasks({ sessionKey, state, @@ -231,7 +235,7 @@ export function useMobileStructuredAgentSession(args: { queuePause, submissions: state.submissions, pendingPrompt: approvalPrompt !== null || questionPrompt !== null, - agentWorking: isStructuredAgentSessionMainAgentWorking(turnId, state.submissions, state.fence), + agentWorking: isWorking, mutate, appendComposerText, onSendError, diff --git a/src/main/claude/claude-subagent-resume-after-restart.test.ts b/src/main/claude/claude-subagent-resume-after-restart.test.ts index d95dd3a48ffc..9ac985a160a1 100644 --- a/src/main/claude/claude-subagent-resume-after-restart.test.ts +++ b/src/main/claude/claude-subagent-resume-after-restart.test.ts @@ -18,6 +18,11 @@ import { createClaudeJournalTranslator } from './claude-structured-journal-trans import { claudeSubagentGroupBody, claudeSubagentGroupIdentity } from './claude-subagent-group-row' import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { claudeProviderHandle } from '../../shared/agent-session-provider-handle-encoding' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../shared/agent-session-record.test-fixture' +import { settleStructuredAgentSessionLeftovers } from '../native-chat/agent-session-wire/structured-agent-session-leftover-settlement' // Frame orders are real sessions', scrubbed. A resumed agent's frames still name its ORIGINAL // spawn call while the announcement names the message call that resumed it, and a new provider @@ -138,10 +143,10 @@ async function openJournal(): Promise { } /** One provider process: a fresh sink and translator over the session's journal. */ -function acquire(journal: AgentSessionJournal) { +function acquire(journal: AgentSessionJournal, fence = 1) { const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) const translator = createClaudeJournalTranslator({ sink: deferred.sink, coalesceMs: 0 }) - deferred.bind({ journal, fence: 1, publish: () => {} }) + deferred.bind({ journal, fence, publish: () => {} }) const settle = async (): Promise => { await expect(deferred.drained()).resolves.toEqual({ ok: true }) } @@ -451,16 +456,36 @@ describe('a Claude subagent resumed after its provider restarted', () => { ) first.translator.handle(taskStarted('agent-a', 'toolu_spawn_a', 'Grok PR')) await first.settle() - // The host dies: nothing sweeps the child, and reopening the journal can say only that - // contact was lost, not when. + // The host dies: nothing sweeps the child. The next startup's reconciliation finds its + // generation's lease released with no proof, so it can say only that contact was lost, not when. await crashed.close() const journal = await openJournal() + expect(rowsListing(journal, 'agent-a')[0]?.agents).toEqual([ + expect.objectContaining({ id: 'agent-a', state: 'working', ownerFence: 1 }) + ]) + const released = agentSessionRecordFixture( + agentSessionLeaseFixture({ + sessionId: 'orca-session', + runtimeFence: 2, + claimStatus: 'released', + ownerProcess: null, + reservedSpawnToken: null + }) + ) + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: (sessionId) => (sessionId === 'orca-session' ? released : null) }, + sessionId: 'orca-session', + journal + }) + ).toMatchObject({ ok: true }) const [lost] = rowsListing(journal, 'agent-a')[0]?.agents ?? [] expect(lost).toMatchObject({ id: 'agent-a', state: 'unverifiable' }) expect(lost?.settledAt).toBeUndefined() vi.setSystemTime(5_000_000) - const second = acquire(journal) + // The next start reserves past the released lease. + const second = acquire(journal, 3) second.translator.handle(userTurn('turn-b')) second.translator.handle( frame({ diff --git a/src/main/codex/codex-structured-journal-exit-convergence.test.ts b/src/main/codex/codex-structured-journal-exit-convergence.test.ts index ab8e5a75ab36..187d130e464d 100644 --- a/src/main/codex/codex-structured-journal-exit-convergence.test.ts +++ b/src/main/codex/codex-structured-journal-exit-convergence.test.ts @@ -23,7 +23,6 @@ import { } from '../native-chat/agent-session-wire/structured-agent-session-child-exit' import type { StructuredAgentSessionEndedEvent } from '../native-chat/agent-session-wire/structured-agent-session-adapter' import { StructuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' -import { NO_STORE_RECEIPTS } from '../native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture' import { createCodexJournalTranslator } from './codex-structured-journal-translation' import { handleCodexSessionExit } from './codex-structured-session-close' import type { CodexSession, CodexStructuredSessionEvent } from './codex-structured-session-state' @@ -229,8 +228,7 @@ it('settles a refused Codex exit through the host in one commit, then permits a transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)), - conversationReceipts: NO_STORE_RECEIPTS + ) => (record = transition(record)) }, sessions: new Map([[SESSION, hostSession]]), flushLifecycle: () => deferred.lifecycleBarrier(), diff --git a/src/main/native-chat/agent-session-journal/journal-item-provenance.ts b/src/main/native-chat/agent-session-journal/journal-item-provenance.ts new file mode 100644 index 000000000000..03e2855994e3 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-item-provenance.ts @@ -0,0 +1,137 @@ +// Execution provenance: the generation whose execution produced an item (`JournalItemRow.ownerFence`) +// and each entry of a container row. +// +// Each entry of a container row: every subagent roster entry and every +// background task. One roster row is revised in place across generations (a resumed provider keeps +// the group's identity), so the row's own provenance cannot say which generation each child +// belongs to. A provider's observation stamps the entries it changed or added with its own +// generation; an entry it carries unchanged keeps the one it had (or, never stamped, the row's +// earlier provenance). Host bookkeeping states no provenance and stamps nothing. + +import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' +import type { JournalReducerState } from './journal-reducer' +import { + isBackgroundTaskBlock, + isSubagentGroupBlock, + type NativeChatBlock +} from '../../../shared/native-chat-types' + +type Entry = { ownerFence?: number } + +/** An entry's observed fields, in a fixed order: its provenance is not part of what was seen. */ +function observationKey(entry: Entry): string { + const { ownerFence: _ownerFence, ...fields } = entry + return JSON.stringify(Object.entries(fields).sort(([left], [right]) => left.localeCompare(right))) +} + +function stamp( + entry: T, + previous: T | undefined, + previousItemFence: number | undefined, + ownerFence: number +): T { + if (previous && observationKey(previous) === observationKey(entry)) { + const kept = previous.ownerFence ?? previousItemFence + return kept === undefined || kept === entry.ownerFence ? entry : { ...entry, ownerFence: kept } + } + // No earlier entry to compare: one that already names its generation (history carried into a new + // epoch) keeps it. + if (!previous && entry.ownerFence !== undefined) { + return entry + } + return entry.ownerFence === ownerFence ? entry : { ...entry, ownerFence } +} + +function hasEntries(body: AgentJournalItemBody): boolean { + return ( + body.kind === 'message' && + body.blocks.some((block) => isSubagentGroupBlock(block) || isBackgroundTaskBlock(block)) + ) +} + +/** The body as written by an observation of generation `ownerFence`, each entry stamped. */ +export function stampJournalEntryProvenance( + body: AgentJournalItemBody, + previous: { body: AgentJournalItemBody; ownerFence?: number } | undefined, + ownerFence: number +): AgentJournalItemBody { + if (body.kind !== 'message' || !hasEntries(body)) { + return body + } + const earlier: readonly NativeChatBlock[] = + previous?.body.kind === 'message' ? previous.body.blocks : [] + const blocks = body.blocks.map((block): NativeChatBlock => { + if (isSubagentGroupBlock(block)) { + const group = earlier.find( + (candidate) => isSubagentGroupBlock(candidate) && candidate.groupId === block.groupId + ) + const agents = group && isSubagentGroupBlock(group) ? group.agents : [] + return { + ...block, + agents: block.agents.map((agent) => + stamp( + agent, + agents.find((candidate) => candidate.id === agent.id), + previous?.ownerFence, + ownerFence + ) + ) + } + } + if (isBackgroundTaskBlock(block)) { + const task = earlier.find( + (candidate) => isBackgroundTaskBlock(candidate) && candidate.taskId === block.taskId + ) + return stamp( + block, + task && isBackgroundTaskBlock(task) ? task : undefined, + previous?.ownerFence, + ownerFence + ) + } + return block + }) + return { ...body, blocks } +} + +/** An entry's generation: its own stamp, else the row's provenance. */ +export function journalEntryOwnerFence( + entry: Entry, + itemFence: number | undefined +): number | undefined { + return entry.ownerFence ?? itemFence +} + +/** The provenance a row states: the writer's own claim when it has one — a provider's observation, + * or history carried into a new epoch — else host bookkeeping's, which keeps the item's, and for + * an item it creates is the fence it writes at. Every item row states it. */ +export function journalItemOwnerFence( + state: Pick, + resolvedItemId: string, + write: { fence: number; ownerFence?: number } +): number { + if (write.ownerFence !== undefined) { + return write.ownerFence + } + const kept = state.items.has(resolvedItemId) ? state.itemFences.get(resolvedItemId) : undefined + return kept ?? write.fence +} + +/** A provider's observation stamps each roster entry and background task it saw + * (`stampJournalEntryProvenance`); host bookkeeping states no provenance and stamps nothing. */ +export function journalObservedBody( + state: Pick, + resolvedItemId: string, + body: AgentJournalItemBody, + ownerFence: number | undefined +): AgentJournalItemBody { + if (ownerFence === undefined) { + return body + } + const previous = state.items.get(resolvedItemId) + return stampJournalEntryProvenance( + body, + previous && { body: previous.body, ownerFence: state.itemFences.get(resolvedItemId) }, + ownerFence + ) +} diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index c40e673e948b..99548fa5a81f 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -20,7 +20,6 @@ import type { JournalResolvedLifecycleBatchInput } from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' -import type { JournalOperationReceipt } from './journal-row-writer' import { journalQueuedRejectionRowBuilders } from './journal-pending-submission-recovery' export class JournalLifecycleBatchAppender { @@ -29,8 +28,7 @@ export class JournalLifecycleBatchAppender { state: () => JournalReducerState cursor: () => AgentJournalCursor enqueueRows: ( - plan: () => readonly ((seq: number, ts: number) => JournalRow)[], - receipt?: JournalOperationReceipt + plan: () => readonly ((seq: number, ts: number) => JournalRow)[] ) => Promise } ) {} @@ -77,7 +75,7 @@ export class JournalLifecycleBatchAppender { ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), ...this.planMutations(input, mutations) ] - }, input.receipt) + }) .then((rows) => this.cursorAfter(rows)) } diff --git a/src/main/native-chat/agent-session-journal/journal-row-builders.ts b/src/main/native-chat/agent-session-journal/journal-row-builders.ts index 59a21943484d..229e1656ec5f 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-builders.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-builders.ts @@ -13,6 +13,7 @@ import { agentSessionJournalProviderHandle } from '../../../shared/agent-session import { journalRowSchemaVersion } from '../../../shared/agent-session-journal-types' import { agentJournalLinkageFields } from '../../../shared/agent-session-journal-producer' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { journalItemOwnerFence, journalObservedBody } from './journal-item-provenance' import type { JournalReducerState } from './journal-reducer' import type { JournalDispatchRow, @@ -189,7 +190,15 @@ export function journalLifecycleBatchRowBuilder( const revision = nextJournalItemRevision(current, itemId, runningRevisions) return journalLifecycleMutationRow( mutation.kind === 'item' - ? { ...mutation, body: turnEndAfterStop(current, resolved, mutation.body) } + ? { + ...mutation, + body: journalObservedBody( + current, + resolved, + turnEndAfterStop(current, resolved, mutation.body), + mutation.ownerFence + ) + } : mutation, itemId, revision, @@ -229,21 +238,6 @@ export function journalRowBase( return { v: journalRowSchemaVersion(bodies), epoch, seq, fence, ts } } -/** The provenance a row states: the writer's own claim when it has one — a provider's observation, - * or history carried into a new epoch — else host bookkeeping's, which keeps the item's, and for - * an item it creates is the fence it writes at. Every item row states it. */ -export function journalItemOwnerFence( - state: Pick, - resolvedItemId: string, - write: { fence: number; ownerFence?: number } -): number { - if (write.ownerFence !== undefined) { - return write.ownerFence - } - const kept = state.items.has(resolvedItemId) ? state.itemFences.get(resolvedItemId) : undefined - return kept ?? write.fence -} - export function buildJournalItemRow( input: JournalItemAddress & { state: JournalReducerState @@ -264,7 +258,12 @@ export function buildJournalItemRow( // A tombstoned row keeps its revision in `tombstones`, and the reducer drops // any item at or below it — so a re-add has to outrank the tombstone too. const revision = nextJournalItemRevision(input.state, itemId, input.revisions) - const body = turnEndAfterStop(input.state, resolved, input.body) + const body = journalObservedBody( + input.state, + resolved, + turnEndAfterStop(input.state, resolved, input.body), + input.ownerFence + ) return { kind: 'item', itemId, diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 2deac0e645cc..4eb2ff6bdd8e 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -85,8 +85,7 @@ export class JournalRowWriter { return this.deps.serialize(() => this.writeRows(plan, receipt)) } - /** `enqueueRows`' write, for a caller already running at its own turn in the queue. A receipt - * commits even when the plan is empty: what it records is already true of the rows. */ + /** `enqueueRows`' write, for a caller already running at its own turn in the queue. */ writeRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[], receipt?: JournalOperationReceipt @@ -95,7 +94,7 @@ export class JournalRowWriter { const first = this.deps.nextSequence() const ts = this.deps.now() const rows = plan().map((build, index) => build(first + index, ts)) - if (rows.length === 0 && !receipt) { + if (rows.length === 0) { return rows } for (const row of rows) { @@ -103,9 +102,7 @@ export class JournalRowWriter { } try { this.deps.database().transaction((db) => { - if (rows.length > 0) { - this.deps.founding?.inTransaction(db) - } + this.deps.founding?.inTransaction(db) for (const row of rows) { insertJournalRow(db, this.deps.sessionId, row) this.runBookkeeping(db, row) @@ -116,11 +113,9 @@ export class JournalRowWriter { this.deps.rolledBack?.() throw error } + this.deps.founding?.settled() receipt?.committed() - if (rows.length > 0) { - this.deps.founding?.settled() - this.deps.commit(rows) - } + this.deps.commit(rows) return rows } diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 790201d56f8a..1d1ced079f77 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -118,7 +118,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal const lifecycleBatchAppender = new JournalLifecycleBatchAppender({ state: host.state, cursor: host.cursor, - enqueueRows: (plan, receipt) => rowWriter.enqueueRows(plan, receipt) + enqueueRows: (plan) => rowWriter.enqueueRows(plan) }) return { epochController, diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index c8f4d117edcc..5f16fcfc4f06 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -16,7 +16,6 @@ import type { AgentSessionMessageSource } from '../../../shared/agent-session-me import type { JournalHostDatabase } from './journal-host-database' import type { JournalLifecycleMutationInput } from './journal-row-builders' import type { JournalRow } from './journal-row-schema' -import type { JournalOperationReceipt } from './journal-row-writer' export type AgentSessionJournalOptions = { identity: AgentSessionJournalIdentity @@ -102,8 +101,6 @@ export type JournalPlannedLifecycleBatchInput = Pick< mutations: readonly JournalLifecycleMutationInput[] dispatches: readonly ResolveDispatchInput[] } - /** Committed in the same transaction as the rows, even when the plan is empty. */ - receipt?: JournalOperationReceipt } export type JournalResolvedLifecycleBatchInput = Omit< diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index f47415804f8a..1363b12536d9 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -59,7 +59,7 @@ import { journalQueueReopenRowBuilder, journalStopEventRowBuilder } from './journal-stop-and-resume-rows' -import type { AgentJournalEpochReason, JournalStopEvent } from './journal-row-schema' +import type { AgentJournalEpochReason, JournalRow, JournalStopEvent } from './journal-row-schema' import type { JournalOperationReceipt, JournalRowTransactionHook, @@ -86,7 +86,7 @@ export class AgentSessionJournal { private state: JournalReducerState private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } private reopenUnmarked: AgentJournalCursor | null = null - private onCommitted: (() => void) | null = null + private onCommitted: ((rows?: readonly JournalRow[]) => void) | null = null private readonly queue: JournalWriteQueue private readonly rowWriter: JournalRowWriter private readonly epochController: JournalEpochController @@ -126,7 +126,7 @@ export class AgentSessionJournal { for (const row of rows) { applyJournalRow(this.state, row) } - this.onCommitted?.() + this.onCommitted?.(rows) }, notifyCommitted: () => this.onCommitted?.(), journal: () => this, @@ -166,6 +166,9 @@ export class AgentSessionJournal { ) } + /** Where this handle opened the journal (`wroteBeforeOpen`). */ + openedAt = (): AgentJournalCursor => this.openedThrough + /** Where the reopen's pause begins when this handle could not write its mark: where the mark * would have gone. Null once a mark is written. Per handle, so the next open marks again. */ reopenFloor(): AgentJournalCursor | null { @@ -186,7 +189,7 @@ export class AgentSessionJournal { /** Told of every durable change, epoch replacements included, so a reader learns of a write * without its writer saying so. One listener: a later call replaces it. It must not throw. */ - observeCommits(listener: () => void): void { + observeCommits(listener: (rows?: readonly JournalRow[]) => void): void { this.onCommitted = listener } diff --git a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts index 41dcc19f2a73..073c2559308c 100644 --- a/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts +++ b/src/main/native-chat/agent-session-journal/journal-subagent-liveness.ts @@ -11,7 +11,8 @@ // // Reconciles JOURNAL ROWS, not roster state. A producer that continues a roster // from the journal reads these rows after this revision, so it inherits the -// verdict rather than the dead host's `working`. +// verdict rather than the dead host's `working`. Each entry is settled only when the +// generation that last observed it is the one that ended (`journal-item-provenance.ts`). import type { AgentJournalItemBody } from '../../../shared/agent-session-journal-types' import { @@ -32,30 +33,50 @@ import { type NativeChatSubagentGroupBlock } from '../../../shared/native-chat-types' +/** Which entries the lost session ran, by each entry's own provenance + * (`journal-item-provenance.ts`); absent, every entry. */ +export type LostLiveWorkEntries = (entry: { ownerFence?: number }) => boolean + +const EVERY_ENTRY: LostLiveWorkEntries = () => true + /** The row once the host lost the session running its live work: every working child and - * in-flight background task `unverifiable`, plain-text twins restated. Null when none is live. */ -export function lostLiveWorkJournalBody(body: AgentJournalItemBody): AgentJournalItemBody | null { - if (body.kind !== 'message' || !body.blocks.some(hasStaleLiveWork)) { + * in-flight background task `unverifiable`, plain-text twins restated. Null when none is live. + * `lost` narrows it to the entries the lost session ran: a later generation's stay as they are. */ +export function lostLiveWorkJournalBody( + body: AgentJournalItemBody, + lost: LostLiveWorkEntries = EVERY_ENTRY +): AgentJournalItemBody | null { + if ( + body.kind !== 'message' || + !body.blocks.some((block) => hasWorkingChild(block, lost) || hasLiveBackgroundTask(block, lost)) + ) { return null } - return { ...body, blocks: settleBlocks(body.blocks) } + return { ...body, blocks: settleBlocks(body.blocks, lost) } } -function hasStaleLiveWork(block: NativeChatBlock): boolean { - return hasWorkingChild(block) || hasLiveBackgroundTask(block) +function isWorkingChild(agent: NativeChatSubagentGroupBlock['agents'][number]): boolean { + return normalizeSubagentState(agent.state) === 'working' } -function hasWorkingChild(block: NativeChatBlock): block is NativeChatSubagentGroupBlock { +function hasWorkingChild( + block: NativeChatBlock, + lost: LostLiveWorkEntries +): block is NativeChatSubagentGroupBlock { return ( isSubagentGroupBlock(block) && - block.agents.some((agent) => normalizeSubagentState(agent.state) === 'working') + block.agents.some((agent) => isWorkingChild(agent) && lost(agent)) ) } -function hasLiveBackgroundTask(block: NativeChatBlock): block is NativeChatBackgroundTaskBlock { +function hasLiveBackgroundTask( + block: NativeChatBlock, + lost: LostLiveWorkEntries +): block is NativeChatBackgroundTaskBlock { return ( isBackgroundTaskBlock(block) && - !isSettledBackgroundTaskState(normalizeBackgroundTaskState(block.state)) + !isSettledBackgroundTaskState(normalizeBackgroundTaskState(block.state)) && + lost(block) ) } @@ -63,13 +84,16 @@ function hasLiveBackgroundTask(block: NativeChatBlock): block is NativeChatBackg * stamping the reopen would report the time the app was down as how long it * ran. Readers already draw an unverifiable child with no stamp as having no * known run length. */ -function settleBlocks(blocks: readonly NativeChatBlock[]): NativeChatBlock[] { +function settleBlocks( + blocks: readonly NativeChatBlock[], + lost: LostLiveWorkEntries +): NativeChatBlock[] { const backgroundTaskTwinText = new Map() const settled = blocks.map((block) => { - if (hasWorkingChild(block)) { - return settleGroup(block) + if (hasWorkingChild(block, lost)) { + return settleGroup(block, lost) } - if (hasLiveBackgroundTask(block)) { + if (hasLiveBackgroundTask(block, lost)) { const next = settleBackgroundTask(block) backgroundTaskTwinText.set( backgroundTaskFallbackText(block), @@ -99,13 +123,14 @@ function settleBlocks(blocks: readonly NativeChatBlock[]): NativeChatBlock[] { ) } -function settleGroup(block: NativeChatSubagentGroupBlock): NativeChatSubagentGroupBlock { +function settleGroup( + block: NativeChatSubagentGroupBlock, + lost: LostLiveWorkEntries +): NativeChatSubagentGroupBlock { return { ...block, agents: block.agents.map((agent) => - normalizeSubagentState(agent.state) === 'working' - ? { ...agent, state: 'unverifiable' as const } - : agent + isWorkingChild(agent) && lost(agent) ? { ...agent, state: 'unverifiable' as const } : agent ) } } diff --git a/src/main/native-chat/agent-session-record-test-fixture.ts b/src/main/native-chat/agent-session-record-test-fixture.ts index 485bdea37b01..21e7f1d5191e 100644 --- a/src/main/native-chat/agent-session-record-test-fixture.ts +++ b/src/main/native-chat/agent-session-record-test-fixture.ts @@ -36,8 +36,7 @@ export function agentSessionRecordFixture( claimKeyId: 'test-key', claimStatus: 'released', unreconciled: false, - deathEvidence: null, - leftoverSettledAt: null + deathEvidence: null }, ...rest, location: { diff --git a/src/main/native-chat/agent-session-wire/agent-session-history-page.ts b/src/main/native-chat/agent-session-wire/agent-session-history-page.ts index e99fe7af5c74..d2714ff3cbf5 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-history-page.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-history-page.ts @@ -43,7 +43,7 @@ import type { StructuredAgentSessionCurrentWork } from './structured-agent-sessi * Absent, a page states the raw record and no prompt ids, as an older host did. */ export type AgentSessionPublishedWork = Pick< StructuredAgentSessionCurrentWork, - 'publishedLatestTurn' | 'actionablePromptIds' + 'publishedLatestTurn' | 'actionablePromptIds' | 'working' > export { AGENT_SESSION_HISTORY_MAX_PAGE_BYTES } from './agent-session-history-page-bounds' @@ -164,10 +164,11 @@ export function readAgentSessionHydrationPage( export function agentSessionPublishedWorkFields( journal: AgentSessionJournal, work: AgentSessionPublishedWork -): Pick { +): Pick { return { latestTurn: work.publishedLatestTurn(snapshotLatestTurn(journal.snapshot())), - actionablePromptIds: work.actionablePromptIds() + actionablePromptIds: work.actionablePromptIds(), + working: work.working() } } @@ -346,7 +347,8 @@ function buildPage(input: { ...(input.work ? { latestTurn: input.work.publishedLatestTurn(snapshotLatestTurn(input.snapshot)), - actionablePromptIds: input.work.actionablePromptIds() + actionablePromptIds: input.work.actionablePromptIds(), + working: input.work.working() } : { latestTurn: snapshotLatestTurn(input.snapshot) }) } diff --git a/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts b/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts index 9f7c600c4fc6..668c433de9e0 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-subscriber-catch-up.ts @@ -83,7 +83,7 @@ export function deliverToSubscriber( ...shared }) subscriber.cursor = page.liveCursor ?? page.window.nextCursor - subscriber.workLiveFence = work?.liveFence + subscriber.workView = work?.viewKey() return } const page = result.page @@ -108,6 +108,7 @@ export function deliverToSubscriber( ...(page.actionablePromptIds !== undefined ? { actionablePromptIds: page.actionablePromptIds } : {}), + ...(page.working !== undefined ? { working: page.working } : {}), ...shared }, // On a multi-page catch-up the draft list rides only the final page, or a @@ -115,7 +116,7 @@ export function deliverToSubscriber( { withholdQueued: page.hasNewer } ) subscriber.cursor = page.window.nextCursor - subscriber.workLiveFence = work?.liveFence + subscriber.workView = work?.viewKey() if (!page.hasNewer || !port.isActive(subscriber)) { return } @@ -136,9 +137,10 @@ function emitCaughtUp( port.hooks.readCommands !== undefined && (port.hooks.readCommands(subscriber.sessionId) ?? null) !== subscriber.commands const queuedChanged = subscriberQueuedMessagesChanged(port.hooks, subscriber) - // A generation that ended (or began) with no row changes what is current: its frame says so. + // A generation that ended (or began) with no row, or an exit this host saw with no lease write, + // changes what is current: the caught-up frame restates the view. const { work } = current - const workChanged = work !== undefined && work.liveFence !== subscriber.workLiveFence + const workChanged = work !== undefined && work.viewKey() !== subscriber.workView if ( emitCheckpoint || shared.activity !== undefined || @@ -155,7 +157,7 @@ function emitCaughtUp( ...shared }) if (workChanged) { - subscriber.workLiveFence = work.liveFence + subscriber.workView = work.viewKey() } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index 78567db9f2c2..965925432a5e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -50,7 +50,7 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-hand import { StructuredAgentArgumentsError } from '../structured-agent-arguments-error' import { USER_MESSAGE_SOURCE } from '../../../shared/agent-session-message-source' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' -import { startUpOwingLeftovers } from './structured-agent-session-leftover-settlement.test-fixture' +import { startUpHost } from './structured-agent-session-leftover-settlement.test-fixture' const CALLER = { callerKey: 'client-1' } @@ -558,7 +558,7 @@ describe('what an earlier host process left behind', () => { await writeAsEarlierProcess(async (journal, fence) => { await journal.appendSubmission({ ...earlierSubmission('queued', 'q', true), fence }) }) - await startUpOwingLeftovers(host, store) + await startUpHost(host) expect(await submission('queued')).toMatchObject({ dispatchState: 'rejected', @@ -584,7 +584,7 @@ describe('what an earlier host process left behind', () => { }) await host.flushAllStreamedEvents() await startHost() - await startUpOwingLeftovers(host, store) + await startUpHost(host) expect(await submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) expect(await submission('handed')).toMatchObject({ dispatchState: 'unknown', recovered: true }) @@ -685,7 +685,7 @@ describe('Stop withdraws what is queued', () => { }) await journal.appendSubmission({ ...earlierSubmission('leftover', 'l', true), fence }) }) - await startUpOwingLeftovers(host, store) + await startUpHost(host) expect(await stop()).toMatchObject({ ok: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts deleted file mode 100644 index 3ca7851f0592..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-acquisition-settlement.ts +++ /dev/null @@ -1,72 +0,0 @@ -// An acquisition's share of the leftover settlement (`structured-agent-session-leftover-settlement.ts`): -// before the reservation, reading the lease directly, while it still holds the proof of how the -// last generation ended. The reservation clears that proof, so after it the same work could only -// settle `unverifiable`. Best effort: a failure is logged and the start goes on. - -import { - agentSessionLeaseIsReleased, - isProvenDeadProbe, - type AgentSessionOwnerProbe -} from '../../../shared/agent-session-lease-adjudication' -import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' -import { settleStructuredAgentSessionLeftoversOrLog } from './structured-agent-session-leftover-settlement' - -const SCOPE = 'acquisition-settlement' - -export async function settleStructuredAgentSessionLeftoversBeforeReserve( - context: Pick< - StructuredAgentSessionAttachContext, - 'deps' | 'sessions' | 'openConversation' | 'now' | 'runtimeState' - >, - sessionId: string, - probe: AgentSessionOwnerProbe -): Promise { - const { store, logger } = context.deps - const lease = store.getRecord(sessionId)?.lease - if (!lease) { - return - } - // A recorded owner the probe proved gone is ended by the reservation that follows; one alive, - // unproven, or not yet reconciled keeps its work, which is still its own. - const ownerProvenGone = - !lease.unreconciled && - lease.claimStatus !== 'conflicted' && - lease.ownerProcess !== null && - isProvenDeadProbe(probe) - if (!agentSessionLeaseIsReleased(lease) && !ownerProvenGone) { - return - } - if (ownerProvenGone && context.sessions.get(sessionId)?.child) { - // A child of this process that died unseen: what it wrote lands before its work is judged. - const barrier = await context.runtimeState.currentEventSink(sessionId)?.drained() - if (barrier && !barrier.ok) { - logger.warn("settling a gone agent's leftover work did not finish", { - scope: SCOPE, - sessionId, - error: barrier.error - }) - return - } - } - const conversation = await context.openConversation(sessionId).catch((error: unknown) => { - // The attach's own open meets the same failure and answers it. - logger.warn("settling a gone agent's leftover work did not finish", { - scope: SCOPE, - sessionId, - error - }) - return null - }) - if (!conversation) { - return - } - await settleStructuredAgentSessionLeftoversOrLog({ - store, - sessionId, - journal: conversation.journal, - now: context.now, - logger, - scope: SCOPE, - ...(ownerProvenGone ? { ownerProvenGone: true as const } : {}) - }) -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts index b330893965f9..93d52511aeef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-orchestration.ts @@ -22,7 +22,6 @@ import { pinnedAgentSessionLaunchEnv } from './structured-agent-session-launch-env' import { refuseAgentSessionMutation } from './structured-agent-session-mutation-admission' -import { settleStructuredAgentSessionLeftoversBeforeReserve } from './structured-agent-session-acquisition-settlement' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { @@ -147,9 +146,6 @@ async function runAttachUnderAbort( const probe = await withAgentSessionCreatePhase('probe_owner', recordPhase, () => context.runtimeState.probeOwner(sessionId) ) - // Before the reserve wipes the proof of how the last generation ended: best effort, so a - // settlement that cannot commit never stops this start. - await settleStructuredAgentSessionLeftoversBeforeReserve(context, sessionId, probe) const priorRecord = context.deps.store.getRecord(sessionId) const launchDirectory = !priorRecord && isFloatingWorkspaceId(params.location.workspaceId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index 722450bbb320..4af996896938 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -7,7 +7,7 @@ import type { AgentChildWorkView } from '../../../shared/agent-status-child-work import { structuredChildWorkLegacyTasks } from '../../../shared/structured-agent-session-child-work-legacy' import { structuredRunningChildWork } from '../../../shared/agent-child-work-listing' import { readStructuredAgentSessionHistoryResult } from './structured-agent-session-history-result' -import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { structuredQueueSendGate, tryReadQueuePublication diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 553ea598107f..e11315074089 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -22,7 +22,7 @@ import { import type { StructuredAgentSessionSinkBarrier } from './structured-agent-session-event-sink' import { settleStructuredAgentSessionLeftovers, - type StructuredAgentSessionLeftoverSettlementInput, + type StructuredAgentSessionExitSettlement, type StructuredAgentSessionLeftoverStore } from './structured-agent-session-leftover-settlement' import type { StaleStructuredAgentSessionStateJournal } from './structured-agent-session-stale-state-settlement' @@ -53,11 +53,12 @@ export type StructuredAgentSessionChildExitSession = Pick< 'child' | 'lastEndedChild' > & { journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal } -/** `StructuredAgentSessionClientDelivery.publishGenerationEnded`: the one edge every way a - * generation ends goes through. */ +/** The chat's reconciliation worker's signal (`StructuredAgentSessionReconciliation.signal`): the + * one edge every way a generation ends goes through. `exit`: the exit's own settlement, owed when + * it did not land. */ export type StructuredAgentSessionGenerationEnded = ( sessionId: string, - options?: { restate?: boolean } + options?: { restate?: boolean; exit?: StructuredAgentSessionExitSettlement } ) => void export type StructuredAgentSessionChildExitContext< @@ -67,9 +68,9 @@ export type StructuredAgentSessionChildExitContext< sessions: Map flushLifecycle: (sessionId: string) => Promise publishStatus?: (sessionId: string) => void - /** A generation ended: every reader re-derives current work and the queued-card drain is - * scheduled, with no journal write needed (`publishGenerationEnded`). `restate`: each chat - * re-baselines at the moved fence. */ + /** A generation ended: every reader re-derives current work, the queued-card drain is + * scheduled with no journal write needed, and the chat's reconciliation worker settles what is + * owed. `restate`: each chat re-baselines at the moved fence. */ generationEnded: StructuredAgentSessionGenerationEnded /** The delivery loop hands over whatever is queued once the child is off the record. */ wakeDelivery?: (sessionId: string) => void @@ -183,7 +184,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< context.wakeDelivery?.(sessionId) return } - let settlement: StructuredAgentSessionLeftoverSettlementInput['exit'] | null = null + let settlement: StructuredAgentSessionExitSettlement | null = null try { // The exited child's own writes land first: its dead generation is settled from all of them. try { @@ -239,9 +240,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< } } finally { // The root's exit was observed, so the owner is released first, writing the proof and ending - // the generation; then what it left is settled at the new fence, with the mark that says so. - // A settlement that cannot commit stays owed: the next acquisition or startup settles it. + // the generation; then what it left is settled at the new fence. A release or a settlement + // that cannot commit stays owed: the chat's reconciliation worker retries it. let released = false + let owed: StructuredAgentSessionExitSettlement | null = null try { await releaseStoredStructuredAgentSessionOwnerAfterExit({ store: context.store, @@ -260,10 +262,12 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< store: context.store, sessionId, journal: session.journal, - now: context.now, - exit: settlement + exit: settlement, + // Ended whether or not the release landed: the host saw the root go. + ended: { fence: child.fence, rootGone: true } }) if (!settled.ok) { + owed = settlement logExitFailure(context, sessionId, 'exit-settlement', settled.error) } } @@ -282,7 +286,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // reader holds, as a client resends a message when its fence moves, and discards a reply (the // Stop's own) issued at the fence before. Either way every reader learns the generation ended // and the queued-card drain runs, whether or not the release or the settlement was written. - context.generationEnded(sessionId, { restate: released && !expected }) + context.generationEnded(sessionId, { + restate: released && !expected, + ...(owed ? { exit: owed } : {}) + }) context.wakeDelivery?.(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index 00a0644136c3..af06ae1ff2e5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -397,9 +397,9 @@ it('sends after a proven exit whose resume-point write and lease release both fa expect.arrayContaining(['claude-close-resume-point', 'exit-owner-release']) ) ) - expect(lease()?.claimStatus).toBe('live') + // The chat's worker writes the release from this host's proof of that exit; no send waits on it. + await eventually(() => expect(lease()?.claimStatus).toBe('released')) - // The start writes the release from this host's proof of that exit. await send('Carry on.') await resumedWith(connection, 'Carry on.') }) @@ -419,7 +419,8 @@ it('sends after a crash with a long reason whose own lease release failed', asyn ) await eventually(() => expect(child()).toBeNull()) await eventually(() => expect(scopes()).toContain('exit-owner-release')) - expect(lease()?.claimStatus).toBe('live') + // The chat's worker writes the release the exit's own write could not. + await eventually(() => expect(lease()?.claimStatus).toBe('released')) await send('Carry on.') await resumedWith(connection, 'Carry on.') @@ -526,7 +527,8 @@ it('has the idle reaper join a close still unverifiable, ending the record with expect(connection.closeCount).toBe(2) expect(lease()).toMatchObject({ claimStatus: 'released', ownerProcess: null }) - expect(host.hasSession(SESSION)).toBe(false) + // Closed once the chat's worker has settled what the ended child left. + await eventually(() => expect(host.hasSession(SESSION)).toBe(false)) }) it("ends a Claude journal-sink failure as Orca's own fault, never a quiet rest", async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index c4ebfa5a962c..92d468ca6b94 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -189,10 +189,12 @@ export class StructuredAgentSessionClientDelivery { /** The current-work projection over this journal (`structuredAgentSessionCurrentWork`). */ private currentWork(sessionId: string, journal: AgentSessionJournal) { - const ended = this.sessions.get(sessionId)?.lastEndedChild + const session = this.sessions.get(sessionId) + const ended = session?.lastEndedChild return structuredAgentSessionCurrentWork(journal, { record: this.deps().store.getRecord(sessionId), - ...(ended ? { ended } : {}) + ...(ended ? { ended } : {}), + revision: session?.operationalRevision ?? 0 }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 4211929805cc..e6698a7d10b2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -44,6 +44,8 @@ export function createStructuredAgentSessionConversationLifetime(host: { closeStatus: (sessionId: string, options: { listed: boolean }) => void /** The session's child records, the host's one read of them. */ readChildWork: (sessionId: string) => readonly AgentChildWorkView[] | undefined + /** `StructuredAgentSessionReconciliation.deferCloseAtRest`. */ + deferCloseAtRest: (sessionId: string) => boolean }) { let disposed = false const { sessions, serialize } = host @@ -59,6 +61,10 @@ export function createStructuredAgentSessionConversationLifetime(host: { if (await releaseLeaseOfEndedStructuredAgentSessionChild(host.context(), sessionId)) { return false } + // At rest it waits for the chat's reconciliation, which would only open it again. + if (atRest && host.deferCloseAtRest(sessionId)) { + return false + } const closed = await closeStructuredAgentSessionConversationUnderSerialize( { sessions, @@ -106,6 +112,9 @@ export function createStructuredAgentSessionConversationLifetime(host: { return { idleSweep, + /** The idle sweep's close, for a conversation only the reconciliation worker opened. */ + closeAtRest: (sessionId: string): Promise => + serialize(sessionId, () => closeConversation(sessionId, true)), stopAgent, /** A host stop of the child the expired attempt published, if it is still that one and still * starting: its end fails what is queued, as the idle sweep's stop of a start does. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts index 94d08c054c51..e3b808989089 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.test.ts @@ -67,7 +67,7 @@ describe('a conversation delivers what its journal commits', () => { // Already delivered when the writer's await returns, as an explicit publish would have been. expect(deliver).toHaveBeenCalledOnce() - expect(deliver).toHaveBeenCalledWith('session-1', journal) + expect(deliver).toHaveBeenCalledWith('session-1', journal, { lowestFence: 0 }) }) it('binds a handle set through a plain map reference', async () => { @@ -84,7 +84,7 @@ describe('a conversation delivers what its journal commits', () => { await appendStatus(journal, 'through the plain map') - expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal) + expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal, expect.anything()) }) it('delivers an epoch replacement, which readers must reload from', async () => { @@ -99,7 +99,7 @@ describe('a conversation delivers what its journal commits', () => { await journal.replaceEpochItems('handle_forked', 0, []) - expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal) + expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', journal, expect.anything()) }) it('delivers nothing for a handle the conversation has replaced', async () => { @@ -118,7 +118,7 @@ describe('a conversation delivers what its journal commits', () => { expect(deliver).not.toHaveBeenCalled() await appendStatus(current, 'live') - expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', current) + expect(deliver).toHaveBeenCalledExactlyOnceWith('session-1', current, expect.anything()) }) it('delivers nothing once the conversation is dropped', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts index b4ee124597f6..745c33f8d9dd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts @@ -21,7 +21,12 @@ export class StructuredAgentSessionConversations extends Map< constructor( private readonly delivery: { - deliver: (sessionId: string, journal: AgentSessionJournal) => void + /** `lowestFence`: the lowest fence a row committed since the last delivery was written at. */ + deliver: ( + sessionId: string, + journal: AgentSessionJournal, + committed: { lowestFence: number | null } + ) => void /** A person's Stop settle opened or closed: no row, so neither a publish nor activity. */ deliverSettleEdge?: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger @@ -36,18 +41,24 @@ export class StructuredAgentSessionConversations extends Map< override set(sessionId: string, session: StructuredAgentSessionHostSession): this { const { journal } = session let queued = false - journal.observeCommits(() => { + let lowestFence: number | null = null + journal.observeCommits((rows) => { + for (const row of rows ?? []) { + lowestFence = Math.min(lowestFence ?? row.fence, row.fence) + } if (queued) { return } queued = true queueMicrotask(() => { queued = false + const committed = { lowestFence } + lowestFence = null if (this.get(sessionId)?.journal !== journal) { return } try { - this.delivery.deliver(sessionId, journal) + this.delivery.deliver(sessionId, journal, committed) } catch (error) { this.delivery.logger.warn('delivering a journal commit failed', { scope: 'journal-delivery', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index 727e9108289a..98b097b72fec 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -102,8 +102,7 @@ function crashedClaudeRecord(): AgentSessionRecord { claimKeyId: 'key-1', claimStatus: 'live', unreconciled: false, - deathEvidence: null, - leftoverSettledAt: null + deathEvidence: null } } } @@ -204,6 +203,7 @@ describe('a turn a crash cut short mid-tool', () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) await host.restoreReadableSessions() + await reconciled() expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ kind: 'pid-absent', @@ -226,6 +226,7 @@ describe('a turn a crash cut short mid-tool', () => { openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }) }) await host.restoreReadableSessions() + await reconciled() const { items } = await host.journalSnapshot(SESSION) // As a reader's transcript shows it: the stored row is the explanation, so none is derived. @@ -262,6 +263,7 @@ describe('a turn a crash cut short mid-tool', () => { openHost({ probeOwner, stopOwnerProcess }) await host.restoreReadableSessions() + await reconciled() expect(stopOwnerProcess).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease.deathEvidence).toMatchObject({ @@ -296,15 +298,22 @@ function attach(fence: number) { ) } -/** Runs whatever this session's serialize already has queued. */ -function drainSession(): Promise { - return host.collaboratorsForTests().serialize(SESSION, async () => {}) +/** Runs whatever this session's serialize already has queued, then the chat's reconciliation. */ +async function drainSession(): Promise { + await host.collaboratorsForTests().serialize(SESSION, async () => {}) + await reconciled() } -/** Startup: the reconcile, then the settlement it begins. */ +/** Once the chat's reconciliation worker settled what a generation's end left. */ +function reconciled(): Promise { + return host.collaboratorsForTests().reconciliation.idle(SESSION) +} + +/** Startup: the reconcile, then the scan it begins. */ async function startUp(): Promise { await host.reconcileRestartLeases() await host.startupSettled() + await reconciled() } // On desktop the chat on screen at relaunch opens before the startup reconcile has probed its owner. @@ -505,25 +514,30 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ]) }) - it('stays as it was when the startup settlement cannot be written, and the next trigger settles it', async () => { + it('stays as it was when the startup settlement cannot be written, and the worker retries it', async () => { const log = recordingStructuredAgentSessionLogger() openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }), logger: log.logger }) await host.history({ sessionId: SESSION, direction: 'tail' }) const { journal } = host.collaboratorsForTests().sessions.get(SESSION)! - vi.spyOn(journal, 'appendPlannedLifecycleBatch').mockRejectedValueOnce(new Error('disk full')) + const full = vi + .spyOn(journal, 'appendPlannedLifecycleBatch') + .mockRejectedValue(new Error('disk full')) await startUp() - expect(log.scopes()).toEqual(['startup-settlement']) + expect(new Set(log.scopes())).toEqual(new Set(['reconciliation'])) expect(await settledTurn()).toEqual(RUNNING_TURN) - expect(store.getRecord(SESSION)?.lease.leftoverSettledAt).toBeNull() - // The proof is durable on the record and the mark still owes it: the restore's startup share - // converges, as would the next acquisition. - await host.restoreReadableSessions() - expect(await settledTurn()).toMatchObject({ - state: 'interrupted', - completedAt: LAST_RENEWED_AT - }) + full.mockRestore() + // Nothing stored says it is owed: the worker re-derives it after its backoff, from the proof + // the record holds. + await vi.waitFor( + async () => + expect(await settledTurn()).toMatchObject({ + state: 'interrupted', + completedAt: LAST_RENEWED_AT + }), + { timeout: 10_000 } + ) }) }) @@ -587,8 +601,9 @@ const PROVEN_TURN = { completedAt: LAST_RENEWED_AT } -// The relaunch proved the fence-13 owner gone, and startup settled the turn by that proof before any -// start; what the record holds after a start reserving fence 15 fails is about the start's own child. +// The relaunch proved the fence-13 owner gone, and the chat's worker settles the turn by that proof, +// held across the starts that clear it from the record; what the record holds after a start +// reserving fence 15 fails is about the start's own child. describe('a turn a newer start could not settle before it failed', () => { it('is not judged by the death of the child it left for recovery', async () => { const { attach, advance } = await hostWithFailingFirstStart( @@ -609,6 +624,8 @@ describe('a turn a newer start could not settle before it failed', () => { lastProvenAliveAt: RELAUNCHED_AT }) await expect(attach(16)).resolves.toMatchObject({ ok: true }) + await host.startupSettled() + await reconciled() expect(await settledTurn()).toEqual(PROVEN_TURN) }) @@ -624,6 +641,8 @@ describe('a turn a newer start could not settle before it failed', () => { }) advance(60_000) await expect(attach(16)).resolves.toMatchObject({ ok: true }) + await host.startupSettled() + await reconciled() // Never ended at the failed start, an hour after the crash, with the start's reason. expect(await settledTurn()).toEqual(PROVEN_TURN) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts index d0ac1f96eb04..d27d29e64128 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts @@ -237,7 +237,8 @@ describe('an observed exit whose release write failed', () => { vi.spyOn(current.store, 'transitionHandoff').mockImplementation((sessionId, change) => transition(sessionId, (record) => { const next = change(record) - if (refused === 0 && next.lease.claimStatus === 'released') { + // The exit's own release, and the chat worker's first repair of it. + if (refused < 2 && next.lease.claimStatus === 'released') { refused += 1 throw new Error('disk full') } @@ -258,8 +259,9 @@ describe('an observed exit whose release write failed', () => { acquisitionGeneration: child.generation }) await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + await current.host.collaboratorsForTests().reconciliation.idle(SESSION) - expect(refused).toBe(1) + expect(refused).toBe(2) // The release never landed: the lease still names the gone child. expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index efca4ba2c835..2edd4704c04f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -23,10 +23,6 @@ import type { AgentSessionLatestTurn } from '../../../shared/agent-session-wire' import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' import { isUnansweredStructuredAgentSessionDispatch } from '../../../shared/structured-agent-session-unanswered-dispatch' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { - StructuredAgentSessionEndedChild, - StructuredAgentSessionHostSession -} from './structured-agent-session-host-types' export type StructuredAgentSessionCurrentWorkJournal = Pick< AgentSessionJournal, @@ -37,7 +33,9 @@ export type StructuredAgentSessionCurrentWorkJournal = Pick< export type StructuredAgentSessionWorkEvidence = { record: Pick | null /** The last child this host saw end (`StructuredAgentSessionHostSession.lastEndedChild`). */ - ended?: Pick + ended?: { fence: number; rootGone?: boolean } + /** `StructuredAgentSessionHostSession.operationalRevision`. */ + revision?: number } /** The live or reserved generation's fence, or null when no generation is live. With no record @@ -81,9 +79,18 @@ export class StructuredAgentSessionCurrentWork { constructor( private readonly journal: StructuredAgentSessionCurrentWorkJournal, /** `structuredAgentSessionLiveFence`. */ - readonly liveFence: number | null + readonly liveFence: number | null, + /** Moves with every generation end this host saw, lease write or not: a reader that caches + * what it derived from this answer keys it here and on the journal's cursor. */ + readonly revision = 0 ) {} + /** What a reader that published this answer compares to learn it may have changed with no row: + * the live generation and the revision of what this host saw end. */ + viewKey(): string { + return `${this.liveFence}:${this.revision}` + } + /** Whether the live generation's execution produced this item. */ isCurrentItem(itemId: string): boolean { if (this.liveFence === null) { @@ -168,31 +175,11 @@ export function structuredAgentSessionCurrentWork( journal: StructuredAgentSessionCurrentWorkJournal, evidence: StructuredAgentSessionWorkEvidence ): StructuredAgentSessionCurrentWork { - return new StructuredAgentSessionCurrentWork(journal, structuredAgentSessionLiveFence(evidence)) -} - -/** The host's sessions and lease store, as the projection reads them. */ -export type StructuredAgentSessionCurrentWorkHost = { - store: { getRecord: (sessionId: string) => AgentSessionRecord | null } - sessions: { - get( - sessionId: string - ): Pick | undefined - } -} - -/** A held chat's current work; null when this host holds no conversation for it. */ -export function hostStructuredAgentSessionCurrentWork( - host: StructuredAgentSessionCurrentWorkHost, - sessionId: string -): StructuredAgentSessionCurrentWork | null { - const session = host.sessions.get(sessionId) - return session - ? structuredAgentSessionCurrentWork(session.journal, { - record: host.store.getRecord(sessionId), - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) - }) - : null + return new StructuredAgentSessionCurrentWork( + journal, + structuredAgentSessionLiveFence(evidence), + evidence.revision + ) } /** An operation context's current work: the host projection it was built with, or, for one built diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index dd685da629c6..2c351fb2941f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -19,7 +19,6 @@ import { runningCallEnd, terminalAgentJournalBody } from '../agent-session-journal/journal-terminal-settlement' -import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { ResolveDispatchInput } from '../agent-session-journal/journal-store-contracts' import { lostLiveWorkJournalBody } from '../agent-session-journal/journal-subagent-liveness' import { @@ -76,8 +75,6 @@ export type StructuredAgentSessionDeadGenerationInput = { exit?: StructuredAgentSessionWatchedExit /** A person's Stop ended a starting child before what it was handed could run: each is rejected so. */ unrunRejection?: SubmissionRejectionFact - /** Committed in the same transaction as the rows, even when nothing is left to settle. */ - receipt?: JournalOperationReceipt } /** The exited child's own settlement, planned at the batch's own turn in the write queue. */ @@ -86,7 +83,6 @@ export async function settleStructuredAgentSessionDeadGeneration( ): Promise { try { if ( - !input.receipt && !input.showUnexpectedExitOutcome && !hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) ) { @@ -96,7 +92,6 @@ export async function settleStructuredAgentSessionDeadGeneration( settlementId: `dead-generation:${input.settlementId}`, fence: input.fence, recovered: true, - ...(input.receipt ? { receipt: input.receipt } : {}), plan: () => planStructuredAgentSessionDeadGeneration(input) }) return { ok: true } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts index 2dd6fa8c926f..6fad454e993f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-forget-status.test.ts @@ -88,8 +88,7 @@ function ownerRecord(): AgentSessionRecord { claimKeyId: 'fixture-key', claimStatus: 'live', unreconciled: false, - deathEvidence: null, - leftoverSettledAt: null + deathEvidence: null } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts new file mode 100644 index 000000000000..d58a51f2ada0 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts @@ -0,0 +1,40 @@ +// The host's current-work projection (`structured-agent-session-current-work.ts`) for a chat it +// holds, read from its sessions and lease store. Apart from the projection itself, which stays free +// of host types so shared and client code can import it. + +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + structuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWork +} from './structured-agent-session-current-work' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' + +/** The host's sessions and lease store, as the projection reads them. */ +export type StructuredAgentSessionCurrentWorkHost = { + store: { getRecord: (sessionId: string) => AgentSessionRecord | null } + sessions: { + get( + sessionId: string + ): + | Pick< + StructuredAgentSessionHostSession, + 'journal' | 'lastEndedChild' | 'operationalRevision' + > + | undefined + } +} + +/** A held chat's current work; null when this host holds no conversation for it. */ +export function hostStructuredAgentSessionCurrentWork( + host: StructuredAgentSessionCurrentWorkHost, + sessionId: string +): StructuredAgentSessionCurrentWork | null { + const session = host.sessions.get(sessionId) + return session + ? structuredAgentSessionCurrentWork(session.journal, { + record: host.store.getRecord(sessionId), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), + revision: session.operationalRevision ?? 0 + }) + : null +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 44abe4ae31af..c6b7cdea1995 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -2,7 +2,7 @@ // messages to a provider child. Bundled because they share one invariant — a conversation open // with a message queued has a delivery loop — and the open is where a loop for leftovers wakes. -import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { holdClosedStructuredAgentSessionSends } from './structured-agent-session-host-lifetime' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts index c87ce9d485bc..dd4d0976bd88 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.test.ts @@ -53,8 +53,7 @@ function reservedRecord(): AgentSessionRecord { claimKeyId: 'key-1', claimStatus: 'reserved', unreconciled: false, - deathEvidence: null, - leftoverSettledAt: null + deathEvidence: null }, createdAt: NOW, updatedAt: NOW diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts index d6372878b2bd..35fa639691df 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-types.ts @@ -110,6 +110,9 @@ export type StructuredAgentSessionHostSession = { * observed exit. */ child: StructuredAgentSessionProviderChild | null lastEndedChild?: StructuredAgentSessionEndedChild + /** Bumped by every signal that a generation ended, which can change what is current with no + * journal row or lease write: readers key what they cache of current work on it. */ + operationalRevision?: number /** Owned by one live restart action, so another caller cannot replace its progress. */ restartResume?: NonNullable & { operationId: string } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index e32b2d8fd11d..edb957ca3172 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -48,7 +48,7 @@ import { } from './structured-agent-session-restart-resume-host' import { structuredAgentSessionRestartResumeSurfaces } from './structured-agent-session-restart-resume-wiring' import * as conversation from './structured-agent-session-host-delivery' -import { wireStructuredAgentSessionQueuedMessages } from './structured-agent-session-queued-wiring' +import { wireStructuredAgentSessionQueuedMessages as wireQueuedMessages } from './structured-agent-session-queued-wiring' import * as sessionLogger from './structured-agent-session-logger' export type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' @@ -58,18 +58,17 @@ export class StructuredAgentSessionHost { this ) private readonly sessions = new StructuredAgentSessionConversations({ - deliver: (sessionId, journal) => { + deliver: (sessionId, journal, committed) => { this.subscribers.publish(sessionId, journal) this.conversationDelivery.afterCommit(sessionId, journal) + this.restore.reconciliation.observeCommit(sessionId, committed.lowestFence) }, deliverSettleEdge: (id, journal) => this.conversationDelivery.afterSettleEdge(id, journal), logger: sessionLogger.deferredStructuredAgentSessionLogger(() => this.deps.logger), onOpened: (sessionId) => this.queued.drain.schedule(sessionId), now: () => this.now() }) - private readonly queued = wireStructuredAgentSessionQueuedMessages(this.sessions, () => - this.mutationContext() - ) + private readonly queued = wireQueuedMessages(this.sessions, () => this.mutationContext()) // Every journal publish is activity: the one renewal the idle sweep reads. private readonly clientDelivery = new StructuredAgentSessionClientDelivery( this.sessions, @@ -136,7 +135,8 @@ export class StructuredAgentSessionHost { // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened, onUnopened: (sessionId) => this.tabs.markUnopened(sessionId), - startup: { sessions: this.sessions, tasks: this.tasks } + closeAtRest: (sessionId) => this.lifetime.closeAtRest(sessionId), + startup: { sessions: this.sessions, tasks: this.tasks, clientDelivery: this.clientDelivery } }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -144,7 +144,7 @@ export class StructuredAgentSessionHost { sessions: this.sessions, flushLifecycle: (sessionId) => this.runtimeState.lifecycleBarrier(sessionId), publishStatus: this.clientDelivery.publishStatusAndSettlement, - generationEnded: this.clientDelivery.publishGenerationEnded, + generationEnded: this.restore.reconciliation.signal, serialize: (sessionId, task) => this.tasks.trackAttach(this.serialize(sessionId, task)), now: () => this.now(), runtimeState: this.runtimeState, @@ -162,7 +162,8 @@ export class StructuredAgentSessionHost { open: (sessionId) => this.conversationDelivery.open(sessionId), deliveryActive: (sessionId) => this.conversationDelivery.loop.isRunning(sessionId), closeStatus: (sessionId, options) => this.clientDelivery.closeSession(sessionId, options), - readChildWork: this.clientDelivery.readChildWork + readChildWork: this.clientDelivery.readChildWork, + deferCloseAtRest: (sessionId) => this.restore.reconciliation.deferCloseAtRest(sessionId) }) this.runtimeState.startLeaseRenewal() this.lifetime.idleSweep.start() @@ -184,7 +185,7 @@ export class StructuredAgentSessionHost { sessions: this.sessions, now: () => this.now(), publishStatus: this.clientDelivery.publishStatus, - generationEnded: this.clientDelivery.publishGenerationEnded, + generationEnded: this.restore.reconciliation.signal, wakeDelivery: (sessionId: string) => this.conversationDelivery.loop.wake(sessionId), endExitedChild: this.eventRecovery.endExitedChildUnderSerialize } satisfies StructuredAgentSessionLifetimeContext @@ -258,7 +259,9 @@ export class StructuredAgentSessionHost { /** Quit: no exit or recovery settled after this starts a child or hands a message over, and the * queue hands no card over. */ stopDelivery = (): void => - [this.conversationDelivery, this.queued.drain].forEach((d) => d.dispose()) + [this.conversationDelivery, this.queued.drain, this.restore.reconciliation].forEach((d) => + d.dispose() + ) async flushAllStreamedEvents(options?: { trigger?: 'quit' | 'update' }): Promise { this.stopDelivery() @@ -288,6 +291,7 @@ export class StructuredAgentSessionHost { wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), stopAgent: (sessionId, ending) => this.lifetime.stopAgent(sessionId, ending), wakeQueuedDrain: (sessionId) => this.queued.drain.schedule(sessionId), + awaitingReopenMark: (sessionId) => this.restore.reconciliation.awaitingReopenMark(sessionId), acquireAborts: this.runtimeState.acquireAborts, optionRevisions: this.runtimeState.optionRevisions, now: () => this.now() @@ -359,6 +363,7 @@ export class StructuredAgentSessionHost { runtimeState: this.runtimeState, conversationDelivery: this.conversationDelivery, lifetime: this.lifetime, - serialize: this.serialize + serialize: this.serialize, + reconciliation: this.restore.reconciliation }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts index 04858d8c996e..f95c2b346e9e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -5,10 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { - HOST_TEST_NOW as NOW, - HOST_TEST_SESSION as SESSION -} from './structured-agent-session-host-test-data' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' import { createQueuedMessageTestRig, QUEUED_RIG_CALLER as CALLER, @@ -59,8 +56,9 @@ describe('reading a chat', () => { // Answered, so no send left pending keeps the conversation open. await current.settleAccepted(await current.workingSend(), 'work') await leaveUnfinishedWork(current, { prompt: true }) + // The storage fault stays: the worker keeps retrying in the background, and every attempt + // rolls back, so any write the readers made would be the only one logged. const { database } = await exitWhileSettlementFails(current) - database.db.exec('DROP TRIGGER reject_recovered') await current.host.close(SESSION, 'evict') expect(current.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) const writes = logEveryWrite(database.db) @@ -76,8 +74,8 @@ describe('reading a chat', () => { unsubscribe() expect(writes()).toEqual([]) - // Still owed, and still as the dead generation left it: only an ownership event settles it. - expect(current.store.getRecord(SESSION)?.lease.leftoverSettledAt).toBeNull() + // Still owed, and still as the dead generation left it: only the chat's worker settles it. + expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(true) expect(turnState(current)).toBe('running') expect(holds(current)).toEqual({ working: false, queueHeld: false }) }) @@ -113,8 +111,7 @@ describe('a release nothing proved', () => { expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', - deathEvidence: null, - leftoverSettledAt: NOW + deathEvidence: null }) expect(turnState(current)).toBe('unverifiable') expect(currentJournal(current).itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts index ce90d5547cdc..f0cf5e13f9f3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -1,9 +1,12 @@ +// The chat's reconciliation worker, driven by every way a generation ends: what it settles, that +// it settles in one commit, and that nothing a person does ever waits on it. + import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { - agentJournalItemKey, - agentJournalSubmissionKey -} from '../../../shared/agent-session-journal-item-key' -import type { AgentJournalItemIdentity } from '../../../shared/agent-session-journal-types' + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' import { agentSessionWriteNoticeEnglish, agentSessionWriteNoticeParts @@ -14,8 +17,8 @@ import { projectStructuredAgentSessionStatusState } from '../../../shared/struct import { codexSubagentGroupIdentity } from '../../codex/codex-subagent-roster' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { - HOST_TEST_NOW as NOW, HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, hostTestOperationId } from './structured-agent-session-host-test-data' import { @@ -33,15 +36,17 @@ import { exitWhileSettlementFails, leaveUnfinishedWork, REASONING, + reconciled, recoveredRows, + REJECT_RECOVERED_ROWS, rowFence, ROSTER_GROUP, sendText, + startUp, TASK, TOOL, - TURN_KEY, turnState, - watchMarks + watchSettlementCommits } from './structured-agent-session-leftover-settlement.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -52,46 +57,32 @@ afterEach(async () => { vi.restoreAllMocks() }) -function leftoverSettledAt(current: QueuedMessageTestRig): number | null | undefined { - return current.store.getRecord(SESSION)?.lease.leftoverSettledAt -} - -function settlementWarnings(): string[] { - return vi - .mocked(console.warn) - .mock.calls.flatMap(([, fields]) => - typeof fields === 'object' && fields !== null && 'scope' in fields - ? [String(fields.scope)] - : [] - ) -} - -describe('the observed exit, then the next start', () => { - it('settles at the start in one commit at the released fence, with its mark, before the successor binds', async () => { +describe('an observed exit whose settlement the database refused', () => { + it('is settled by the worker in one commit, while the next send starts and delivers', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig const firstSend = await current.workingSend() await leaveUnfinishedWork(current, { everything: true }) const { session, database, deadFence } = await exitWhileSettlementFails(current) - database.db.exec('DROP TRIGGER reject_recovered') - const marks = watchMarks(database.db) - - const { clientOperationId, result } = sendText(current, 'continue after storage recovered') + // The fault holds: the send is accepted, starts its agent and is delivered all the same. + const { clientOperationId, result } = sendText(current, 'continue while storage fails') expect(await result).toMatchObject({ ok: true }) - // An ordinary send: no card waits behind the dead generation's turn. expect(session.journal.queuedMessages.list()).toEqual([]) expect(session.journal.submission(clientOperationId)).toBeDefined() await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) - const settled = recoveredRows(database.db) - expect(settled.length).toBeGreaterThan(0) - // Every settlement row, at the released fence, was in the database when the mark was set. - expect(marks()).toEqual([{ fence: deadFence + 1, rows: settled.length }]) - expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 1)) - // The successor reserved after it, at the next fence. expect(session.child?.fence).toBe(deadFence + 2) - expect(leftoverSettledAt(current)).toBe(NOW) - // The exit's proof named the dead generation: its turn was interrupted. + expect(turnState(current)).toBe('running') + + const commits = watchSettlementCommits() + database.db.exec('DROP TRIGGER reject_recovered') + await reconciled(current) + + // One transaction, at the lease's current fence (the successor's), judged by the exit's proof + // though the successor's reservation cleared it from the lease. + expect(commits()).toHaveLength(1) + const settled = recoveredRows(database.db) + expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 2)) expect(turnState(current)).toBe('interrupted') const body = (identity: AgentJournalItemIdentity) => session.journal.itemBody(agentJournalItemKey(identity)) @@ -109,10 +100,9 @@ describe('the observed exit, then the next start', () => { ] }) // Handed over and never answered: doubt, never "not delivered". - expect(session.journal.submission(firstSend)).toMatchObject({ - dispatchState: 'unknown', - reason: 'provider_exited_before_acknowledgement' - }) + expect(session.journal.submission(firstSend)).toMatchObject({ dispatchState: 'unknown' }) + // The successor's send is its own, untouched. + expect(session.journal.submission(clientOperationId)?.dispatchState).not.toBe('unknown') }) it('settles at the exit itself when nothing refuses it, once', async () => { @@ -121,16 +111,18 @@ describe('the observed exit, then the next start', () => { await current.workingSend() await leaveUnfinishedWork(current, { prompt: true }) const database = openTestJournalHostDatabase(current.root) - const marks = watchMarks(database.db) + const commits = watchSettlementCommits() await exitChild(current) + await reconciled(current) - const settled = recoveredRows(database.db) expect(turnState(current)).toBe('interrupted') - expect(marks()).toEqual([{ fence: 2, rows: settled.length }]) - // The next start finds nothing left: its settlement writes no row. + expect(commits()).toHaveLength(1) + const settled = recoveredRows(database.db) + // The next start finds nothing left: the worker its end signals writes no row. expect((await sendText(current, 'next').result).ok).toBe(true) await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + await reconciled(current) expect(recoveredRows(database.db)).toEqual(settled) }) }) @@ -177,43 +169,88 @@ describe('a settlement that cannot commit never gates the user', () => { await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) }) - it('starts, delivers and logs when the start’s own settlement fails; the next trigger settles once as unverifiable', async () => { + it('drains a card that waited on the dead generation, with no journal write', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig await current.workingSend() - await leaveUnfinishedWork(current, { prompt: true }) - const { session, database } = await exitWhileSettlementFails(current) + await leaveUnfinishedWork(current) + const queued = await current.send('after this turn', 'queue-if-active').result + expect(queued).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + expect(currentJournal(current).queuedMessages.list()).toHaveLength(1) + const database = openTestJournalHostDatabase(current.root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + database.db.exec(REJECT_RECOVERED_ROWS) + const child = current.host.collaboratorsForTests().sessions.get(SESSION)?.child + if (!child?.generation) { + throw new Error('expected the live child') + } - const { clientOperationId, result } = sendText(current, 'meets the storage fault') + await current.host.handleAdapterEvent({ + type: 'ended', + sessionId: SESSION, + cause: 'unexpected-exit', + reason: 'observed exit', + fence: child.fence, + acquisitionGeneration: child.generation + }) - expect(await result).toMatchObject({ ok: true }) - expect(session.journal.submission(clientOperationId)).toBeDefined() + // The end alone woke the drain: the card went out though nothing settled the turn. await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) - expect(settlementWarnings()).toContain('acquisition-settlement') - // Nothing partial committed; the reservation left the mark as it was. expect(recoveredRows(database.db)).toEqual([]) - expect(leftoverSettledAt(current)).toBeNull() expect(turnState(current)).toBe('running') - // The successor's turn is the one that counts. - expect(currentWork(current).activeTurnId()).toBeNull() - - database.db.exec('DROP TRIGGER reject_recovered') - const marks = watchMarks(database.db) - await exitChild(current) + }) +}) - // The reservation wiped the proof that named it: less specific, never wrong. - expect(turnState(current)).toBe('unverifiable') - expect(session.journal.itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ - resolution: { state: 'cancelled' } - }) - const revisions = recoveredRows(database.db).filter((row) => row.rowJson.includes(TURN_KEY)) - expect(revisions).toHaveLength(1) - expect(marks()).toHaveLength(1) +describe('the worker’s retry', () => { + /** Lets every attempt a fired timer began run to its end, with no fake time passing. */ + async function settleTurns(): Promise { + for (let turn = 0; turn < 50; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } + } + + it('backs off from 1 s, coalesces the signals meanwhile, and settles once the fault clears', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + try { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const { database } = await exitWhileSettlementFails(current) + const reconciliation = current.host.collaboratorsForTests().reconciliation + const attempts = vi.spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') + expect(reconciliation.owes(SESSION)).toBe(true) + + // Signals during the backoff join the one worker; none runs an attempt of its own. + reconciliation.signal(SESSION) + reconciliation.signal(SESSION, { restate: true }) + await settleTurns() + await vi.advanceTimersByTimeAsync(990) + await settleTurns() + expect(attempts).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(10) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(1) + expect(reconciliation.owes(SESSION)).toBe(true) + expect(turnState(current)).toBe('running') + + database.db.exec('DROP TRIGGER reject_recovered') + // The second failure doubled the wait. + await vi.advanceTimersByTimeAsync(1_990) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(10) + await settleTurns() + expect(reconciliation.owes(SESSION)).toBe(false) + expect(turnState(current)).toBe('interrupted') + } finally { + vi.useRealTimers() + } }) }) describe('a stale prompt', () => { - it('holds no send, ends cancelled once settled, and refuses a stale answer', async () => { + it('holds no send, refuses a stale answer in plain words, and ends cancelled once settled', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig await current.workingSend() @@ -261,7 +298,7 @@ describe('a stale prompt', () => { expect(current.answerPrompt).not.toHaveBeenCalled() database.db.exec('DROP TRIGGER reject_recovered') - await exitChild(current) + await reconciled(current) expect(session.journal.itemBody(prompt.itemId)).toMatchObject({ resolution: { state: 'cancelled' } @@ -271,87 +308,73 @@ describe('a stale prompt', () => { }) }) +describe('a late write from an ended generation', () => { + it('is settled by the worker its commit wakes', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + // Answered, so the exit leaves nothing to settle and writes no row at the moved fence. + await current.settleAccepted(await current.workingSend(), 'work') + const deadFence = current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await exitChild(current) + await reconciled(current) + const database = openTestJournalHostDatabase(current.root) + expect(recoveredRows(database.db)).toEqual([]) + const lateCall: AgentJournalItemIdentity = { + provider: 'codex', + threadId: THREAD, + turnId: 'late', + ordinal: 1 + } + + // An operation the closed sink had already taken lands at the generation's own fence. + await currentJournal(current).appendItem( + lateCall, + { kind: 'tool-call', name: 'shell', input: { command: 'make' }, state: 'running' }, + { fence: deadFence, ownerFence: deadFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + + await vi.waitFor(() => + expect(currentJournal(current).itemBody(agentJournalItemKey(lateCall))).toMatchObject({ + state: 'failed' + }) + ) + await reconciled(current) + expect(recoveredRows(database.db).map(rowFence)).toEqual([deadFence + 1]) + }) +}) + describe('startup', () => { - it('settles a failed exit settlement once and sets the mark, starting nothing', async () => { + it('settles a failed exit settlement with no stored mark, starting nothing', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig await current.workingSend() await leaveUnfinishedWork(current, { prompt: true }) const { deadFence } = await exitWhileSettlementFails(current) - let owed: number | null | undefined // A new process loads the database and the record store back from disk, and starts up. - await current.crashReloadHostProcess(() => { - owed = leftoverSettledAt(current) - }) + await current.crashReloadHostProcess() + await reconciled(current) const database = openTestJournalHostDatabase(current.root) - expect(owed).toBeNull() + // The worker opened the chat itself, so it closes it again: nothing else had it open. + await vi.waitFor(() => + expect(current.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) + ) + await current.host.journalSnapshot(SESSION) expect(turnState(current)).toBe('interrupted') expect(currentJournal(current).itemBody(agentJournalItemKey(APPROVAL))).toMatchObject({ resolution: { state: 'cancelled' } }) - expect(leftoverSettledAt(current)).toBe(NOW) const settled = recoveredRows(database.db) expect(settled.map(rowFence)).toEqual(settled.map(() => deadFence + 1)) expect(current.starts).toHaveBeenCalledOnce() // A second pass finds nothing owed. - await current.host.reconcileRestartLeases() - await current.host.startupSettled() + await startUp(current) expect(recoveredRows(database.db)).toEqual(settled) }) }) -describe('the mark is only an index', () => { - it('a wrong "settled" holds nothing: delivery goes on, and the next start settles and corrects it', async () => { - rig = await createQueuedMessageTestRig({ restartable: true }) - const current = rig - await current.workingSend() - await leaveUnfinishedWork(current) - const { database } = await exitWhileSettlementFails(current) - database.db.exec('DROP TRIGGER reject_recovered') - await current.store.transitionHandoff(SESSION, (record) => ({ - ...record, - lease: { ...record.lease, leftoverSettledAt: 1 } - })) - await current.crashReloadHostProcess() - // Startup believed it and skipped the chat; a reader's open writes nothing either. - await current.host.journalSnapshot(SESSION) - expect(turnState(current)).toBe('running') - const marks = watchMarks(openTestJournalHostDatabase(current.root).db) - - expect((await sendText(current, 'after a wrong mark').result).ok).toBe(true) - - await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) - expect(turnState(current)).toBe('interrupted') - // The receipt replaced the wrong mark; a set mark is not re-set, so no row logs it. - expect(marks()).toEqual([]) - expect(leftoverSettledAt(current)).toBe(NOW) - }) - - it('a wrong "owed" costs one journal open and a settlement that plans nothing', async () => { - rig = await createQueuedMessageTestRig({ restartable: true }) - const current = rig - await current.workingSend() - await leaveUnfinishedWork(current) - await exitChild(current) - expect(turnState(current)).toBe('interrupted') - await current.store.transitionHandoff(SESSION, (record) => ({ - ...record, - lease: { ...record.lease, leftoverSettledAt: null } - })) - const settled = recoveredRows(openTestJournalHostDatabase(current.root).db) - - await current.crashReloadHostProcess() - - expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual(settled) - expect(leftoverSettledAt(current)).toBe(NOW) - expect((await sendText(current, 'after a wrong owed mark').result).ok).toBe(true) - await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) - }) -}) - describe('a folder workspace', () => { - it('settles a folder chat’s dead generation the same way', async () => { + it('settles a folder chat’s dead generation the same way, and delivers', async () => { rig = await createQueuedMessageTestRig({ restartable: true, location: { @@ -365,17 +388,14 @@ describe('a folder workspace', () => { await current.workingSend() await leaveUnfinishedWork(current) const { session, database } = await exitWhileSettlementFails(current) - database.db.exec('DROP TRIGGER reject_recovered') const { clientOperationId, result } = sendText(current, 'continue in the folder') expect(await result).toMatchObject({ ok: true }) expect(session.journal.submission(clientOperationId)).toBeDefined() await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + database.db.exec('DROP TRIGGER reject_recovered') + await reconciled(current) expect(turnState(current)).toBe('interrupted') - expect(leftoverSettledAt(current)).toBe(NOW) - expect( - session.journal.item(agentJournalSubmissionKey(clientOperationId))?.sequence - ).toBeGreaterThan(recoveredRows(database.db).at(-1)?.seq ?? 0) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts index 3986f7d4bbbe..f24c78e8af4f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts @@ -18,12 +18,11 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import type Database from '../../sqlite/sync-database' -import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { JournalRowWriter } from '../agent-session-journal/journal-row-writer' +import type { JournalRow } from '../agent-session-journal/journal-row-schema' import type { StructuredAgentSessionHost } from './structured-agent-session-host' -import { - hostStructuredAgentSessionCurrentWork, - type StructuredAgentSessionCurrentWork -} from './structured-agent-session-current-work' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { HOST_TEST_SESSION as SESSION, HOST_TEST_THREAD as THREAD, @@ -129,7 +128,8 @@ export async function leaveUnfinishedWork( } /** The child dies on its own while the database refuses every settlement row: the exit releases - * its lease (proof written, fence moved, leftovers owed) and its work stays saved as running. */ + * its lease (proof written, fence moved) and its work stays saved as running. The chat's + * reconciliation worker is left backing off. */ export async function exitWhileSettlementFails(current: QueuedMessageTestRig) { const context = current.host.collaboratorsForTests() const session = context.sessions.get(SESSION) @@ -154,8 +154,7 @@ export async function exitWhileSettlementFails(current: QueuedMessageTestRig) { expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', runtimeFence: child.fence + 1, - deathEvidence: { kind: 'exit-observed', ownerFence: child.fence }, - leftoverSettledAt: null + deathEvidence: { kind: 'exit-observed', ownerFence: child.fence } }) expect(liveTestJournalRows(database.db, SESSION)).toEqual(before) return { session, database, deadFence: child.fence } @@ -238,25 +237,24 @@ export function rowFence(row: { rowJson: string }): number { : Number.NaN } -/** Each time the lease's mark is set: the fence it was set at, and how many settlement rows the - * database held at that moment, inside the transaction that set it. */ -export function watchMarks(db: Database.Database): () => { fence: number; rows: number }[] { - db.exec(` - CREATE TEMP TABLE mark_log (fence INTEGER, rows INTEGER); - CREATE TEMP TRIGGER note_mark AFTER UPDATE ON agent_session_records - WHEN json_extract(NEW.record_json, '$.lease.leftoverSettledAt') IS NOT NULL - AND json_extract(OLD.record_json, '$.lease.leftoverSettledAt') IS NULL - BEGIN - INSERT INTO mark_log - SELECT json_extract(NEW.record_json, '$.lease.runtimeFence'), COUNT(*) FROM journal_rows - WHERE session_id = NEW.session_id AND json_extract(row_json, '$.recovered') = 1; - END; - `) +/** Each journal transaction that wrote settlement rows, by the rows it wrote: one entry per + * commit. */ +export function watchSettlementCommits(): () => JournalRow[][] { + const writes = vi.spyOn(JournalRowWriter.prototype, 'writeRows') return () => - db - .prepare('SELECT fence, rows FROM mark_log') - .all() - .map((row) => ({ fence: Number(row.fence), rows: Number(row.rows) })) + writes.mock.results.flatMap((result) => + result.type === 'return' && result.value.some((row) => row.recovered === true) + ? [result.value] + : [] + ) +} + +/** Once the chat's reconciliation worker has nothing owed: it retired. */ +export function reconciled(current: QueuedMessageTestRig, timeout = 10_000): Promise { + return vi.waitFor( + () => expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(false), + { timeout } + ) } /** Logs every INSERT, UPDATE and DELETE on every table of the file, from this connection on. */ @@ -279,16 +277,26 @@ export function logEveryWrite(db: Database.Database): () => string[] { .map((row) => String(row.entry)) } -/** Startup as the runtime runs it, after an earlier process whose last generation ended with - * nothing settling it, as a crash leaves it. */ -export async function startUpOwingLeftovers( - host: Pick, - store: Pick +/** Startup as the runtime runs it, on any host: the restart reconcile, then the scan of every chat + * it begins, with no stored mark of what is owed, until the chat's worker is idle. */ +export async function startUpHost( + host: Pick< + StructuredAgentSessionHost, + 'reconcileRestartLeases' | 'startupSettled' | 'collaboratorsForTests' + > +): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() + await host.collaboratorsForTests().reconciliation.idle(SESSION) +} + +/** Startup as the runtime runs it: the restart reconcile, then the scan of every chat it begins, + * until the chat's worker has nothing owed. */ +export async function startUp( + current: QueuedMessageTestRig, + host: Pick = current.host ): Promise { - await store.transitionHandoff(SESSION, (record) => ({ - ...record, - lease: { ...record.lease, leftoverSettledAt: null } - })) await host.reconcileRestartLeases() await host.startupSettled() + await reconciled(current) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts index 05a6f7303751..9071328594da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts @@ -1,16 +1,18 @@ -// The one settlement of what ended generations left unfinished, run only at the host's ownership -// events: an observed exit (after its release), an acquisition (before its reservation), and -// startup. Opening a chat never runs it. +// The one settlement of what ended generations left unfinished. An observed exit runs it first, +// with what it saw; the chat's reconciliation worker runs it after every generation end and at +// startup (`structured-agent-session-reconciliation-worker.ts`). Opening a chat never runs it. // -// Re-derived each time from the journal and the lease: every turn, call, prompt, reasoning row, -// subagent roster and background task an ended generation wrote and never ended (below the lease's -// fence, or at it once released), and every send handed to one and never answered. Such a -// generation can no longer write, since every release moves the fence. Its turns end `interrupted` only when the lease's death evidence -// names it, else `unverifiable`. Rows land at the lease's current fence, in ONE transaction with the -// lease's `leftoverSettledAt` receipt, so a second run plans nothing. Best effort at every trigger: -// a failure is the caller's to log, and its rows hold nothing (holds count only current-fence work). +// Re-derived each time from the journal, the lease and this host's sight of an exit: every turn, +// call, prompt, reasoning row, subagent roster entry and background task whose execution belongs +// to a generation that is not live (`structuredAgentSessionLiveFence`), and every send handed to +// one and never answered. Its turns end `interrupted` only when death evidence names that +// generation, else `unverifiable`, and a later proof revises an earlier `unverifiable`. Rows land at +// the lease's current fence in ONE transaction; a run with nothing left plans no row and writes +// nothing. A failure is the caller's to retry: its rows hold nothing, since the projection counts +// only the live generation's work. import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import { planStructuredAgentSessionDeadGeneration, type StructuredAgentSessionDeadGenerationInput @@ -22,27 +24,36 @@ import { import type { DeadGenerationJournal } from './structured-agent-session-unfinished-work' import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' -import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import { + structuredAgentSessionLiveFence, + type StructuredAgentSessionWorkEvidence +} from './structured-agent-session-current-work' -export type StructuredAgentSessionLeftoverStore = Pick & { - conversationReceipts: Pick -} +export type StructuredAgentSessionLeftoverStore = Pick + +/** An observed exit's settlement: what that child's generation left, settled as the exit says. */ +export type StructuredAgentSessionExitSettlement = Omit< + StructuredAgentSessionDeadGenerationInput, + 'journal' | 'sessionId' | 'fence' +> & { ownerFence: number } export type StructuredAgentSessionLeftoverSettlementInput = { store: StructuredAgentSessionLeftoverStore sessionId: string journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal - now: () => number - /** An observed exit: what that child's generation left is settled as the exit says. */ - exit?: Omit & { - ownerFence: number - } - /** An acquisition about to reserve over an owner a probe proved gone: that owner's generation - * ends with the reservation, so its work is settled with the rest. */ - ownerProvenGone?: true + exit?: StructuredAgentSessionExitSettlement + /** This host's sight of a child's root exit, which ends its generation even when the release + * write failed (`StructuredAgentSessionHostSession.lastEndedChild`). */ + ended?: StructuredAgentSessionWorkEvidence['ended'] + /** Proof a generation end carried that the lease no longer holds (a reservation that replaced a + * proven-dead owner clears it, and a later release writes its own): only what that generation + * and earlier ones left is settled, judged by it. Absent: the lease's own evidence judges. */ + proof?: AgentSessionDeathEvidence & { ownerFence: number } } -export type StructuredAgentSessionLeftoverSettlement = { ok: true } | { ok: false; error: unknown } +export type StructuredAgentSessionLeftoverSettlement = + | { ok: true; planned: number } + | { ok: false; error: unknown } export async function settleStructuredAgentSessionLeftovers( input: StructuredAgentSessionLeftoverSettlementInput @@ -50,28 +61,32 @@ export async function settleStructuredAgentSessionLeftovers( try { const record = input.store.getRecord(input.sessionId) if (!record) { - return { ok: true } + return { ok: true, planned: 0 } } - const { lease } = record - const fence = lease.runtimeFence - const { exit, journal, sessionId } = input - // A released fence was never a child's (a reservation moves it first), so all of it is ended; - // what an older build left at it is settled with the rest. - const ended = input.ownerProvenGone || lease.claimStatus === 'released' - const below = exit ? exit.ownerFence : ended ? fence + 1 : fence + const fence = record.lease.runtimeFence + const { journal, sessionId } = input + // Everything below the live generation is ended; with none live, all of it is. + const liveFence = structuredAgentSessionLiveFence({ + record, + ...(input.ended ? { ended: input.ended } : {}) + }) + // An exit's own account holds only until a later generation is live: its rows and sends at + // this fence would be that generation's. The general rule settles what the exit left instead. + const exit = liveFence === null && !input.proof ? input.exit : undefined + const { proof } = input + const below = proof + ? Math.min(proof.ownerFence + 1, liveFence ?? fence + 1) + : exit + ? exit.ownerFence + : (liveFence ?? fence + 1) const failureTextContext = structuredAgentSessionFailureWordsContext(record) + let planned = 0 await journal.appendPlannedLifecycleBatch({ settlementId: exit ? `dead-generation:${exit.settlementId}` - : `${STALE_SESSION_ROW_PREFIX}${sessionId}:${fence}:seq-${journal.cursor().sequence}`, + : `${STALE_SESSION_ROW_PREFIX}${sessionId}:${fence}:${proof ? `proof-${proof.ownerFence}:` : ''}seq-${journal.cursor().sequence}`, fence, recovered: true, - // Only a released lease is owed anything: a reservation leaves the mark as it was. - ...(lease.claimStatus === 'released' - ? { - receipt: input.store.conversationReceipts.leftoverSettled(sessionId, fence, input.now()) - } - : {}), plan: () => { const exited = exit ? planStructuredAgentSessionDeadGeneration( @@ -86,40 +101,23 @@ export async function settleStructuredAgentSessionLeftovers( fence, acquisitionGeneration: null, // Read with the rows: the evidence the release before this settlement wrote. - deathEvidence: input.store.getRecord(sessionId)?.lease.deathEvidence ?? null, + deathEvidence: proof ?? input.store.getRecord(sessionId)?.lease.deathEvidence ?? null, failureTextContext, - below + below, + // With an exit's account, nothing is live: every entry of what is settled is ended. + ...(exit ? {} : { entriesBelow: below }) }) - return { - mutations: [...exited.mutations, ...stale.mutations], - dispatches: [ - ...exited.dispatches, - ...stale.dispatches.filter((entry) => !settledByExit.has(entry.clientMessageId)) - ] - } + const mutations = [...exited.mutations, ...stale.mutations] + const dispatches = [ + ...exited.dispatches, + ...stale.dispatches.filter((entry) => !settledByExit.has(entry.clientMessageId)) + ] + planned = mutations.length + dispatches.length + return { mutations, dispatches } } }) - return { ok: true } + return { ok: true, planned } } catch (error) { return { ok: false, error } } } - -/** The settlement at a trigger that must go on whatever happens: a failure is logged under the - * trigger's scope, and the leftovers stay owed for the next trigger. */ -export async function settleStructuredAgentSessionLeftoversOrLog( - input: StructuredAgentSessionLeftoverSettlementInput & { - logger: StructuredAgentSessionLogger - scope: string - } -): Promise { - const settled = await settleStructuredAgentSessionLeftovers(input) - if (!settled.ok) { - input.logger.warn("settling a gone agent's leftover work did not finish", { - scope: input.scope, - sessionId: input.sessionId, - error: settled.error - }) - } - return settled.ok -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index e1f6f40f0b00..509e5a28ccf5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -46,6 +46,9 @@ export type StructuredAgentSessionMutationContext = { * journal commit (a conversation command). Draft-table changes need no call: * the draft store notifies through the journal's own commit listener. */ wakeQueuedDrain?: (sessionId: string) => void + /** The queue sends nothing on its own before the chat's startup reopen mark + * (`StructuredAgentSessionReconciliation.awaitingReopenMark`). */ + awaitingReopenMark?: (sessionId: string) => boolean /** The provider wait each session's serialize is on (a start, an option write), which a caller * outside that serialize aborts. */ acquireAborts: Pick diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts index e9bc67d48ebe..350e80f83072 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -130,6 +130,10 @@ beforeEach(async () => { startHost() expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) await host.close(SESSION, 'evict') + // The stop's release wakes the chat's worker, which opens the chat and closes it again; the + // tests below write its journal directly, so it must be shut first. + await host.collaboratorsForTests().reconciliation.idle(SESSION) + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) }) afterEach(async () => { @@ -359,10 +363,16 @@ describe('settling an earlier child before the next one takes its message', () = { fence: releasedFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) await journal.close() + // The process died between that release and its settlement: the next startup settles it, in + // the background, from the evidence the release kept. + await restartHost() + await host.reconcileRestartLeases() + await host.startupSettled() + await host.collaboratorsForTests().reconciliation.idle(SESSION) const id = await accept('for the next child') await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) - // The exit was observed, so its receipt ends the turn, and the chat says why it stopped. + // The exit was observed, so its evidence ends the turn, and the chat says why it stopped. const items = conversation()!.journal.snapshot().items expect(items.map((item) => readAgentJournalTurn(item.body)).filter(Boolean)).toContainEqual( expect.objectContaining({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts index a95f066bad03..caf2242071bb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-card-holds.test.ts @@ -554,10 +554,11 @@ describe('where the host would refuse the send', () => { } // The clear's generation end wakes the drain, which would send the card; holding the session's // lane keeps it waiting while the test reads the gate (`release` lets the drain run). + // The gate exists before the task runs: the clear's end may have queued the chat's + // reconciliation pass ahead of it on the lane. let release = (): void => undefined - const held = rig.host - .collaboratorsForTests() - .serialize(HOST_TEST_SESSION, () => new Promise((resolve) => (release = resolve))) + const gate = new Promise((resolve) => (release = resolve)) + const held = rig.host.collaboratorsForTests().serialize(HOST_TEST_SESSION, () => gate) const card = 'left-behind' await journal.queuedMessages.insert({ messageId: card, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 48a380142842..a033c718eec4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -233,6 +233,9 @@ export type QueuedMessageDrainDeps = { currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void + /** Nothing sends on its own yet: the chat's startup reopen mark has not landed. Its landing + * wakes the drain again. */ + held?: (sessionId: string) => boolean logger: StructuredAgentSessionLogger } @@ -259,7 +262,7 @@ export class StructuredAgentSessionQueuedMessageDrain { schedule(sessionId: string): void { const journal = this.disposed ? undefined : this.deps.sessions.get(sessionId)?.journal - if (!journal) { + if (!journal || this.deps.held?.(sessionId)) { return } // Cheap pre-check so token streams do not pay a serialized step per delta. @@ -298,7 +301,7 @@ export class StructuredAgentSessionQueuedMessageDrain { private async step(sessionId: string): Promise { const session = this.deps.sessions.get(sessionId) - if (this.disposed || !session) { + if (this.disposed || !session || this.deps.held?.(sessionId)) { return } const journal = session.journal diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index f3c97dd2d713..ac9da94d19ee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -19,10 +19,8 @@ import { nextStructuredQueuedMessage, type StructuredQueueGateInput } from './structured-agent-session-queued-messages' -import { - structuredAgentSessionCurrentWork, - type StructuredAgentSessionCurrentWorkHost -} from './structured-agent-session-current-work' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { StructuredAgentSessionCurrentWorkHost } from './structured-agent-session-host-current-work' export type QueuePublication = { queuedMessages: AgentSessionQueuedMessage[] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index 783da04803c6..2fedb5e9a8f8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -8,7 +8,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' -import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { StructuredAgentSessionQueuedMessageDrain } from './structured-agent-session-queued-messages' import { deleteQueuedStructuredAgentMessage, @@ -35,6 +35,7 @@ export function wireStructuredAgentSessionQueuedMessages( currentWork: (sessionId) => hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), + held: (sessionId) => context().awaitingReopenMark?.(sessionId) === true, // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts new file mode 100644 index 000000000000..ca4490428122 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -0,0 +1,162 @@ +// One attempt of a chat's reconciliation worker, for a caller inside the chat's action lane: the +// short recheck and the writes. Everything owed is re-derived here from the lease, this host's +// sight of an exit and the journal; only what cannot be derived rides in from the signals (an +// exit's own account, a proof the lease no longer holds, and the startup share not yet done). +// +// The steps stay apart, so one never retires another's debt: +// (a) lease-release repair: an exit this host observed whose release write failed; +// (b) the startup share: the cards' repair and prune, an earlier host process's unsent sends kept +// as cards, the queue's reopen mark after them, and a rewind it left in doubt; +// (c) item settlement (`settleStructuredAgentSessionLeftovers`), one planned batch. +// An empty item plan says nothing of (a) or (b). + +import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' +import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + releaseLeaseOfEndedStructuredAgentSessionChild, + structuredAgentSessionEndedChildHoldsLease +} from './structured-agent-session-child-close' +import { + settleStructuredAgentSessionLeftovers, + type StructuredAgentSessionExitSettlement +} from './structured-agent-session-leftover-settlement' +import { structuredAgentSessionHostInstance } from './structured-agent-session-queued-pause' +import { recoverStructuredRewind } from './structured-rewind-recovery' + +export type StructuredAgentSessionReconciliationPassContext = Pick< + StructuredAgentSessionLifetimeContext, + 'deps' | 'sessions' | 'now' +> + +/** What the signals left that the journal and the lease cannot tell. */ +export type StructuredAgentSessionReconciliationDebts = { + exit?: StructuredAgentSessionExitSettlement + /** Each ended generation's proof, as its end carried it: a reservation clears the lease's, and a + * later release replaces it with its own. */ + evidence?: AgentSessionDeathEvidence[] + /** This process has not yet finished the chat's startup share. */ + startupShare?: { reopenMarked: boolean; leaseSettled: boolean } +} + +export type StructuredAgentSessionReconciliationPass = { + /** A step could not finish: the worker retries after its backoff. */ + failed: unknown[] + /** Rows were written, so the next pass verifies nothing is left. */ + wrote: boolean + /** The conversation closed before this pass took the lane: nothing was checked. */ + absent?: true +} + +export async function runStructuredAgentSessionReconciliationPass( + context: StructuredAgentSessionReconciliationPassContext, + sessionId: string, + debts: StructuredAgentSessionReconciliationDebts +): Promise { + const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false } + const session = context.sessions.get(sessionId) + if (!session) { + return { ...pass, absent: true } + } + if ( + structuredAgentSessionEndedChildHoldsLease(context, sessionId) && + (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId)) + ) { + pass.failed.push(new Error('agent_session_exit_release_owed')) + } + if (debts.startupShare) { + const failedBefore = pass.failed.length + await runStartupShare(context, sessionId, session, debts.startupShare, pass) + // Done only once every step landed; a failed conversion or repair is owed still. + if (pass.failed.length === failedBefore) { + delete debts.startupShare + } + } + const settle = (proof?: AgentSessionDeathEvidence & { ownerFence: number }) => + settleStructuredAgentSessionLeftovers({ + store: context.deps.store, + sessionId, + journal: session.journal, + ...(debts.exit ? { exit: debts.exit } : {}), + ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), + ...(proof ? { proof } : {}) + }) + // Each proof the lease no longer holds judges what its own generation left, oldest first. + const leaseEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null + for (const proof of heldProofs(debts, leaseEvidence)) { + const settled = await settle(proof) + if (!settled.ok) { + pass.failed.push(settled.error) + return pass + } + pass.wrote ||= settled.planned > 0 + } + const settled = await settle() + if (!settled.ok) { + pass.failed.push(settled.error) + return pass + } + // Kept until everything they judge is settled: a later reservation clears the lease's copy. + delete debts.evidence + // Written, or overtaken by a later generation: either way the exit's account is spent. + delete debts.exit + pass.wrote ||= settled.planned > 0 + return pass +} + +function heldProofs( + debts: StructuredAgentSessionReconciliationDebts, + leaseEvidence: AgentSessionDeathEvidence | null +): (AgentSessionDeathEvidence & { ownerFence: number })[] { + return (debts.evidence ?? []) + .filter( + (evidence): evidence is AgentSessionDeathEvidence & { ownerFence: number } => + evidence.ownerFence !== undefined && + !( + leaseEvidence?.ownerFence === evidence.ownerFence && + leaseEvidence.observedAt === evidence.observedAt + ) + ) + .sort((left, right) => left.ownerFence - right.ownerFence) +} + +async function runStartupShare( + context: StructuredAgentSessionReconciliationPassContext, + sessionId: string, + { journal }: StructuredAgentSessionHostSession, + share: NonNullable, + pass: StructuredAgentSessionReconciliationPass +): Promise { + const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence + if (fence === undefined) { + return + } + const attempt = async (step: () => Promise): Promise => + step().then( + () => true, + (error: unknown) => { + pass.failed.push(error) + return false + } + ) + await attempt(() => journal.queuedMessages.repairAndPrune()) + // Before a Stop can withdraw one: a Stop never withdraws a card. + const converted = await attempt(() => + holdUnsentSends(journal, { + fence, + hostInstance: structuredAgentSessionHostInstance(), + hold: { cause: 'hostRestarted' } + }) + ) + // Once, after every card the earlier process left: it starts where this handle opened, so a + // card this process queued is never held by it. + if (converted && !share.reopenMarked) { + const opened = journal.openedAt() + const since = opened.epoch === journal.cursor().epoch ? opened.sequence + 1 : undefined + share.reopenMarked = await attempt(() => journal.markQueueReopen(fence, since)) + } + // A rewind the earlier process left prepared refuses every send until settled; one only its + // provider can prove stays for the next acquisition. + await attempt(() => recoverStructuredRewind(context.deps, sessionId, journal, fence)) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts new file mode 100644 index 000000000000..d6f541242851 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts @@ -0,0 +1,163 @@ +// The chat's reconciliation worker: what retires it, what keeps it, and what readers see before it +// has written anything. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' +import { + currentJournal, + exitChild, + exitWhileSettlementFails, + leaveUnfinishedWork, + reconciled, + recoveredRows, + REJECT_RECOVERED_ROWS, + turnState, + watchSettlementCommits +} from './structured-agent-session-leftover-settlement.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +function worker(current: QueuedMessageTestRig) { + return current.host.collaboratorsForTests().reconciliation +} + +describe('a retry dies', () => { + it('on supersession: a later settlement covered what it owed, so it writes nothing and retires', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + // A crash, and a new process whose storage refuses settlement rows from its first write; its + // recovery releases the owner it cannot prove gone. + await current.crashRestartHostProcess(() => + openTestJournalHostDatabase(current.root).db.exec(REJECT_RECOVERED_ROWS) + ) + const database = openTestJournalHostDatabase(current.root) + expect(worker(current).owes(SESSION)).toBe(true) + expect(turnState(current)).toBe('running') + database.db.exec('DROP TRIGGER reject_recovered') + + // Another settlement lands first, as a later generation's end would run it. + expect( + await settleStructuredAgentSessionLeftovers({ + store: current.store, + sessionId: SESSION, + journal: currentJournal(current) + }) + ).toMatchObject({ ok: true }) + expect(turnState(current)).toBe('unverifiable') + const commits = watchSettlementCommits() + const settled = recoveredRows(database.db) + + await reconciled(current) + + expect(commits()).toEqual([]) + expect(recoveredRows(database.db)).toEqual(settled) + }) + + it('on the chat’s deletion: no record, nothing owed', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { database } = await exitWhileSettlementFails(current) + const getRecord = current.store.getRecord + vi.spyOn(current.store, 'getRecord').mockImplementation((sessionId) => + sessionId === SESSION ? null : getRecord(sessionId) + ) + database.db.exec('DROP TRIGGER reject_recovered') + + await reconciled(current) + + expect(recoveredRows(database.db)).toEqual([]) + }) + + it('at shutdown: its timer stops, and nothing is written after', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const { database } = await exitWhileSettlementFails(current) + database.db.exec('DROP TRIGGER reject_recovered') + + current.host.stopDelivery() + + expect(worker(current).owes(SESSION)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_500)) + expect(recoveredRows(database.db)).toEqual([]) + expect(turnState(current)).toBe('running') + }) +}) + +describe('an observed exit whose release write failed', () => { + it('is Working for no reader at once, restated with no row, and the worker owes the release until it lands', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + const fence = current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + const events: AgentSessionSubscribeEvent[] = [] + const unsubscribe = await current.host.subscribe({ + id: 'reader', + sessionId: SESSION, + emit: (event) => events.push(event) + }) + const cursor = currentJournal(current).cursor() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const database = openTestJournalHostDatabase(current.root) + database.db.exec(REJECT_RECOVERED_ROWS) + const release = vi + .spyOn(current.store, 'transitionHandoff') + .mockRejectedValue(new Error('temporary storage failure')) + + await exitChild(current) + + // The lease still names the gone child, and the journal has not moved... + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + runtimeFence: fence, + claimStatus: 'live' + }) + expect(currentJournal(current).cursor()).toEqual(cursor) + expect(turnState(current)).toBe('running') + // ...yet the host's view already says nothing runs, and the reader was told so. + await vi.waitFor(() => + expect(events.at(-1)).toMatchObject({ + type: 'batch', + batch: { items: [], cursor }, + working: false, + latestTurn: null + }) + ) + + // The items settle once storage takes rows again; the release is still owed, and an empty + // item plan never retires it. + database.db.exec('DROP TRIGGER reject_recovered') + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 10_000 }) + await worker(current).idle(SESSION) + expect(worker(current).owes(SESSION)).toBe(true) + expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + + release.mockRestore() + await reconciled(current) + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + runtimeFence: fence + 1, + claimStatus: 'released', + deathEvidence: { kind: 'exit-observed', ownerFence: fence } + }) + unsubscribe() + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts new file mode 100644 index 000000000000..0d9b2bda9525 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -0,0 +1,341 @@ +// One in-memory reconciliation worker per chat: what ended generations left, and what an earlier +// host process left, settled in the background after every signal that a generation ended. +// +// Signals: every committed revocation or supersession of a generation (the record store's +// `onGenerationEnded`), an exit this host observed (whether or not its release landed), a row an +// ended generation committed late, and startup. Each one re-derives what readers see at once (the +// operational revision, the published view, the queued-card drain), then wakes the worker. +// +// The worker coalesces: a chat has at most one attempt running and one timer waiting, and every +// attempt re-derives everything owed (`runStructuredAgentSessionReconciliationPass`). An attempt +// that finds the chat closed replays its journal in one of a few background slots outside the +// chat's lane, and publishes it; only the publish and the pass take the lane, as separate steps, so +// a send that arrived meanwhile goes first. A failure backs off from 1 s to 30 s, in memory only. +// The worker retires once a pass finds nothing owed and wrote nothing, when the chat's record is +// gone, when the database is read-only, or at shutdown. A conversation it opened itself, or one the +// idle sweep left to it, it closes again on retiring. Nothing a person does (send, start, open, +// Stop, answer) ever waits on it. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' +import { PrioritySemaphore } from '../../../shared/priority-semaphore' +import type { OpenedStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' +import { + runStructuredAgentSessionReconciliationPass, + type StructuredAgentSessionReconciliationDebts, + type StructuredAgentSessionReconciliationPassContext +} from './structured-agent-session-reconciliation-pass' + +const BACKGROUND_SLOTS = 4 +export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 30_000 } as const + +export type StructuredAgentSessionReconciliationSignal = { + /** Proof the lease no longer holds (`AgentSessionGenerationEnd.evidence`). */ + evidence?: AgentSessionGenerationEnd['evidence'] + /** An observed exit whose own settlement did not land. */ + exit?: StructuredAgentSessionExitSettlement + /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ + restate?: boolean + /** This process owes the chat its startup share. */ + startup?: true +} + +export type StructuredAgentSessionReconciliationContext = + StructuredAgentSessionReconciliationPassContext & { + deps: StructuredAgentSessionReconciliationPassContext['deps'] + /** The chat's action lane. */ + serialize: (sessionId: string, task: () => Promise) => Promise + /** Lets a quit wait for an attempt already writing. */ + track: (operation: Promise) => Promise + /** Indexes and publishes a journal this worker opened. */ + adopt: (sessionId: string, opened: OpenedStructuredAgentSessionConversation) => Promise + /** Publishes what is current now and wakes the queued-card drain. */ + publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void + /** Exits a lease latched in recovery when present-time evidence permits; a no-op otherwise. */ + resolveRecovery: (sessionId: string) => Promise + /** Closes a conversation that is only a cache, as the idle sweep does; takes the lane. */ + closeAtRest: (sessionId: string) => Promise + } + +type ChatWorker = { + debts: StructuredAgentSessionReconciliationDebts + running: boolean + /** A signal arrived while an attempt ran: the next attempt follows at once. */ + dirty: boolean + /** `soon`: an attempt queued for this turn of the event loop; else a backoff's timer. */ + timer: ReturnType | 'soon' | null + failures: number + attempted: (() => void)[] + /** Closed at rest when this worker retires, if still open: one it opened, or one the idle sweep + * would have closed but for it. */ + closeOnRetire?: StructuredAgentSessionHostSession +} + +export class StructuredAgentSessionReconciliation { + private readonly workers = new Map() + private readonly slots = new PrioritySemaphore(BACKGROUND_SLOTS) + private disposed = false + private readonly unsubscribe: () => void + + constructor(private readonly context: StructuredAgentSessionReconciliationContext) { + this.unsubscribe = context.deps.store.onGenerationEnded((ended) => + this.signal(ended.sessionId, { evidence: ended.evidence }) + ) + } + + signal = (sessionId: string, signal: StructuredAgentSessionReconciliationSignal = {}): void => { + const session = this.context.sessions.get(sessionId) + if (session) { + session.operationalRevision = (session.operationalRevision ?? 0) + 1 + } + this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) + if (this.disposed) { + return + } + const worker = this.workerFor(sessionId) + if (signal.evidence) { + worker.debts.evidence = [...(worker.debts.evidence ?? []), signal.evidence] + } + if (signal.exit) { + worker.debts.exit = signal.exit + } + if (signal.startup && !worker.debts.startupShare) { + worker.debts.startupShare = { reopenMarked: false, leaseSettled: false } + } + if (worker.running) { + worker.dirty = true + } else if (worker.failures === 0 && !worker.timer) { + // Coalesced with whatever else this tick signals; during a backoff the timer runs it. + this.schedule(sessionId, worker, 0) + } + } + + /** A committed row below the lease's fence: a generation that ended wrote it late. */ + observeCommit = (sessionId: string, lowestFence: number | null): void => { + const fence = this.context.deps.store.getRecord(sessionId)?.lease.runtimeFence + if (lowestFence !== null && fence !== undefined && lowestFence < fence) { + this.signal(sessionId) + } + } + + /** Resolves once the chat's next attempt finished, or at once when it has no worker. */ + attempted = (sessionId: string): Promise => { + const worker = this.workers.get(sessionId) + return worker ? new Promise((resolve) => worker.attempted.push(resolve)) : Promise.resolve() + } + + /** The chat's startup share has not yet written its reopen mark: the queue sends nothing on its + * own until it has, so every card the earlier process left waits for a turn first. */ + awaitingReopenMark = (sessionId: string): boolean => { + const share = this.workers.get(sessionId)?.debts.startupShare + return share !== undefined && !share.reopenMarked + } + + /** Whether the chat has a worker: something was owed at its last attempt. */ + owes = (sessionId: string): boolean => this.workers.has(sessionId) + + /** The idle sweep's close of a chat this worker still owes: deferred to its retirement, so the + * worker does not open it again. False when nothing is owed and the close can go ahead. */ + deferCloseAtRest = (sessionId: string): boolean => { + const worker = this.workers.get(sessionId) + if (!worker) { + return false + } + worker.closeOnRetire = this.context.sessions.get(sessionId) + return true + } + + /** Resolves once the chat's worker retired, or is waiting out a backoff. */ + idle = async (sessionId: string): Promise => { + for (;;) { + const worker = this.workers.get(sessionId) + if (!worker || (!worker.running && worker.failures > 0)) { + return + } + await new Promise((resolve) => worker.attempted.push(resolve)) + } + } + + /** Shutdown: every timer stops; an attempt already writing finishes, tracked. */ + dispose = (): void => { + this.disposed = true + this.unsubscribe() + for (const [sessionId, worker] of this.workers) { + this.retire(sessionId, worker) + } + } + + private workerFor(sessionId: string): ChatWorker { + let worker = this.workers.get(sessionId) + if (!worker) { + worker = { debts: {}, running: false, dirty: false, timer: null, failures: 0, attempted: [] } + this.workers.set(sessionId, worker) + } + return worker + } + + private attempt(sessionId: string): void { + const worker = this.workers.get(sessionId) + if (!worker || worker.running) { + return + } + worker.timer = null + worker.running = true + worker.dirty = false + void this.context + .track(this.runAttempt(sessionId, worker)) + .then((outcome) => this.afterAttempt(sessionId, worker, outcome)) + } + + private async runAttempt( + sessionId: string, + worker: ChatWorker + ): Promise<'retire' | 'again' | 'failed'> { + try { + const { store } = this.context.deps + if (this.disposed || store.readOnly || !store.getRecord(sessionId)) { + return 'retire' + } + const share = worker.debts.startupShare + // Once per startup share, before it publishes: a lease latched in recovery is decided first, + // as the restore decides it, so no reader sees a gone owner's turn as running. + if (share && !share.leaseSettled) { + await this.inSlot(() => this.context.resolveRecovery(sessionId)).catch((error: unknown) => + this.warn(sessionId, error) + ) + share.leaseSettled = true + } + // Replayed in a background slot outside the chat's lane, and published before the pass: a + // reader or a send never waits on the replay, and the lane is held only to publish and for + // the pass's recheck and writes. + if (!this.context.sessions.has(sessionId) && !(await this.load(sessionId, worker))) { + return 'retire' + } + const held = this.awaitingReopenMark(sessionId) + const pass = await this.context.serialize(sessionId, () => + runStructuredAgentSessionReconciliationPass(this.context, sessionId, worker.debts) + ) + if (held && !this.awaitingReopenMark(sessionId)) { + // The queue may send on its own again. + this.context.publishGenerationEnded(sessionId) + } + // Closed while this attempt waited for the lane: the next one opens it again. + if (pass.absent) { + return 'again' + } + if (pass.failed.length > 0) { + this.warn(sessionId, pass.failed[0]) + return 'failed' + } + return pass.wrote ? 'again' : 'retire' + } catch (error) { + this.warn(sessionId, error) + return 'failed' + } + } + + private warn(sessionId: string, error: unknown): void { + this.context.deps.logger.warn("settling a gone agent's leftover work did not finish", { + scope: 'reconciliation', + sessionId, + error + }) + } + + private async inSlot(task: () => Promise): Promise { + const release = await this.slots.acquire(0) + try { + return await task() + } finally { + release() + } + } + + /** Replays the journal in a background slot, then publishes it; false when it has none. */ + private async load(sessionId: string, worker: ChatWorker): Promise { + const opened = await this.inSlot(async () => { + // A replay is synchronous SQLite: a macrotask before each keeps a send or a read from waiting + // behind a whole scan's worth of them. + await yieldToEventLoop() + return this.context.sessions.has(sessionId) + ? null + : restoreStructuredAgentSessionRead(this.context.deps, sessionId) + }) + if (!opened) { + // Open already, or no journal: nothing was ever written, so nothing is owed. + return this.context.sessions.has(sessionId) + } + const replayed = opened + // Published under the lane, so it never replaces a conversation a reader or a send opened + // meanwhile; theirs is the conversation, and this replay is dropped. + await this.context.serialize(sessionId, async () => { + if (this.context.sessions.has(sessionId)) { + await replayed.session.journal.close() + return + } + await this.context.adopt(sessionId, replayed) + worker.closeOnRetire = this.context.sessions.get(sessionId) + }) + return true + } + + private afterAttempt( + sessionId: string, + worker: ChatWorker, + outcome: 'retire' | 'again' | 'failed' + ): void { + worker.running = false + const attempted = worker.attempted.splice(0) + attempted.forEach((resolve) => resolve()) + if (this.workers.get(sessionId) !== worker || this.disposed) { + return + } + if (outcome === 'failed') { + worker.failures += 1 + const delay = Math.min( + RECONCILIATION_BACKOFF_MS.max, + RECONCILIATION_BACKOFF_MS.first * 2 ** (worker.failures - 1) + ) + this.schedule(sessionId, worker, delay) + return + } + worker.failures = 0 + if (outcome === 'again' || worker.dirty) { + this.schedule(sessionId, worker, 0) + return + } + this.retire(sessionId, worker) + // Only while it is still that conversation: a reader's reopen is theirs. + if (worker.closeOnRetire && this.context.sessions.get(sessionId) === worker.closeOnRetire) { + void this.context + .closeAtRest(sessionId) + .catch((error: unknown) => this.warn(sessionId, error)) + } + } + + private schedule(sessionId: string, worker: ChatWorker, delay: number): void { + if (delay === 0) { + worker.timer = 'soon' + queueMicrotask(() => this.attempt(sessionId)) + return + } + const timer = setTimeout(() => this.attempt(sessionId), delay) + // A backoff alone never keeps the process alive. + timer.unref?.() + worker.timer = timer + } + + private retire(sessionId: string, worker: ChatWorker): void { + if (worker.timer && worker.timer !== 'soon') { + clearTimeout(worker.timer) + } + worker.timer = null + this.workers.delete(sessionId) + if (!worker.running) { + worker.attempted.splice(0).forEach((resolve) => resolve()) + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts index e286c8a58ae8..955992c36dca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts @@ -76,6 +76,9 @@ describe('an expired send id', () => { it('is answered expired while its chat is closed, not as a chat this host lacks', async () => { await attach() await host.close(SESSION, 'evict') + // The stop's release wakes the chat's worker, which opens it, finds nothing owed, and closes it. + await host.collaboratorsForTests().reconciliation.idle(SESSION) + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) await expect(host.send(CALLER, expiredParams('long gone'))).resolves.toMatchObject(EXPIRED) expect(host.hasSession(SESSION)).toBe(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts index 639cdc9b3ae7..1d14597a546b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -225,8 +225,12 @@ export async function foundRestTestChat(rig: RestTestRig): Promise { } /** Runs one sweep pass now, for a test that set `idleSweep.intervalMs` out of reach. */ -export function sweepOnce(host: StructuredAgentSessionHost): Promise { - return host.collaboratorsForTests().lifetime.idleSweep.tick() +export async function sweepOnce(host: StructuredAgentSessionHost): Promise { + const { lifetime, reconciliation, serialize } = host.collaboratorsForTests() + await lifetime.idleSweep.tick() + // A close the sweep left to the chat's reconciliation worker lands once that retires. + await reconciliation.idle(SESSION) + await serialize(SESSION, async () => {}) } /** Waits long enough for several sweep ticks to have run. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index d7d59add30fb..6c6efc49d728 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -40,12 +40,6 @@ export type StructuredAgentSessionReadRestoreDeps = { sessionId: string, opened: OpenedStructuredAgentSessionConversation ) => Promise | void - /** Inside the chat's serialize, once its recovery is resolved: what that ended is settled, for - * a chat this restore opened (`opened`) before `onReadable` publishes it. */ - afterReadable?: ( - sessionId: string, - opened?: OpenedStructuredAgentSessionConversation - ) => Promise } /** One session's share of the restart restore. Startup maps this over every supported record. */ @@ -62,26 +56,19 @@ async function restoreOneStructuredAgentSessionRead( /** The serialized half of the restore. */ async function restoreOneStructuredAgentSessionReadUnderSerialize( - input: Pick< - StructuredAgentSessionReadRestoreDeps, - 'openDeps' | 'hasSession' | 'onReadable' | 'afterReadable' - >, + input: Pick, sessionId: string ): Promise { - // A read or a send mid-restore may already have opened this one. if (input.hasSession(sessionId)) { - await input.afterReadable?.(sessionId) + // A read or a send mid-restore already opened this one. return } const opened = await restoreStructuredAgentSessionRead(input.openDeps, sessionId) if (!opened) { return } - try { - await input.afterReadable?.(sessionId, opened) - } finally { - await input.onReadable(sessionId, opened) - } + // The open settled what a gone generation left running, so no reader sees it run. + await input.onReadable(sessionId, opened) } export async function restoreStructuredAgentSessionsOnRestart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 1c11acb1ef47..a846c5a1628a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -23,11 +23,11 @@ import type { StructuredAgentSessionHostDeps, StructuredAgentSessionReveal } from './structured-agent-session-host-types' +import { scanStructuredAgentSessionsAtStartup } from './structured-agent-session-startup-settlement' import { - settleOneAtStartup, - settleStructuredAgentSessionsAtStartup, - type StructuredAgentSessionStartupSettlementContext -} from './structured-agent-session-startup-settlement' + StructuredAgentSessionReconciliation, + type StructuredAgentSessionReconciliationContext +} from './structured-agent-session-reconciliation-worker' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' /** Throws its refusal as the code itself. */ @@ -58,46 +58,55 @@ export async function revealStructuredAgentSession( } } -/** The host's startup: reconcile, then settle what the earlier process left (in the background: - * no chat's tab waits on it), and the readable-restore sweep that resolves and opens each chat. - * Its lease bookkeeping is a reader's, which never fails a read or startup; startup shares it. */ +/** The host's startup and its reconciliation: reconcile, then hand every chat to its reconciliation + * worker in the background (no chat's tab or send waits on it), and the readable-restore sweep + * that resolves and opens each chat. Its lease bookkeeping is a reader's, which never fails a read + * or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< ConstructorParameters[0], - 'openDeps' | 'reconcile' | 'resolveRecovery' | 'afterReadable' + 'openDeps' | 'reconcile' | 'resolveRecovery' > & { reconcileLeases: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise - startup: Pick & { + startup: Pick & { tasks: Pick + clientDelivery: Pick } + closeAtRest: StructuredAgentSessionReconciliationContext['closeAtRest'] } ): { reconcileRestartLeases: () => Promise - /** The startup settlement the last reconcile began; resolved once every chat's share ran. */ + /** The startup scan the last reconcile began; resolved once every chat's first attempt ran. */ startupSettled: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise + reconciliation: StructuredAgentSessionReconciliation } { - const { reconcileLeases, resolveRecovery, startup, ...rest } = wiring + const { reconcileLeases, resolveRecovery, startup, closeAtRest, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) const reconcile = createReaderReconcile(reconcileLeases, failures) - const settlement: StructuredAgentSessionStartupSettlementContext = { + const reconciliation = new StructuredAgentSessionReconciliation({ deps, sessions: startup.sessions, + now: () => deps.now?.() ?? Date.now(), serialize: rest.serialize, + // Tracked like a start, so a quit waits for an attempt before it closes what it writes to. + track: (operation) => startup.tasks.trackAttach(operation), adopt: async (sessionId, opened) => { await rest.onReadable(sessionId, opened) }, - now: () => deps.now?.() ?? Date.now(), - sharesDone: new WeakSet() - } + publishGenerationEnded: (sessionId, options) => + startup.clientDelivery.publishGenerationEnded(sessionId, options), + // A lease latched in recovery is decided before the share publishes the chat, as the restore + // decides it; a failure is the worker's to report, and the next attach or send resolves it. + resolveRecovery, + closeAtRest + }) let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ openDeps: deps, reconcile, - // A recovery the restore resolved can end a generation the startup pass found still owned. - afterReadable: (sessionId, opened) => settleOneAtStartup(settlement, sessionId, opened), // The next attach or send resolves recovery again, strictly, before it acts. resolveRecovery: (sessionId) => resolveRecovery(sessionId).then( @@ -113,10 +122,10 @@ export function createStructuredAgentSessionHostRestore( return { reconcileRestartLeases: async () => { await reconcile('startup') - // Tracked like a start, so a quit waits for it before it closes what it writes to. - startupSettled = startup.tasks.trackAttach(settleStructuredAgentSessionsAtStartup(settlement)) + startupSettled = scanStructuredAgentSessionsAtStartup(deps.store, reconciliation) }, startupSettled: () => startupSettled, - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)) + restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), + reconciliation } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts new file mode 100644 index 000000000000..b81e9aec3bd1 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts @@ -0,0 +1,277 @@ +// A roster row and a background-task row are revised in place across generations, so each entry +// carries the generation that last observed it. Settling an ended generation settles only the +// entries it observed; a later live generation's children stay as they are. And a proof the +// reconciliation worker holds for an ended generation revises what an earlier settle of it could +// only call unverifiable. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentJournalItemBody +} from '../../../shared/agent-session-journal-types' +import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../../shared/agent-session-record.test-fixture' +import { backgroundTaskJournalBody } from '../../../shared/native-chat-background-task-row' +import { + isBackgroundTaskBlock, + isSubagentGroupBlock, + type NativeChatBackgroundTaskBlock, + type NativeChatSubagentEntry +} from '../../../shared/native-chat-types' +import { + claudeBackgroundTaskIdentity, + claudeBackgroundTaskBody +} from '../../claude/claude-background-task-row-journal' +import { + claudeSubagentGroupBody, + claudeSubagentGroupIdentity +} from '../../claude/claude-subagent-group-row' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import { stampJournalEntryProvenance } from '../agent-session-journal/journal-item-provenance' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { createDeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' +import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' +import { testEventSinkLogging } from './structured-agent-session-logger-test-support' + +const SESSION = 'orca-session' +const GROUP = 'claude-session:turn-a' +const ENDED = 1 +const LIVE = 3 + +const journals = createTrackedJournalOpener() +let root: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-roster-provenance-')) +}) + +afterEach(async () => { + await journals.closeAll() + await rm(root, { recursive: true, force: true }) +}) + +function child(id: string, extra: Partial = {}): NativeChatSubagentEntry { + return { id, label: `Child ${id}`, state: 'working', startedAt: 1, ...extra } +} + +function task( + taskId: string, + extra: Partial = {} +): NativeChatBackgroundTaskBlock { + return { + type: 'background-task', + taskId, + kind: 'command', + label: `Task ${taskId}`, + state: 'working', + startedAt: 1, + ...extra + } +} + +/** One provider process of generation `fence`, observing rows through the host's event sink. */ +async function observeAt( + journal: AgentSessionJournal, + fence: number, + rows: readonly (readonly [ReturnType, AgentJournalItemBody])[] +): Promise { + const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) + deferred.bind({ journal, fence, publish: () => {} }) + for (const [identity, body] of rows) { + deferred.sink.appendItem(identity, body, { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) + } + await expect(deferred.drained()).resolves.toEqual({ ok: true }) + deferred.close() +} + +function children(journal: AgentSessionJournal): NativeChatSubagentEntry[] { + return journal + .snapshot() + .items.flatMap((item) => + item.body.kind === 'message' + ? item.body.blocks.flatMap((block) => (isSubagentGroupBlock(block) ? block.agents : [])) + : [] + ) +} + +function tasks(journal: AgentSessionJournal): NativeChatBackgroundTaskBlock[] { + return journal + .snapshot() + .items.flatMap((item) => + item.body.kind === 'message' ? item.body.blocks.filter(isBackgroundTaskBlock) : [] + ) +} + +function openJournal(): Promise { + return journals.open({ + identity: { + sessionId: SESSION, + workspaceId: 'workspace-1', + hostId: 'local', + agent: 'claude', + providerHandle: claudeProviderHandle('claude-session', null) + }, + now: () => 9_000, + stateDirectory: root + }) +} + +describe('a roster entry’s provenance', () => { + const group = (agents: NativeChatSubagentEntry[]) => claudeSubagentGroupBody(GROUP, agents) + const agentsOf = (body: AgentJournalItemBody) => + body.kind === 'message' + ? body.blocks.flatMap((b) => (isSubagentGroupBlock(b) ? b.agents : [])) + : [] + + it('is the observing generation’s for a new or changed entry, and kept for one carried unchanged', () => { + const before = group([child('a', { ownerFence: ENDED }), child('b', { ownerFence: ENDED })]) + const after = stampJournalEntryProvenance( + group([child('a'), child('b', { state: 'completed' }), child('c')]), + { body: before, ownerFence: ENDED }, + LIVE + ) + expect(agentsOf(after).map(({ id, ownerFence }) => [id, ownerFence])).toEqual([ + ['a', ENDED], + ['b', LIVE], + ['c', LIVE] + ]) + }) + + it('falls back to the row’s own provenance for an entry an older build never stamped', () => { + const before = group([child('a')]) + const after = stampJournalEntryProvenance( + group([child('a')]), + { body: before, ownerFence: 2 }, + LIVE + ) + expect(agentsOf(after)[0]?.ownerFence).toBe(2) + }) + + it('keeps a stamp history carried into a new epoch already names', () => { + const after = stampJournalEntryProvenance( + group([child('a', { ownerFence: ENDED })]), + undefined, + LIVE + ) + expect(agentsOf(after)[0]?.ownerFence).toBe(ENDED) + }) +}) + +describe('settling an ended generation’s roster', () => { + it('settles only the children and tasks it observed; the live generation’s stay working', async () => { + const journal = await openJournal() + // Generation F spawns a child and starts two background commands, then dies uncleaned. + await observeAt(journal, ENDED, [ + [claudeSubagentGroupIdentity(GROUP), claudeSubagentGroupBody(GROUP, [child('a')])], + [claudeBackgroundTaskIdentity('bg-1'), claudeBackgroundTaskBody(task('bg-1'))], + [claudeBackgroundTaskIdentity('bg-2'), claudeBackgroundTaskBody(task('bg-2'))] + ]) + // Generation F+1 resumes the same group row with a new child of its own, carrying F's child + // unchanged, and reports progress on one of F's commands, which it now runs. + await observeAt(journal, LIVE, [ + [ + claudeSubagentGroupIdentity(GROUP), + claudeSubagentGroupBody(GROUP, [child('a'), child('b')]) + ], + [ + claudeBackgroundTaskIdentity('bg-2'), + backgroundTaskJournalBody(task('bg-2', { tokens: 50 })) + ] + ]) + expect(children(journal).map(({ id, ownerFence }) => [id, ownerFence])).toEqual([ + ['a', ENDED], + ['b', LIVE] + ]) + expect(tasks(journal).map(({ taskId, ownerFence }) => [taskId, ownerFence])).toEqual([ + ['bg-1', ENDED], + ['bg-2', LIVE] + ]) + + // F+1 holds the lease, live; the settlement judges everything below it. + const live = agentSessionRecordFixture( + agentSessionLeaseFixture({ sessionId: SESSION, runtimeFence: LIVE }) + ) + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: (sessionId) => (sessionId === SESSION ? live : null) }, + sessionId: SESSION, + journal + }) + ).toMatchObject({ ok: true }) + + expect(children(journal).map(({ id, state }) => [id, state])).toEqual([ + ['a', 'unverifiable'], + ['b', 'working'] + ]) + expect(tasks(journal).map(({ taskId, state }) => [taskId, state])).toEqual([ + ['bg-1', 'unverifiable'], + ['bg-2', 'working'] + ]) + // Settled once: a second pass finds nothing of F's left. + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => live }, + sessionId: SESSION, + journal + }) + ).toEqual({ ok: true, planned: 0 }) + }) +}) + +describe('a held proof', () => { + it('upgrades a turn an unproven settle left unverifiable to interrupted, once the lease forgot it', async () => { + const journal = await openJournal() + const deferred = createDeferredStructuredAgentSessionEventSink(testEventSinkLogging()) + deferred.bind({ journal, fence: ENDED, publish: () => {} }) + deferred.sink.appendItem( + { provider: 'claude', sessionId: 'claude-session', uuid: 'turn' }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: 1_000 }, + { turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + await expect(deferred.drained()).resolves.toEqual({ ok: true }) + deferred.close() + const turn = () => + journal + .snapshot() + .items.map((item) => readAgentJournalTurn(item.body)) + .find(Boolean) + // Released with nothing proving the owner gone: less specific, never wrong. + const released = agentSessionRecordFixture( + agentSessionLeaseFixture({ + sessionId: SESSION, + runtimeFence: ENDED + 1, + claimStatus: 'released', + ownerProcess: null, + reservedSpawnToken: null + }) + ) + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => released }, + sessionId: SESSION, + journal + }) + expect(turn()).toMatchObject({ state: 'unverifiable' }) + + // A later probe proved that owner gone; a reservation since cleared the lease's copy, and the + // worker holds the proof it was handed. + const live = agentSessionRecordFixture( + agentSessionLeaseFixture({ sessionId: SESSION, runtimeFence: LIVE }) + ) + expect( + await settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => live }, + sessionId: SESSION, + journal, + proof: { kind: 'pid-absent', detail: 'owner gone', observedAt: 2_000, ownerFence: ENDED } + }) + ).toMatchObject({ ok: true }) + expect(turn()).toMatchObject({ state: 'interrupted' }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts index 63337ea24e3a..fb7f7ef81e18 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -112,7 +112,10 @@ it.each(PROBES)( 'settles a turn a dead generation left running when a send opens the chat and its start fails, when %s', async (_when, probe, settled) => { const { host, acquire } = await relaunchAfterCrashMidTurn(probe) - expect(host.hasSession(SESSION)).toBe(false) + // The startup's worker opens the chat in the background and closes it again once done. + await host.startupSettled() + await host.collaboratorsForTests().reconciliation.idle(SESSION) + await eventually(() => expect(host.hasSession(SESSION)).toBe(false)) const body = hostTestMessage('sent to a chat nobody has open') const sendEnvelope = envelope('agentSession.send', { body }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts index 67dd06ed8c1e..eb6dc1c28be0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-settlement-double.test-fixture.ts @@ -1,6 +1,5 @@ // Doubles for unit tests that drive a settlement without a journal database: a planned batch is -// planned at once and handed to an `appendLifecycleBatch` mock as the batch it would write, and a -// store's leftover receipt commits nothing. +// planned at once and handed to an `appendLifecycleBatch` mock as the batch it would write. import type { AgentJournalCursor, @@ -12,7 +11,6 @@ import type { JournalLifecycleBatchInput, JournalPlannedLifecycleBatchInput } from '../agent-session-journal/journal-store-contracts' -import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' const NO_CURSOR: AgentJournalCursor = { epoch: 'epoch-1', sequence: 0 } @@ -24,7 +22,6 @@ export function plannedBatchThrough( return async (input) => { const { mutations, dispatches } = input.plan() if (mutations.length === 0 && dispatches.length === 0) { - input.receipt?.committed() return NO_CURSOR } const cursor = await append({ @@ -34,16 +31,10 @@ export function plannedBatchThrough( mutations, dispatches }) - input.receipt?.committed() return cursor } } -/** A store's receipts, for a store double: each commits nothing. */ -export const NO_STORE_RECEIPTS = { - leftoverSettled: (): JournalOperationReceipt => ({ write: () => {}, committed: () => {} }) -} - /** An exited child's journal for a test without a database: the reads a settlement makes, with a * planned batch handed to `appendLifecycleBatch` (`plannedBatchThrough`). */ export function exitJournalDouble(fields: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts index 974fbff54dbf..19771952de13 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-state-settlement.ts @@ -14,10 +14,10 @@ import type { AgentSessionFailureWordsContext } from '../../../shared/agent-sess import { journalPendingSubmissionResolutions } from '../agent-session-journal/journal-pending-submission-recovery' import { DISPATCH_DOUBT_PROVIDER_EXITED } from '../agent-session-journal/journal-dispatch-doubt-reasons' import type { JournalLifecycleMutationInput } from '../agent-session-journal/journal-row-builders' -import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { ResolveDispatchInput } from '../agent-session-journal/journal-store-contracts' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { lostLiveWorkJournalBody } from '../agent-session-journal/journal-subagent-liveness' +import { journalEntryOwnerFence } from '../agent-session-journal/journal-item-provenance' import { endedUnseenMessageBody, runningCallEnd, @@ -59,8 +59,10 @@ export type StaleStructuredAgentSessionStateInput = { /** Only what generations below this fence wrote, which can no longer write, is settled; their * sends handed over and never answered become doubt. Absent: every item, and no send. */ below?: number - /** Committed in the same transaction as the rows, even when nothing is left to settle. */ - receipt?: JournalOperationReceipt + /** Roster entries and background tasks are settled by their own provenance + * (`journal-item-provenance.ts`): below this fence, in any row, even a live generation's. + * Absent: every entry of a row settled here, and no other row's. */ + entriesBelow?: number } /** Settles it, and answers how many items it revised. */ @@ -75,7 +77,6 @@ export async function settleStaleStructuredAgentSessionState( settlementId: `${STALE_SESSION_ROW_PREFIX}${input.sessionId}:${input.fence}:${generation}`, fence: input.fence, recovered: true, - ...(input.receipt ? { receipt: input.receipt } : {}), plan: () => { const plan = planStaleStructuredAgentSessionState(input) planned = plan.mutations.length @@ -95,7 +96,8 @@ export function planStaleStructuredAgentSessionState( const { journal, below } = input const ended = (fence: number | undefined) => below === undefined || (fence !== undefined && fence < below) - const items = journal.snapshot().items.filter((item) => ended(journal.itemFence(item.itemId))) + const all = journal.snapshot().items + const items = all.filter((item) => ended(journal.itemFence(item.itemId))) const dispatches = below === undefined ? [] @@ -105,7 +107,37 @@ export function planStaleStructuredAgentSessionState( // Doubt, never non-delivery: what a gone owner was handed and never answered. { reason: DISPATCH_DOUBT_PROVIDER_EXITED } ) - return { mutations: staleMutations(input, items), dispatches } + const mutations = staleMutations(input, items) + const { entriesBelow } = input + if (entriesBelow !== undefined) { + // A live generation's row can still carry a child or a task an ended one ran. + for (const item of all) { + const itemFence = journal.itemFence(item.itemId) + const body = ended(itemFence) + ? null + : lostLiveWorkJournalBody(item.body, (entry) => + entryEndedBelow(entry, itemFence, entriesBelow) + ) + if (body) { + mutations.push({ + kind: 'item', + itemId: item.itemId, + body, + turnScope: item.turnScope ?? AGENT_JOURNAL_THREAD_SCOPE + }) + } + } + } + return { mutations, dispatches } +} + +function entryEndedBelow( + entry: { ownerFence?: number }, + itemFence: number | undefined, + below: number +): boolean { + const fence = journalEntryOwnerFence(entry, itemFence) + return fence !== undefined && fence < below } function staleMutations( @@ -127,8 +159,14 @@ function staleMutations( // leaves them to the evidence. const end = runningCallEnd(item.turnScope, (id) => journal.itemBody(id), verdictFor(item).state) const terminal = endedUnseenMessageBody(item.body) ?? terminalAgentJournalBody(item.body, end) - // One revision per item: its live subagents and background tasks end with it. - const body = lostLiveWorkJournalBody(terminal ?? item.body) ?? terminal + // One revision per item: its live subagents and background tasks end with it, but for any a + // live generation observed since. + const itemFence = journal.itemFence(item.itemId) + const { entriesBelow } = input + const body = + lostLiveWorkJournalBody(terminal ?? item.body, (entry) => + entriesBelow === undefined ? true : entryEndedBelow(entry, itemFence, entriesBelow) + ) ?? terminal if (body) { mutations.push({ kind: 'item', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts index 7bd37bcaa590..85432b2b901a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts @@ -1,6 +1,6 @@ -// A turn an exited agent left running, whose settlement write failed, is settled at the next -// ownership event: the next send's acquisition, or the next startup. A reader opening the chat in -// between writes nothing, and the turn holds nothing: it is an ended generation's. +// A turn an exited agent left running, whose settlement write failed, is settled by the chat's +// reconciliation worker once storage takes it, or by the next startup. A reader opening the chat +// in between writes nothing, and the turn holds nothing: it is an ended generation's. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' @@ -11,9 +11,7 @@ import { createRestTestRig, foundRestTestChat, IDLE_MS, - REST_TEST_CALLER, REST_TEST_SESSION as SESSION, - restTestSend, REST_TEST_THREAD, sweepOnce, type RestTestRig @@ -85,30 +83,37 @@ describe('a turn its gone agent left running', () => { expect(rig.adapter.acquire).toHaveBeenCalledOnce() }) - it('stays as it was through an idle-closed reopen, and the next send settles it first', async () => { - await exitWithUnwrittenSettlement() + it('is settled by its chat’s worker once storage takes the write, with no send; a reader writes nothing meanwhile', async () => { + // The exit's own write and the worker's first attempt both fail; the backoff's retry lands. + await exitWithUnwrittenSettlement(2) + const { reconciliation } = rig.host.collaboratorsForTests() + await reconciliation.idle(SESSION) + expect(reconciliation.owes(SESSION)).toBe(true) expect(await workingTurnState()).toBe('running') - rig.clock.now += IDLE_MS + 1 - await sweepOnce(rig.host) - expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) + const cursor = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal.cursor() const reader = collectSubscriber() await rig.host.subscribe({ id: 'reader', sessionId: SESSION, emit: reader.emit }) - expect(await workingTurnState()).toBe('running') + expect(rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal.cursor()).toEqual(cursor) - const fence = rig.store.getRecord(SESSION)!.lease.runtimeFence - expect(await rig.host.send(REST_TEST_CALLER, restTestSend('next', fence))).toMatchObject({ - ok: true + await vi.waitFor(async () => expect(await workingTurnState()).toBe('interrupted'), { + timeout: 5_000 }) - await vi.waitFor(async () => expect(await workingTurnState()).toBe('interrupted')) // The death evidence is Orca's log text: the row says only that the provider stopped. expect(JSON.stringify(reader.events)).toContain( 'Codex stopped while this response was in progress. You can continue in this conversation.' ) + expect(rig.adapter.acquire).toHaveBeenCalledOnce() + // Nothing owed once it landed: the idle sweep closes the chat as it would any other. + await reconciliation.idle(SESSION) + rig.clock.now += IDLE_MS + 1 + await sweepOnce(rig.host) + expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) }) it('is settled by the startup after a restart, even when a read reaches the chat first', async () => { - await exitWithUnwrittenSettlement() + // Storage refuses it for the rest of this process. + await exitWithUnwrittenSettlement(100) await rig.restart() setStructuredAgentSessionHost(rig.host) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index 36a4ac14a219..9414a69b9fc4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -12,10 +12,7 @@ import { } from './structured-agent-session-child-exit' import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import type { AgentJournalSubmission } from '../../../shared/agent-session-journal-types' -import { - exitJournalDouble, - NO_STORE_RECEIPTS -} from './structured-agent-session-settlement-double.test-fixture' +import { exitJournalDouble } from './structured-agent-session-settlement-double.test-fixture' const SESSION = 'session-1' const GENERATION = 'generation-1' @@ -60,8 +57,7 @@ function contextFor(session: StructuredAgentSessionChildExitSession) { transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)), - conversationReceipts: NO_STORE_RECEIPTS + ) => (record = transition(record)) }, sessions: new Map([[SESSION, session]]), flushLifecycle: async () => ({ ok: true }), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts new file mode 100644 index 000000000000..fecf478fd3e6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts @@ -0,0 +1,279 @@ +// Startup's scan of every chat on record: in the background, never ahead of a person's own action +// on a chat, with no stored mark saying which chats are owed anything. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' + +const NOW = 1_800_000_000_000 +const CHATS = ['chat-aaaaaaa1', 'chat-aaaaaaa2', 'chat-aaaaaaa3', 'chat-aaaaaaa4', 'chat-aaaaaaa5'] +const LATE = 'chat-aaaaaaa6' +/** Released long ago with nothing proving its owner gone: no restart moves its fence. */ +const RELEASED = 'chat-released1' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +function record(sessionId: string, released: boolean): AgentSessionRecord { + const linkId = `${sessionId}-link` + return { + schemaVersion: 2, + sessionId, + location: LOCATION, + provider: 'claude', + providerHandleChain: [ + { + linkId, + handle: claudeProviderHandle(`provider-${sessionId}`, null), + origin: 'created', + mintedAtFence: 13, + observedAt: NOW - 60_000 + } + ], + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + createdAt: NOW - 60_000, + updatedAt: NOW, + lease: { + sessionId, + runtimeKind: 'native', + runtimeFence: released ? 14 : 13, + handoffStage: null, + provenHandleLinkId: linkId, + ownerProcess: released + ? null + : { hostId: 'local', pid: 12_000, processStartTimeMs: NOW - 60_000, spawnToken: 'spawn' }, + reservedSpawnToken: released ? null : 'spawn', + leaseDeadlineAt: NOW + 30_000, + lastRenewedAt: NOW, + handoffOperationId: null, + journalCheckpoint: null, + claimKeyId: 'key-1', + claimStatus: released ? 'released' : 'live', + unreconciled: false, + deathEvidence: null + } + } +} + +/** The chat's journal as the crashed process left it: a running turn at fence 13, and, when + * asked, a send it accepted and never handed over. */ +async function seedJournal(sessionId: string, options: { queued?: true } = {}): Promise { + const journal = await openAgentSessionJournal({ + identity: { + sessionId, + workspaceId: LOCATION.workspaceId, + hostId: LOCATION.executionHostId, + agent: 'claude', + providerHandle: claudeProviderHandle(`provider-${sessionId}`, null) + }, + database: openTestJournalHostDatabase(root), + now: () => NOW + }) + await journal.appendItem( + { provider: 'claude', sessionId: `provider-${sessionId}`, uuid: 'turn' }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: NOW - 30_000 }, + { fence: 13, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + if (options.queued) { + await journal.appendSubmission({ + clientMessageId: `${sessionId}-queued`, + payloadFingerprint: '0'.repeat(64), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, + fence: 14, + handoverRecorded: true, + origin: 'client', + source: { kind: 'user' } + }) + } + await journal.close() +} + +function openHost(): StructuredAgentSessionHost { + host = new StructuredAgentSessionHost({ + agents: NO_STRUCTURED_AGENTS, + logger: createStructuredAgentSessionLogger(), + store, + adapter: { + acquire: vi.fn(), + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => NOW + }) + return host +} + +async function turnState(current: StructuredAgentSessionHost, sessionId: string) { + return (await current.journalSnapshot(sessionId)).items + .map((item) => readAgentJournalTurn(item.body)) + .find(Boolean)?.state +} + +async function idle(current: StructuredAgentSessionHost, sessionIds: readonly string[]) { + const { reconciliation } = current.collaboratorsForTests() + await vi.waitFor( + () => expect(sessionIds.filter((sessionId) => reconciliation.owes(sessionId))).toEqual([]), + { timeout: 10_000 } + ) +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-scan-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +describe('the startup scan', () => { + it('never puts a chat late in the scan behind its own share: a read and the lane go first', async () => { + const all = [...CHATS, LATE] + await seedTestAgentSessionRecordStore(root, { records: all.map((id) => record(id, false)) }) + for (const sessionId of all) { + await seedJournal(sessionId) + } + store = await openTestAgentSessionRecordStore(root) + // The first opens the scan makes hold every background slot until the test lets them go. + let release: () => void = () => undefined + const gate = new Promise((resolve) => (release = resolve)) + const open = AgentSessionJournal.prototype.open + let gated = 0 + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + gated += 1 + if (gated <= 4) { + await gate + } + return open.call(this) + }) + const current = openHost() + const events: string[] = [] + const { sessions, serialize } = current.collaboratorsForTests() + const set = sessions.set.bind(sessions) + vi.spyOn(sessions, 'set').mockImplementation((sessionId, session) => { + events.push(`publish:${sessionId}`) + return set(sessionId, session) + }) + const shares: string[] = [] + const mark = AgentSessionJournal.prototype.markQueueReopen + vi.spyOn(AgentSessionJournal.prototype, 'markQueueReopen').mockImplementation(async function ( + this: AgentSessionJournal, + ...args + ) { + const owner = [...sessions].find(([, session]) => session.journal === this)?.[0] + shares.push(owner ?? 'unknown') + events.push(`share:${owner}`) + return mark.apply(this, args) + }) + + await current.reconcileRestartLeases() + await vi.waitFor(() => expect(gated).toBe(4)) + + // The person's own actions on the late chat run now, ahead of its share. + const order: string[] = [] + await serialize(LATE, async () => { + order.push('lane') + }) + expect(await turnState(current, LATE)).toBe('running') + order.push('read') + expect(shares).not.toContain(LATE) + + release() + await current.startupSettled() + await idle(current, all) + expect(order).toEqual(['lane', 'read']) + for (const sessionId of all) { + expect(await turnState(current, sessionId)).toBe('interrupted') + // Published before its share wrote anything. + expect(events.indexOf(`publish:${sessionId}`)).toBeGreaterThanOrEqual(0) + expect(events.indexOf(`publish:${sessionId}`)).toBeLessThan( + events.indexOf(`share:${sessionId}`) + ) + } + expect(new Set(shares)).toEqual(new Set(all)) + }) + + it('finds what an earlier process left with no fence move and no stored mark, and keeps its send as a card', async () => { + await seedTestAgentSessionRecordStore(root, { records: [record(RELEASED, true)] }) + await seedJournal(RELEASED, { queued: true }) + store = await openTestAgentSessionRecordStore(root) + const current = openHost() + const ended = vi.fn() + store.onGenerationEnded(ended) + + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [RELEASED]) + + // Already released: the restart moved nothing and ended no generation. + expect(ended).not.toHaveBeenCalled() + expect(store.getRecord(RELEASED)?.lease.runtimeFence).toBe(14) + // Nothing proved its owner gone: less specific, never wrong. + expect(await turnState(current, RELEASED)).toBe('unverifiable') + const { journal } = current.collaboratorsForTests().sessions.get(RELEASED)! + expect(journal.submission(`${RELEASED}-queued`)).toMatchObject({ dispatchState: 'rejected' }) + expect(journal.queuedMessages.list()).toEqual([ + expect.objectContaining({ body: expect.objectContaining({ role: 'user' }) }) + ]) + }) + + it('retries a conversion that failed instead of marking it done', async () => { + await seedTestAgentSessionRecordStore(root, { records: [record(RELEASED, true)] }) + await seedJournal(RELEASED, { queued: true }) + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const database = openTestJournalHostDatabase(root) + database.db.exec(` + CREATE TEMP TRIGGER reject_conversion BEFORE INSERT ON journal_rows + WHEN json_extract(NEW.row_json, '$.kind') = 'dispatch' + AND json_extract(NEW.row_json, '$.state') = 'rejected' + BEGIN SELECT RAISE(ABORT, 'temporary storage failure'); END; + `) + const current = openHost() + + await current.reconcileRestartLeases() + await current.startupSettled() + const { reconciliation, sessions } = current.collaboratorsForTests() + expect(reconciliation.owes(RELEASED)).toBe(true) + const journal = sessions.get(RELEASED)!.journal + expect(journal.queuedMessages.list()).toEqual([]) + + database.db.exec('DROP TRIGGER reject_conversion') + await idle(current, [RELEASED]) + + expect(journal.submission(`${RELEASED}-queued`)).toMatchObject({ dispatchState: 'rejected' }) + expect(journal.queuedMessages.list()).toHaveLength(1) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts index 2dde6314cbf9..6dfe2f58fa30 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -1,177 +1,26 @@ -// What the host owes its chats when it comes up as a new instance: the earlier process's leftovers. -// Run after the whole-host restart reconcile, per chat under its serialize, queued before anything -// else can reach that chat, so its share runs first. +// What the host owes its chats when it comes up as a new instance: what the earlier process left. +// Run after the whole-host restart reconcile, in the background, for EVERY chat on record: no +// stored "settled" mark says which are owed, so each is derived from its journal. // -// For each chat it opens, in the order an open used to: the cards' repair and prune; sends the -// earlier process accepted and never handed over, kept as cards or rejected (`holdUnsentSends`); -// the queue's reopen mark; the leftover settlement (`structured-agent-session-leftover-settlement.ts`) -// when the lease says it is owed; and a rewind it left in doubt. Each step is best effort: a failure -// is logged, and the next trigger re-derives it. - -import { setImmediate as yieldToEventLoop } from 'node:timers/promises' -import { agentSessionLeaseIsReleased } from '../../../shared/agent-session-lease-adjudication' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { PrioritySemaphore } from '../../../shared/priority-semaphore' -import { listSessionsWithQueuedMessages } from '../agent-session-journal/queued-message-table' -import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' -import type { - StructuredAgentSessionHostDeps, - StructuredAgentSessionHostSession -} from './structured-agent-session-host-types' -import { settleStructuredAgentSessionLeftoversOrLog } from './structured-agent-session-leftover-settlement' -import { - markStructuredQueueReopen, - structuredAgentSessionHostInstance -} from './structured-agent-session-queued-pause' -import { recoverStructuredRewind } from './structured-rewind-recovery' -import type { OpenedStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' -import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' - -const STARTUP_SETTLEMENT_CONCURRENCY = 4 - -export type StructuredAgentSessionStartupSettlementContext = { - deps: Pick - sessions: Map - serialize: (sessionId: string, task: () => Promise) => Promise - /** Indexes and publishes a chat this share opened, once it is settled. */ - adopt: (sessionId: string, opened: OpenedStructuredAgentSessionConversation) => Promise - now: () => number - /** Journals whose cards and sends this startup already settled: the restore's share after the - * pass's settles only what its recovery ended, or a second reopen mark would hold a later send. */ - sharesDone: WeakSet -} - -/** Owed: released, and no settlement since the release that ended its last generation. */ -export function structuredAgentSessionLeftoversOwed(record: AgentSessionRecord | null): boolean { - return ( - record !== null && - agentSessionLeaseIsReleased(record.lease) && - record.lease.leftoverSettledAt === null - ) -} - -function rewindInDoubt(record: AgentSessionRecord): boolean { - return record.rewind?.phase === 'prepared' || record.rewind?.phase === 'provider-succeeded' -} - -/** The chats startup opens: owed ones and ones with a rewind in doubt, from the record store, and - * ones holding cards, from an index of the cards' own table. Every other chat opens only to read. */ -function startupCandidates(context: StructuredAgentSessionStartupSettlementContext): string[] { - const { store, journalDatabase, logger } = context.deps - const ids = new Set( - store - .listRecords() - .filter((record) => structuredAgentSessionLeftoversOwed(record) || rewindInDoubt(record)) - .map((record) => record.sessionId) - ) - try { - for (const sessionId of listSessionsWithQueuedMessages(journalDatabase.db)) { - if (store.getRecord(sessionId)) { - ids.add(sessionId) - } - } - } catch (error) { - logger.warn('listing chats with saved cards at startup failed', { - scope: 'startup-settlement', - error - }) - } - return [...ids] -} - -/** Every candidate's share is queued on its chat's serialize at once, so whatever that chat runs - * next (a reader's open and the queue it wakes, a send, an attach) runs after its share; a few - * chats at a time do the work. */ -export async function settleStructuredAgentSessionsAtStartup( - context: StructuredAgentSessionStartupSettlementContext -): Promise { - const slots = new PrioritySemaphore(STARTUP_SETTLEMENT_CONCURRENCY) - await Promise.all( - startupCandidates(context).map((sessionId) => - context - .serialize(sessionId, async () => { - const release = await slots.acquire(0) - try { - // A journal open is synchronous SQLite: a macrotask per chat keeps startup responsive. - await yieldToEventLoop() - await settleOneAtStartup(context, sessionId) - } finally { - release() - } - }) - .catch((error: unknown) => - context.deps.logger.warn('settling a chat at startup failed', { - scope: 'startup-settlement', - sessionId, - error - }) - ) - ) - ) -} - -/** One chat's share, for a caller inside its serialize: the startup pass, or the restore once it - * resolved the chat's recovery (which can end a generation after the pass ran). A chat not yet - * open is settled before it is published, so no reader sees what the share ends; `opened` is - * one the restore opened and publishes itself. A share already done finds nothing left to do. */ -export async function settleOneAtStartup( - context: StructuredAgentSessionStartupSettlementContext, - sessionId: string, - opened?: OpenedStructuredAgentSessionConversation -): Promise { - const indexed = opened?.session ?? context.sessions.get(sessionId) - // No journal: nothing was ever written, so nothing is left to settle. - const fresh = indexed ? null : await restoreStructuredAgentSessionRead(context.deps, sessionId) - const session = indexed ?? fresh?.session - try { - // A child here was acquired since: its acquisition settled first, and its loop owns the queue. - if (session && !session.child) { - await settleShare(context, sessionId, session) - } - } finally { - if (fresh) { - await context.adopt(sessionId, fresh) - } - } -} - -async function settleShare( - context: StructuredAgentSessionStartupSettlementContext, - sessionId: string, - { journal }: StructuredAgentSessionHostSession +// Each chat goes to its reconciliation worker with its startup share owed +// (`structured-agent-session-reconciliation-pass.ts`): the worker takes a background slot outside +// the chat's action lane, opens and publishes the journal, and enters the lane only for the short +// recheck and its writes. So a send, a start or a read of a chat late in the scan never waits +// behind its share. A share that fails is not done: the worker retries it. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionReconciliation } from './structured-agent-session-reconciliation-worker' + +/** Resolves once every chat's first attempt finished, whatever it found. */ +export function scanStructuredAgentSessionsAtStartup( + store: Pick, + reconciliation: Pick ): Promise { - const { store, logger } = context.deps - const warn = (scope: string) => (error: unknown) => - logger.warn('startup bookkeeping for a chat failed', { scope, sessionId, error }) - const fence = store.getRecord(sessionId)?.lease.runtimeFence - const firstShare = !context.sharesDone.has(journal) - context.sharesDone.add(journal) - if (firstShare && fence !== undefined) { - await journal.queuedMessages.repairAndPrune().catch(warn('startup-queued-repair')) - // Before a Stop can withdraw one: a Stop never withdraws a card. - await holdUnsentSends(journal, { - fence, - hostInstance: structuredAgentSessionHostInstance(), - hold: { cause: 'hostRestarted' } - }).catch(warn('startup-leftover-sends')) - // After the leftovers became cards, so the mark follows every card the earlier process left. - await markStructuredQueueReopen(sessionId, journal, fence, logger) - } - if (structuredAgentSessionLeftoversOwed(store.getRecord(sessionId))) { - await settleStructuredAgentSessionLeftoversOrLog({ - store, - sessionId, - journal, - now: context.now, - logger, - scope: 'startup-settlement' - }) - } - if (firstShare && fence !== undefined) { - // A rewind the earlier process left prepared refuses every send until settled; one only its - // provider can prove stays for the next acquisition. - await recoverStructuredRewind(context.deps, sessionId, journal, fence).catch( - warn('rewind-recovery') - ) + const sessionIds = store.listRecords().map((record) => record.sessionId) + for (const sessionId of sessionIds) { + reconciliation.signal(sessionId, { startup: true }) } + return Promise.all(sessionIds.map((sessionId) => reconciliation.attempted(sessionId))).then( + () => undefined + ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts index 5aeb9f3755d2..6a92e7e3ba57 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-feed.ts @@ -51,6 +51,7 @@ type StatusFeedSession = { child?: Pick | null /** What the current-work projection reads of the last child this host saw end. */ lastEndedChild?: Pick + operationalRevision?: number restartResume?: AgentSessionStatusSummary['restartResume'] } @@ -215,9 +216,7 @@ export class StructuredAgentSessionStatusFeed { ): StructuredAgentSessionJournalProjection | null { const session = this.deps.sessions.get(sessionId) const source = journal ?? session?.journal - return source - ? this.projections.read(source, this.deps.getRecord(sessionId), session?.lastEndedChild) - : null + return source ? this.projections.read(source, this.deps.getRecord(sessionId), session) : null } /** Re-projects one session after its journal changed; equal projections are not re-sent. */ @@ -228,7 +227,7 @@ export class StructuredAgentSessionStatusFeed { } const source = journal ?? session.journal const record = this.deps.getRecord(sessionId) - const projection = this.projections.read(source, record, session.lastEndedChild) + const projection = this.projections.read(source, record, session) this.retireSettledChildrenOnNewTurn(sessionId, session, projection.acceptedSendKey) const summary = structuredAgentSessionStatusSummary({ sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index 1053b4c993c6..81f104bb0d6a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -25,6 +25,9 @@ export type StructuredAgentSessionJournalProjection = { /** The live generation's fence (`structuredAgentSessionLiveFence`): an exit this host saw, or a * release, changes what is current without a journal row. */ liveFence: number | null + /** `StructuredAgentSessionCurrentWork.revision`: what this host saw end with no row or lease + * write still changes the row. */ + workRevision: number /** The Stop marks' settle revision: a settle edge writes no row, so it is a key of its own. */ stopRevision: number state: StructuredAgentSessionStatusState @@ -45,12 +48,21 @@ export class StructuredAgentSessionJournalProjections { read( journal: AgentSessionJournal, record: AgentSessionRecord | null, - ended?: StructuredAgentSessionWorkEvidence['ended'] + /** What this host saw end (`StructuredAgentSessionHostSession`). */ + seen?: { + lastEndedChild?: StructuredAgentSessionWorkEvidence['ended'] + operationalRevision?: number + } ): StructuredAgentSessionJournalProjection { const cursor = journal.cursor() // The conversation's fence, which a child's end moves: its unanswered sends stop counting. const fence = record?.lease.runtimeFence - const work = structuredAgentSessionCurrentWork(journal, { record, ...(ended ? { ended } : {}) }) + const ended = seen?.lastEndedChild + const work = structuredAgentSessionCurrentWork(journal, { + record, + ...(ended ? { ended } : {}), + revision: seen?.operationalRevision ?? 0 + }) const stopRevision = journal.stopMarks.revision() let projection = this.byJournal.get(journal) if ( @@ -59,6 +71,7 @@ export class StructuredAgentSessionJournalProjections { projection.sequence !== cursor.sequence || projection.fence !== fence || projection.liveFence !== work.liveFence || + projection.workRevision !== work.revision || projection.stopRevision !== stopRevision ) { // A journalled submission bumps `lastSequence`, so the send-time working @@ -92,6 +105,7 @@ export class StructuredAgentSessionJournalProjections { submissionCount: snapshot.submissions.length, fence, liveFence: work.liveFence, + workRevision: work.revision, stopRevision, // The host's projection of current work: an ended generation's is not Working. state: projectStructuredAgentSessionStatusState( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index 683d0667c7a8..f41865aab51a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -44,9 +44,9 @@ export type Subscriber = { queuePublication?: QueuePublication /** Fingerprint of the background-task roster last SENT. */ backgroundTasks?: string - /** The live generation the last frame's current-work fields were read under; a change with no - * row still owes this subscriber a frame. */ - workLiveFence?: number | null + /** The current-work view the last frame's fields were read under + * (`StructuredAgentSessionCurrentWork.viewKey`); a change with no row still owes a frame. */ + workView?: string } export type AgentSessionSubscribersHooks = { @@ -115,7 +115,7 @@ export class AgentSessionSubscribers { } else { const work = this.hooks.readCurrentWork?.(input.sessionId, input.journal) const page = readAgentSessionHydrationPage(input.journal, input.fence, work) - subscriber.workLiveFence = work?.liveFence + subscriber.workView = work?.viewKey() this.emit(subscriber, { type: 'snapshot', sessionId: input.sessionId, @@ -180,7 +180,7 @@ export class AgentSessionSubscribers { const page = readAgentSessionHydrationPage(journal, fence, work) const hostNow = this.now() for (const subscriber of this.subscribers(sessionId)) { - subscriber.workLiveFence = work?.liveFence + subscriber.workView = work?.viewKey() this.emit(subscriber, { type: 'snapshot', sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index fc3a9ed9bd9d..1ceb551108a8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -16,7 +16,6 @@ import { import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import { exitJournalDouble, - NO_STORE_RECEIPTS, plannedBatchThrough } from './structured-agent-session-settlement-double.test-fixture' import type { JournalLifecycleBatchInput } from '../agent-session-journal/journal-store-contracts' @@ -73,8 +72,7 @@ function mutableStore() { transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)), - conversationReceipts: NO_STORE_RECEIPTS + ) => (record = transition(record)) } } } @@ -90,8 +88,7 @@ describe('provider-exit settlement', () => { transitionHandoff: async ( _sessionId: string, transition: (current: AgentSessionRecord) => AgentSessionRecord - ) => (record = transition(record)), - conversationReceipts: NO_STORE_RECEIPTS + ) => (record = transition(record)) } const appendLifecycleBatch = vi .fn() @@ -196,8 +193,7 @@ describe('provider-exit settlement', () => { reservedSpawnToken: null } }), - transitionHandoff: async () => ({ lease: { runtimeFence: 8 } }), - conversationReceipts: NO_STORE_RECEIPTS + transitionHandoff: async () => ({ lease: { runtimeFence: 8 } }) } await settleStructuredAgentSessionChildExit( @@ -435,7 +431,11 @@ describe('provider-exit settlement', () => { expect(session.child).toBeNull() expect(generationEnded).toHaveBeenCalledTimes(1) - expect(generationEnded).toHaveBeenCalledWith(SESSION, { restate: true }) + // The worker is handed the exit's own account, to write once the journal takes it. + expect(generationEnded).toHaveBeenCalledWith(SESSION, { + restate: true, + exit: expect.objectContaining({ ownerFence: 7, showUnexpectedExitOutcome: true }) + }) expect(log.scopes()).toEqual(['exit-lifecycle-barrier', 'exit-settlement']) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 05b46d867d1e..eaa82e6d7175 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -150,6 +150,21 @@ function openHost(overrides: Partial = {}): void }) } +/** Startup as the runtime runs it: the restart reconcile, every chat's share in the background, + * and the boot sweep that opens each chat for its readers once that share closed it again. */ +async function startUp(): Promise { + await host.reconcileRestartLeases() + await host.startupSettled() + await reconciled() + await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) + await host.restoreReadableSessions() +} + +/** The chat's reconciliation worker has retired, or waits out a backoff. */ +function reconciled(): Promise { + return host.collaboratorsForTests().reconciliation.idle(SESSION) +} + /** The host starting the agent with no message to deliver, as an operation that needs it does. */ function startAgent(): Promise { return host['serialize'](SESSION, () => host['mutationContext']().ensureAgent(SESSION)) @@ -339,7 +354,7 @@ describe('already-wedged profiles become usable on load', () => { : {}) }) - await host.restoreReadableSessions() + await startUp() expect(acquire).not.toHaveBeenCalled() expect(turnLifecycle('turn-1')).toEqual({ @@ -365,8 +380,12 @@ describe('already-wedged profiles become usable on load', () => { expect(published.map((summary) => summary.status)).not.toContain('working') // A crash is not something the user did: a proven one reads as an interruption and an // unprovable one as unconfirmed, so no reader files it as a cancellation the user knows about. - expect(published.map((summary) => summary.turnOutcome)).toEqual( - published.map(() => (verdict.state === 'interrupted' ? 'interruption' : 'unconfirmed')) + // The chat is published before its settlement lands, as not working with no outcome yet. + const outcome = verdict.state === 'interrupted' ? 'interruption' : 'unconfirmed' + const outcomes = published.map((summary) => summary.turnOutcome) + expect(outcomes.at(-1)).toBe(outcome) + expect(new Set(outcomes.filter((reported) => reported !== undefined))).toEqual( + new Set([outcome]) ) } ) @@ -384,6 +403,8 @@ describe('already-wedged profiles become usable on load', () => { const fence = stale.ok ? 13 : (stale.refusal.currentFence ?? 13) expect(stale.ok || stale.refusal.code === 'agent_session_checkpoint_stale').toBe(true) expect(stale.ok || (await host.attach(CALLER, hostTestAttachParams(fence))).ok).toBe(true) + // The reservation over the proven-dead owner ended its generation: the worker settles it. + await reconciled() expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) @@ -422,10 +443,12 @@ describe('already-wedged profiles become usable on load', () => { openHost() expect(await host.attach(CALLER, hostTestAttachParams(13))).toMatchObject({ ok: true }) + // The reservation cleared the release's proof and handed it to the chat's worker. + await reconciled() expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) - // The exit was observed, so its receipt is the turn's end. + // The exit was observed, so its proof is the turn's end. expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', state: 'interrupted', @@ -446,7 +469,7 @@ describe('already-wedged profiles become usable on load', () => { ['a restart eviction', false], ['a proven eviction by recovery', true] ] as const)( - 'settles the turn %s left at the next acquire when the read restore could not write it', + 'settles the turn %s left once storage takes it, when its first settlement could not write it', async (_origin, ownerOutlivedRestart) => { await seedStore( wedgedRecord({ claimStatus: 'live', handoffStage: null, ownerProcess: DEAD_OWNER }) @@ -463,7 +486,7 @@ describe('already-wedged profiles become usable on load', () => { : { outcome: 'pid-absent' }, stopOwnerProcess }) - // The startup settlement fails, and nothing retries it before the next acquisition. + // The worker's settlement fails until storage takes it again; it backs off and retries. const failing = vi .spyOn(JournalLifecycleBatchAppender.prototype, 'appendPlanned') .mockRejectedValue(new Error('journal unavailable')) @@ -477,11 +500,15 @@ describe('already-wedged profiles become usable on load', () => { deathEvidence: { kind: 'pid-absent' } }) + // The next start waits on none of it. await startAgent() expect(acquire).toHaveBeenCalledOnce() expect(store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'live' }) // A pid probe proved the owner gone, so the turn was cut short when it was last proven alive. + await vi.waitFor(() => expect(turnLifecycle('turn-1')?.state).toBe('interrupted'), { + timeout: 5_000 + }) expect(turnLifecycle('turn-1')).toEqual({ turnId: 'turn-1', state: 'interrupted', @@ -553,12 +580,13 @@ describe('already-wedged profiles become usable on load', () => { } ) - it('marks a running turn left behind by a released lease unverifiable on a cold acquire', async () => { + it('marks a running turn left behind by a released lease unverifiable at startup, and a cold acquire starts', async () => { // No settlement latch: the record was released cleanly, but the journal still says a turn is // running. The child that wrote it is gone and nothing observed its exit. await seedStore(wedgedRecord({ claimStatus: 'released', handoffStage: null })) await seedRunningTurn() openHost() + await startUp() expect(await host.attach(CALLER, hostTestAttachParams(13))).toMatchObject({ ok: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 7fe5b1fce056..0881fe6ba3b2 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -88,7 +88,9 @@ export function conversationCommandBlocked( ) } // The same liveness fold the strip's monitoring indicator reads: settled rows block nothing. - if (agentChildWorkLiveness(childWork) !== null) { + // Child records are the live generation's: its adapter ends them on every close, so with no + // generation live, any still running are an ended one's leftovers and hold nothing. + if (work.liveFence !== null && agentChildWorkLiveness(childWork) !== null) { return blocked( 'backgroundTasksRunning', stripOffersStop(childWork ?? [], ctx.adapter.backgroundTaskStops?.(ctx.sessionId)) diff --git a/src/main/runtime/agent-session-conversation-receipts.ts b/src/main/runtime/agent-session-conversation-receipts.ts index c6c5713fbe08..6935f8bb0836 100644 --- a/src/main/runtime/agent-session-conversation-receipts.ts +++ b/src/main/runtime/agent-session-conversation-receipts.ts @@ -5,7 +5,6 @@ import { } from './agent-session-conversation-command-record' import { settleAgentSessionOperationInto } from './agent-session-operation-admission' import { commitAgentSessionRewindCompletion } from './agent-session-rewind-completion' -import { markAgentSessionLeftoverSettled } from './agent-session-lease-transitions' export function createAgentSessionConversationReceipts( transactions: Pick @@ -31,14 +30,6 @@ export function createAgentSessionConversationReceipts( ...args: Parameters extends [unknown, ...infer A] ? A : never - ) => transactions.receipt((draft) => commitAgentSessionRewindCompletion(draft, ...args)), - /** Committed with the settlement of what ended generations left: see `leftoverSettledAt`. */ - leftoverSettled: (sessionId: string, plannedFence: number, now: number) => - transactions.receipt((draft) => { - const record = draft.records.get(sessionId) - if (record) { - draft.records.set(sessionId, markAgentSessionLeftoverSettled(record, plannedFence, now)) - } - }) + ) => transactions.receipt((draft) => commitAgentSessionRewindCompletion(draft, ...args)) } } diff --git a/src/main/runtime/agent-session-death-evidence-persistence.test.ts b/src/main/runtime/agent-session-death-evidence-persistence.test.ts index 73d01b5b88e8..a025d5d1621c 100644 --- a/src/main/runtime/agent-session-death-evidence-persistence.test.ts +++ b/src/main/runtime/agent-session-death-evidence-persistence.test.ts @@ -82,14 +82,14 @@ describe('death evidence on disk', () => { }) }) -describe('who is told a proof of death landed', () => { - it('tells a listener once the proof is committed, and never for a write that proves nothing', async () => { +describe('who is told a generation ended', () => { + it('tells a listener once the end is committed, with its proof, and never for a write that ends nothing', async () => { await seed(null) const store = await open() const told: unknown[] = [] - store.onDeathEvidence((sessionId) => told.push(store.getRecord(sessionId)?.lease.deathEvidence)) + store.onGenerationEnded((ended) => told.push(ended)) const unsubscribed = vi.fn() - store.onDeathEvidence(unsubscribed)() + store.onGenerationEnded(unsubscribed)() await expect( store.evictProvenDeadOwner({ @@ -103,7 +103,13 @@ describe('who is told a proof of death landed', () => { // The proof already on the record is not news to a later write. await store.setSessionTabVisibility(SESSION, true) - expect(told).toEqual([expect.objectContaining({ kind: 'pid-absent', ownerFence: 7 })]) + expect(told).toEqual([ + { + sessionId: SESSION, + endedFence: 7, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: 7 }) + } + ]) expect(unsubscribed).not.toHaveBeenCalled() }) }) diff --git a/src/main/runtime/agent-session-generation-end.test.ts b/src/main/runtime/agent-session-generation-end.test.ts new file mode 100644 index 000000000000..047f333fdaa0 --- /dev/null +++ b/src/main/runtime/agent-session-generation-end.test.ts @@ -0,0 +1,271 @@ +// Every lease write that revokes or supersedes a generation is told once, after commit, from the +// store's one transaction path, with the proof the end carried; a write that ends nothing is not. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { claudeProviderHandle } from '../../shared/agent-session-provider-handle-encoding' +import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' +import { closeTestJournalHostDatabases } from '../native-chat/agent-session-journal/journal-host-database-test-support' +import { releaseStoredStructuredAgentSessionOwnerAfterExit } from '../native-chat/agent-session-wire/structured-agent-session-lease-release' +import type { AgentSessionFailedAcquisitionSettlement } from './agent-session-acquisition-failure-settlement' +import type { AgentSessionGenerationEnd } from './agent-session-generation-end' +import { releaseUnprovenAgentSessionOwner } from './agent-session-lease-transitions' +import type { AgentSessionRecordStore } from './agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from './agent-session-record-store-test-harness' + +const SESSION = 'session-alpha-1' +const NOW = 1_800_000_000_000 +let directory: string +let store: AgentSessionRecordStore +let ended: AgentSessionGenerationEnd[] +let operations = 0 + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'orca-generation-end-')) + await seedTestAgentSessionRecordStore(directory, { records: [] }) + store = await openTestAgentSessionRecordStore(directory) + ended = [] + store.onGenerationEnded((end) => ended.push(end)) +}) + +afterEach(async () => { + closeTestJournalHostDatabases() + await rm(directory, { recursive: true, force: true }) +}) + +function operationId(): string { + operations += 1 + return `${NOW}-${operations.toString(16).padStart(32, '0')}` +} + +/** Reserves at the next fence, as an attach does, over what the probe says of the recorded owner. */ +async function reserve( + probe: Parameters[0]['probe'] +): Promise<{ fence: number; operation: string }> { + const operation = operationId() + const record = store.getRecord(SESSION) + const reserved = await store.reserveOwner({ + sessionId: SESSION, + location: agentSessionRecordFixture().location, + provider: 'claude', + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + expectedFence: record ? record.lease.runtimeFence : null, + spawnToken: `spawn-${operation}`, + claimKeyId: 'key-1', + handoffOperationId: operation, + probe, + operation: { callerKey: 'client-1', operationId: operation, fingerprint: 'fp-1' }, + now: NOW + }) + return { fence: reserved.record.lease.runtimeFence, operation } +} + +/** A reservation whose child spawned and proved its handle: a live owner. */ +async function liveOwner(): Promise<{ fence: number; operation: string }> { + const reserved = await reserve({ outcome: 'indeterminate', reason: 'no owner yet' }) + await store.commitProcessIdentity({ + sessionId: SESSION, + fence: reserved.fence, + process: { + hostId: 'local', + pid: 4_000 + reserved.fence, + processStartTimeMs: NOW, + spawnToken: `spawn-${reserved.operation}` + }, + now: NOW + }) + await store.proveOwner({ + sessionId: SESSION, + fence: reserved.fence, + link: { + linkId: `link-${reserved.fence}`, + handle: claudeProviderHandle('provider-session-1', null), + // The first start creates the conversation; each later one resumes it. + origin: store.getRecord(SESSION)?.providerHandleChain.length ? 'resumed' : 'created', + mintedAtFence: reserved.fence, + observedAt: NOW + }, + now: NOW + }) + return reserved +} + +function failedAcquisition( + reserved: { fence: number; operation: string }, + exitProof: AgentSessionFailedAcquisitionSettlement['exitProof'] +): AgentSessionFailedAcquisitionSettlement { + return { + sessionId: SESSION, + fence: reserved.fence, + spawnToken: `spawn-${reserved.operation}`, + callerKey: 'client-1', + operationId: reserved.operation, + outcome: { status: 'failed', code: 'agent_session_operation_invalid', message: 'failed' }, + exitProof, + now: NOW + 1_000 + } +} + +describe('a generation that ends', () => { + it('is told once for an observed exit’s release, with the exit’s proof', async () => { + const { fence } = await liveOwner() + ended = [] + + await releaseStoredStructuredAgentSessionOwnerAfterExit({ + store, + sessionId: SESSION, + expectedFence: fence, + now: NOW + 1_000, + exitObservedAt: NOW + 900 + }) + + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: fence, + evidence: expect.objectContaining({ kind: 'exit-observed', ownerFence: fence }) + } + ]) + }) + + it('is told once for each failed acquisition’s release', async () => { + const first = await reserve({ outcome: 'indeterminate', reason: 'no owner yet' }) + ended = [] + await store.settleFailedAcquisition(failedAcquisition(first, 'processless')) + expect(ended).toEqual([ + { sessionId: SESSION, endedFence: first.fence, evidence: expect.anything() } + ]) + + const second = await liveOwner() + ended = [] + await store.settleFailedPostAcquisitionAttachment( + failedAcquisition(second, 'root-exit-observed') + ) + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: second.fence, + evidence: expect.objectContaining({ ownerFence: second.fence }) + } + ]) + }) + + it('is told once for an eviction, and once for a release nothing proved', async () => { + const first = await liveOwner() + await store.settleFailedPostAcquisitionAttachment(failedAcquisition(first, 'unproven')) + // Parked in recovery: nothing has ended yet. + expect(ended).toEqual([]) + await store.evictProvenDeadOwner({ + sessionId: SESSION, + expectedFence: first.fence, + probe: { outcome: 'pid-absent' }, + now: NOW + 2_000 + }) + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: first.fence, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: first.fence }) + } + ]) + + const second = await liveOwner() + await store.settleFailedPostAcquisitionAttachment(failedAcquisition(second, 'unproven')) + ended = [] + await store.transitionHandoff(SESSION, (record) => + releaseUnprovenAgentSessionOwner({ record, expectedFence: second.fence, now: NOW + 3_000 }) + ) + expect(ended).toEqual([{ sessionId: SESSION, endedFence: second.fence, evidence: null }]) + }) + + it('is told once for a reservation granted over an owner the probe proved gone, with the probe’s proof the reservation clears', async () => { + const { fence } = await liveOwner() + ended = [] + + const replaced = await reserve({ outcome: 'pid-absent' }) + + expect(replaced.fence).toBe(fence + 1) + expect(store.getRecord(SESSION)?.lease.deathEvidence).toBeNull() + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: fence, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: fence }) + } + ]) + }) + + it('is told once for a restart adjudication that releases it', async () => { + const { fence } = await liveOwner() + closeTestJournalHostDatabases() + store = await openTestAgentSessionRecordStore(directory) + ended = [] + store.onGenerationEnded((end) => ended.push(end)) + + await store.reconcileOnRestart({ probe: async () => ({ outcome: 'pid-absent' }), now: NOW }) + + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: fence, + evidence: expect.objectContaining({ kind: 'pid-absent', ownerFence: fence }) + } + ]) + }) +}) + +describe('a write that ends nothing', () => { + it('tells no one: a reservation over a released lease with no proof, a spawn, a proof, a renewal', async () => { + await store.reconcileOnRestart({ probe: async () => ({ outcome: 'pid-absent' }), now: NOW }) + const first = await reserve({ outcome: 'indeterminate', reason: 'no owner yet' }) + await store.settleFailedAcquisition(failedAcquisition(first, 'processless')) + const released = store.getRecord(SESSION)!.lease + // Strip the release's proof, as one an earlier build wrote without any. + await store.transitionHandoff(SESSION, (record) => ({ + ...record, + lease: { ...record.lease, deathEvidence: null } + })) + expect(released.claimStatus).toBe('released') + ended = [] + + const { fence } = await liveOwner() + await store.renewLease({ + sessionId: SESSION, + fence, + childProbe: { outcome: 'identity-matched', matchedOn: ['held-child'] }, + now: NOW + 5_000 + }) + await store.setSessionTabVisibility(SESSION, true) + + expect(ended).toEqual([]) + }) + + it('hands on, once, the proof a reservation clears from a released lease, for the generation it judges', async () => { + const first = await liveOwner() + await releaseStoredStructuredAgentSessionOwnerAfterExit({ + store, + sessionId: SESSION, + expectedFence: first.fence, + now: NOW + 1_000, + exitObservedAt: NOW + 900 + }) + ended = [] + + const { fence } = await liveOwner() + + expect(fence).toBe(first.fence + 2) + expect(store.getRecord(SESSION)?.lease.deathEvidence).toBeNull() + expect(ended).toEqual([ + { + sessionId: SESSION, + endedFence: first.fence, + evidence: expect.objectContaining({ kind: 'exit-observed', ownerFence: first.fence }) + } + ]) + }) +}) diff --git a/src/main/runtime/agent-session-generation-end.ts b/src/main/runtime/agent-session-generation-end.ts new file mode 100644 index 000000000000..f5f4355d1f83 --- /dev/null +++ b/src/main/runtime/agent-session-generation-end.ts @@ -0,0 +1,93 @@ +// Which agent generations a committed store transaction ended: every lease write that revokes or +// supersedes a generation that could still act. A release of any kind (exit, surface, failed +// acquisition, eviction, unproven, restart adjudication) moves the fence and leaves no owner; a +// reservation granted over an owner a probe proved gone supersedes it directly. Reserving a lease +// already released ends nothing new, but when the write drops the proof the release kept, that +// proof is handed on once more for the generation it judges, so nothing the store forgets is lost +// to the settlement still owed for it. + +import { + agentSessionDeathEvidenceFromProbe, + type AgentSessionOwnerProbe +} from '../../shared/agent-session-lease-adjudication' +import type { + AgentSessionDeathEvidence, + AgentSessionLease, + AgentSessionRecord +} from '../../shared/agent-session-record' + +export type AgentSessionGenerationEnd = { + sessionId: string + /** The generation that can no longer act. */ + endedFence: number + /** The proof of how it ended: the release's own, or, for a reservation that replaced it, the + * probe's, which the reservation clears from the lease. Null when nothing proved it gone. */ + evidence: AgentSessionDeathEvidence | null +} + +/** A transaction's own proof for a generation its reservation superseded. */ +export type AgentSessionSupersessionEvidence = ( + ended: AgentSessionLease +) => AgentSessionDeathEvidence | null + +/** What a store transaction may say of the generations it ends. */ +export type AgentSessionGenerationEndOptions = { + supersessionEvidence?: AgentSessionSupersessionEvidence +} + +/** A reservation granted over an owner the probe proved gone ends it with that probe's proof, which + * the reservation clears from the lease. */ +export function reservationSupersessionEvidence( + probe: AgentSessionOwnerProbe, + now: number +): AgentSessionSupersessionEvidence { + return (ended) => agentSessionDeathEvidenceFromProbe(probe, now, ended) +} + +export function agentSessionLeasesBefore( + records: ReadonlyMap +): Map { + return new Map([...records].map(([sessionId, record]) => [sessionId, record.lease])) +} + +export function agentSessionGenerationEnds( + before: ReadonlyMap, + after: ReadonlyMap, + supersessionEvidence?: AgentSessionSupersessionEvidence +): AgentSessionGenerationEnd[] { + const ends: AgentSessionGenerationEnd[] = [] + for (const [sessionId, { lease }] of after) { + const prior = before.get(sessionId) + if (!prior || prior === lease) { + continue + } + if (prior.claimStatus === 'released') { + const proof = prior.deathEvidence + if (proof && !sameProof(proof, lease.deathEvidence)) { + // A proof from before fences were recorded on it judges everything the lease fenced. + const ownerFence = proof.ownerFence ?? prior.runtimeFence + ends.push({ sessionId, endedFence: ownerFence, evidence: { ...proof, ownerFence } }) + } + continue + } + const released = lease.claimStatus === 'released' + if (!released && lease.runtimeFence === prior.runtimeFence) { + continue + } + ends.push({ + sessionId, + endedFence: prior.runtimeFence, + evidence: released ? lease.deathEvidence : (supersessionEvidence?.(prior) ?? null) + }) + } + return ends +} + +function sameProof(proof: AgentSessionDeathEvidence, other: AgentSessionDeathEvidence | null) { + return ( + other !== null && + other.kind === proof.kind && + other.observedAt === proof.observedAt && + other.ownerFence === proof.ownerFence + ) +} diff --git a/src/main/runtime/agent-session-lease-transitions.ts b/src/main/runtime/agent-session-lease-transitions.ts index 0581dff6904f..950e05a8e78e 100644 --- a/src/main/runtime/agent-session-lease-transitions.ts +++ b/src/main/runtime/agent-session-lease-transitions.ts @@ -33,31 +33,11 @@ export type AgentSessionReservation = { now: number } -/** Every lease write. One that ends a generation (moves the fence and leaves no owner) marks what - * that generation left as owed; only the settlement's own receipt marks it settled. */ export function withLease( record: AgentSessionRecord, lease: AgentSessionLease ): AgentSessionRecord { - const endsGeneration = - lease.claimStatus === 'released' && lease.runtimeFence !== record.lease.runtimeFence - return { - ...record, - lease: endsGeneration ? { ...lease, leftoverSettledAt: null } : lease, - updatedAt: lease.lastRenewedAt - } -} - -/** The settlement's receipt: marks the leftovers settled only while the lease still has the fence - * the settlement was planned at, so a generation that ended meanwhile stays owed. */ -export function markAgentSessionLeftoverSettled( - record: AgentSessionRecord, - plannedFence: number, - now: number -): AgentSessionRecord { - return record.lease.runtimeFence === plannedFence && record.lease.claimStatus === 'released' - ? { ...record, lease: { ...record.lease, leftoverSettledAt: now } } - : record + return { ...record, lease, updatedAt: lease.lastRenewedAt } } export function assertFence(lease: AgentSessionLease, fence: number): void { diff --git a/src/main/runtime/agent-session-record-founding.ts b/src/main/runtime/agent-session-record-founding.ts index 57d5300096b4..41386401351e 100644 --- a/src/main/runtime/agent-session-record-founding.ts +++ b/src/main/runtime/agent-session-record-founding.ts @@ -64,9 +64,7 @@ export function foundAgentSessionRecord( claimKeyId: lease.claimKeyId, claimStatus: 'released', unreconciled: false, - deathEvidence: null, - // No generation ran yet, so none left anything. - leftoverSettledAt: lease.now + deathEvidence: null } } } diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 1b4dd7bd9d60..aa0fccea0a7a 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -28,6 +28,13 @@ import { retireAgentSessionClaimKey } from './agent-session-claim-key-retention' import type { AgentSessionOwnerProbe } from '../../shared/agent-session-lease-adjudication' +import { + agentSessionGenerationEnds, + agentSessionLeasesBefore, + reservationSupersessionEvidence, + type AgentSessionGenerationEnd, + type AgentSessionGenerationEndOptions +} from './agent-session-generation-end' import type { AgentSessionProviderHandleLink } from '../../shared/agent-session-provider-handle' import { agentSessionScopeKey, @@ -85,7 +92,7 @@ export const AGENT_SESSION_LEASE_TTL_MS = 30_000, AGENT_SESSION_LEASE_RENEW_INTERVAL_MS = 10_000 export class AgentSessionRecordStore { - private readonly deathEvidenceListeners = new Set<(sessionId: string) => void>() + private readonly generationEndListeners = new Set<(ended: AgentSessionGenerationEnd) => void>() private readonly firstRecordListeners = new Set<() => void>() readonly conversationReceipts: ReturnType @@ -194,8 +201,9 @@ export class AgentSessionRecordStore { isAgentSessionClaimKeyVerifiable(this.state, keyId, now) async reserveOwner(request: AgentSessionReserveRequest): Promise { - return this.transact((draft) => - commitAgentSessionReservation(draft, request, AGENT_SESSION_LEASE_TTL_MS) + return this.transact( + (draft) => commitAgentSessionReservation(draft, request, AGENT_SESSION_LEASE_TTL_MS), + { supersessionEvidence: reservationSupersessionEvidence(request.probe, request.now) } ) } @@ -339,11 +347,12 @@ export class AgentSessionRecordStore { }) } - /** Told, once committed, of each session a transaction wrote a proof of death for — whichever - * transition wrote it, since every one lands here. Must not throw. */ - onDeathEvidence(listener: (sessionId: string) => void): () => void { - this.deathEvidenceListeners.add(listener) - return () => this.deathEvidenceListeners.delete(listener) + /** Told once, after commit, of each generation a transaction revoked or superseded, whichever + * transition wrote it, since every lease write lands here (`agentSessionGenerationEnds`). Must not + * throw. */ + onGenerationEnded(listener: (ended: AgentSessionGenerationEnd) => void): () => void { + this.generationEndListeners.add(listener) + return () => this.generationEndListeners.delete(listener) } /** Told, once committed, when the store records its first chat. Must not throw. */ @@ -356,26 +365,19 @@ export class AgentSessionRecordStore { * once its rows have committed. */ private transact = async ( apply: (draft: AgentSessionStoreState) => T, - options?: { inMemoryWhenReadOnly?: boolean } + options?: { inMemoryWhenReadOnly?: boolean } & AgentSessionGenerationEndOptions ): Promise => { - let proven: string[] = [] + let ended: AgentSessionGenerationEnd[] = [] let heldBefore = true const result = await this.transactions.transact((draft) => { heldBefore = this.holdsRecords() - if (this.deathEvidenceListeners.size === 0) { - return apply(draft) - } - const before = new Map( - [...draft.records].map(([id, record]) => [id, record.lease.deathEvidence]) - ) + const before = agentSessionLeasesBefore(draft.records) const applied = apply(draft) - proven = [...draft.records] - .filter(([id, { lease }]) => lease.deathEvidence && lease.deathEvidence !== before.get(id)) - .map(([id]) => id) + ended = agentSessionGenerationEnds(before, draft.records, options?.supersessionEvidence) return applied }, options) - for (const sessionId of proven) { - this.deathEvidenceListeners.forEach((listener) => listener(sessionId)) + for (const end of ended) { + this.generationEndListeners.forEach((listener) => listener(end)) } if (!heldBefore && this.holdsRecords()) { this.firstRecordListeners.forEach((listener) => listener()) diff --git a/src/main/runtime/agent-session-reservation-admission.ts b/src/main/runtime/agent-session-reservation-admission.ts index 4ae12274c1b3..ff840c755b35 100644 --- a/src/main/runtime/agent-session-reservation-admission.ts +++ b/src/main/runtime/agent-session-reservation-admission.ts @@ -300,9 +300,7 @@ function createAgentSessionRecord( claimKeyId: reservation.claimKeyId, claimStatus: 'reserved', unreconciled: false, - deathEvidence: null, - // A new record: no earlier generation left anything. - leftoverSettledAt: reservation.now + deathEvidence: null } } } diff --git a/src/main/runtime/structured-agent-session-mobile-restore.test.ts b/src/main/runtime/structured-agent-session-mobile-restore.test.ts index 0e3033f000b2..2891a0361419 100644 --- a/src/main/runtime/structured-agent-session-mobile-restore.test.ts +++ b/src/main/runtime/structured-agent-session-mobile-restore.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { hostTestState, adapter, @@ -80,6 +80,13 @@ describe('prompt delivery after host recovery', () => { notificationSeq: delivered.notificationSeq } unsubscribe() + // The crashed host's last write landed before the new process opens the chat: two live + // processes never share one journal. + await vi.waitFor(async () => + expect((await h.host.journalSnapshot(SESSION)).submissions).toEqual([ + expect.objectContaining({ dispatchState: 'accepted' }) + ]) + ) const restarted = new RuntimeMobileNotificationController() restarted.configureDismissalStore(h.root) const events: MobileNotificationEvent[] = [] @@ -100,6 +107,7 @@ describe('prompt delivery after host recovery', () => { try { await host.reconcileRestartLeases() await host.restoreReadableSessions() + await host.startupSettled() const history = await host.history({ sessionId: SESSION, direction: 'tail' }) expect(history.ok).toBe(true) expect( diff --git a/src/main/runtime/structured-worker-identity.test.ts b/src/main/runtime/structured-worker-identity.test.ts index 7776e04cde85..17ba897de479 100644 --- a/src/main/runtime/structured-worker-identity.test.ts +++ b/src/main/runtime/structured-worker-identity.test.ts @@ -56,8 +56,7 @@ function record(overrides: { claimKeyId: 'k', claimStatus: overrides.claimStatus ?? 'live', unreconciled: false, - deathEvidence: null, - leftoverSettledAt: null + deathEvidence: null }, createdAt: 0, updatedAt: 0 diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts index 2f430e053ada..3fd00f9a1302 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts @@ -1,5 +1,8 @@ import { useMemo } from 'react' -import { runningStructuredAgentSessionTurnId } from '../../../../shared/structured-agent-session-live-turn' +import { + runningStructuredAgentSessionTurnId, + structuredAgentSessionHostSaysWorking +} from '../../../../shared/structured-agent-session-live-turn' import { isStructuredAgentSessionMainAgentWorking } from '../../../../shared/structured-agent-session-main-agent-working' import type { StructuredAgentSessionState } from '../../../../shared/structured-agent-session-reducer' import type { StructuredAgentSubagentRoster } from '../../../../shared/structured-agent-session-subagent-roster' @@ -30,8 +33,12 @@ export function useStructuredAgentSessionTransportState( ) // The host's whole-journal turn, never the loaded rows': a long turn's record is off the page. const turnId = runningStructuredAgentSessionTurnId({ items: journalItems, latestTurn }) - // The rule the host projects every session list's Working from, so this chat cannot disagree. - const isWorking = isStructuredAgentSessionMainAgentWorking(turnId, current.submissions, fence) + // The host's own answer, which counts only the agent running now; an older host gives none, and + // the rule it projects every session list's Working from stands in, so this chat cannot disagree. + const hostWorking = enabled ? state.working : undefined + const isWorking = structuredAgentSessionHostSaysWorking(hostWorking, () => + isStructuredAgentSessionMainAgentWorking(turnId, current.submissions, fence) + ) const nextQueuedMessageId = (enabled ? state.nextQueuedMessageId : null) ?? null // The host names the card its queue sends next. That send lands in a later update than a turn's // end or a Resume, so until then the chat still reads as working and nothing flips in between. diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx new file mode 100644 index 000000000000..b391e808f5ce --- /dev/null +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.working.test.tsx @@ -0,0 +1,147 @@ +// @vitest-environment happy-dom + +// Whether the chat's own agent is working is the host's answer when it gives one: it counts only +// the agent running now, so a turn an ended agent left running reads as stopped and Resume shows. +// An older host gives none, and the chat derives it from its rows as before. + +import { cleanup, renderHook } from '@testing-library/react' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AgentJournalRenderItem } from '../../../../shared/agent-session-journal-types' +import type { AgentSessionHistoryPage } from '../../../../shared/agent-session-wire' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession, + type StructuredAgentSessionState +} from '../../../../shared/structured-agent-session-reducer' + +vi.mock('@/runtime/structured-agent-session-client', () => ({ + callStructuredAgentSession: vi.fn() +})) + +import { useStructuredAgentSessionTransportState } from './use-structured-agent-session-transport-state' +import { useStructuredAgentSessionQueuedMessages } from './use-structured-agent-session-queued-messages' +import { useStructuredAgentSessionMutate } from './use-structured-agent-session-mutate' + +afterEach(cleanup) + +/** A turn row an agent that has since ended left running. */ +const LEFT_RUNNING: AgentJournalRenderItem = { + itemId: 'turn-1', + sequence: 1, + revision: 1, + observedAt: 1, + body: { + kind: 'status', + text: 'Working', + turnLifecycle: { turnId: 'turn-1', state: 'running' } + } +} + +function page(fields: Partial = {}): AgentSessionHistoryPage { + const cursor = { epoch: 'epoch-a', sequence: 1 } + return { + sessionId: 'session-a', + epoch: 'epoch-a', + direction: 'tail', + items: [LEFT_RUNNING], + removedItemIds: [], + submissions: [], + window: { oldest: cursor, newest: cursor, nextCursor: cursor }, + liveCursor: cursor, + hasOlder: false, + hasNewer: false, + ...fields + } +} + +function loaded(fields: Partial): StructuredAgentSessionState { + return reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'history-page', + page: page(fields) + }) +} + +function isWorking(state: StructuredAgentSessionState): boolean { + return renderHook(() => useStructuredAgentSessionTransportState(state, true)).result.current + .isWorking +} + +describe('the chat’s Working', () => { + it('is the host’s answer: a turn an ended agent left running reads as stopped', () => { + expect(isWorking(loaded({ working: false }))).toBe(false) + expect(isWorking(loaded({ items: [], working: true }))).toBe(true) + }) + + it('falls back to the rows when an older host gives no answer', () => { + expect(loaded({}).working).toBeUndefined() + expect(isWorking(loaded({}))).toBe(true) + }) + + it('follows a frame that restates it with no row, and keeps it across a frame that names none', () => { + const stopped = reduceStructuredAgentSession(loaded({ working: true }), { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + items: [], + removedItemIds: [], + submissions: [], + cursor: { epoch: 'epoch-a', sequence: 1 } + }, + working: false + } + }) + expect(stopped.working).toBe(false) + expect(isWorking(stopped)).toBe(false) + const quiet = reduceStructuredAgentSession(stopped, { + type: 'event', + event: { + type: 'batch', + sessionId: 'session-a', + batch: { + items: [], + removedItemIds: [], + submissions: [], + cursor: { epoch: 'epoch-a', sequence: 1 } + } + } + }) + expect(quiet.working).toBe(false) + }) + + it('offers Resume over held cards once the host says the agent stopped; an older host’s rows hide it', () => { + const held = { + queuePause: { reason: 'stopped' as const }, + queuedMessages: [ + { + messageId: 'held', + position: 1, + body: { kind: 'message' as const, role: 'user' as const, blocks: [] }, + state: 'waiting' as const + } + ] + } + const resume = (state: StructuredAgentSessionState) => + renderHook(() => { + const transport = useStructuredAgentSessionTransportState(state, true) + const { mutate } = useStructuredAgentSessionMutate({ + sessionId: 'session-a', + target: { kind: 'local' }, + stateRef: { current: { fence: 1 } } + }) + return useStructuredAgentSessionQueuedMessages({ + enabled: true, + queuedMessages: transport.queuedMessages ?? [], + queuePause: transport.queuePause, + submissions: [], + hasPendingPrompt: false, + isWorking: transport.isWorking, + composerScopeKey: undefined, + mutate + }) + }).result.current.queueResume + expect(resume(loaded({ ...held, working: false }))).toBeDefined() + expect(resume(loaded(held))).toBeUndefined() + }) +}) diff --git a/src/shared/agent-session-journal-schemas.ts b/src/shared/agent-session-journal-schemas.ts index 9e5262693514..6c63c6e1d397 100644 --- a/src/shared/agent-session-journal-schemas.ts +++ b/src/shared/agent-session-journal-schemas.ts @@ -70,7 +70,8 @@ const SubagentEntry = z.object({ state: z.string().min(1), tokens: z.number().optional(), startedAt: z.number().optional(), - settledAt: z.number().optional() + settledAt: z.number().optional(), + ownerFence: z.number().int().nonnegative().optional() }) /** Renderers select blocks by `type` equality and skip what they cannot draw, @@ -125,7 +126,8 @@ const Block = openDiscriminatedUnion( outputFile: z.string().optional(), tokens: z.number().optional(), startedAt: z.number().optional(), - settledAt: z.number().optional() + settledAt: z.number().optional(), + ownerFence: z.number().int().nonnegative().optional() }) ]) ) diff --git a/src/shared/agent-session-lease-adjudication.test.ts b/src/shared/agent-session-lease-adjudication.test.ts index 9c71b87c2f39..35e794efd0d2 100644 --- a/src/shared/agent-session-lease-adjudication.test.ts +++ b/src/shared/agent-session-lease-adjudication.test.ts @@ -34,7 +34,6 @@ function lease(overrides: Partial = {}): AgentSessionLease { claimStatus: 'live', unreconciled: false, deathEvidence: null, - leftoverSettledAt: null, ...overrides } } diff --git a/src/shared/agent-session-lease-adjudication.ts b/src/shared/agent-session-lease-adjudication.ts index 75de4a74c40d..3365159b7a0b 100644 --- a/src/shared/agent-session-lease-adjudication.ts +++ b/src/shared/agent-session-lease-adjudication.ts @@ -83,7 +83,8 @@ export function isProvenAliveProbe(probe: AgentSessionOwnerProbe): boolean { return probe.outcome === 'identity-matched' && probe.matchedOn.length > 0 } -function deathEvidenceFor( +/** The proof a probe gives that the lease's owner is gone, or null when it proves nothing. */ +export function agentSessionDeathEvidenceFromProbe( probe: AgentSessionOwnerProbe, observedAt: number, lease: AgentSessionLease @@ -270,7 +271,7 @@ export function adjudicateAgentSessionRestart(args: { reason: 'owner outlived the runtime that held its transport' } } - const evidence = deathEvidenceFor(probe, observedAt, lease) + const evidence = agentSessionDeathEvidenceFromProbe(probe, observedAt, lease) if (evidence) { return { disposition: 'evicted', nextFence: nextAgentSessionFence(lease), evidence } } diff --git a/src/shared/agent-session-legacy-handoff-lease.ts b/src/shared/agent-session-legacy-handoff-lease.ts index ea99a66d472d..ce06ddce0eb7 100644 --- a/src/shared/agent-session-legacy-handoff-lease.ts +++ b/src/shared/agent-session-legacy-handoff-lease.ts @@ -24,14 +24,9 @@ export type PersistedAgentSessionRuntimeKind = export type PersistedAgentSessionHandoffStage = AgentSessionHandoffStage | LegacyHandoffStage /** A lease as it may appear on disk. */ -export type PersistedAgentSessionLease = Omit< - AgentSessionLease, - 'runtimeKind' | 'handoffStage' | 'leftoverSettledAt' -> & { +export type PersistedAgentSessionLease = Omit & { runtimeKind: PersistedAgentSessionRuntimeKind handoffStage: PersistedAgentSessionHandoffStage | null - /** Absent on a lease an older build wrote: read as owed, so startup settles it once. */ - leftoverSettledAt?: number | null } /** A record as a row stores it. Decode through `decodePersistedAgentSessionRecord`. */ @@ -76,13 +71,11 @@ export function normalizeLegacyHandoffLease(lease: PersistedAgentSessionLease): const { runtimeKind, handoffStage } = lease // Why: every one of these awaited proof about an owner, which is what `recovering` resolves. const stage = isLegacyHandoffStage(handoffStage) ? 'recovering' : handoffStage - const leftoverSettledAt = lease.leftoverSettledAt ?? null if (runtimeKind === 'native') { - return { ...lease, runtimeKind, handoffStage: stage, leftoverSettledAt } + return { ...lease, runtimeKind, handoffStage: stage } } return { ...lease, - leftoverSettledAt, runtimeKind: 'native', handoffStage: stage, // Why: a recorded terminal is the user's foreground agent. `conflicted` is the claim every diff --git a/src/shared/agent-session-record.test-fixture.ts b/src/shared/agent-session-record.test-fixture.ts index 7ba49f60b17e..68a190fbbcae 100644 --- a/src/shared/agent-session-record.test-fixture.ts +++ b/src/shared/agent-session-record.test-fixture.ts @@ -34,7 +34,6 @@ export function agentSessionLeaseFixture( claimStatus: 'live', unreconciled: false, deathEvidence: null, - leftoverSettledAt: null, ...overrides } } diff --git a/src/shared/agent-session-record.ts b/src/shared/agent-session-record.ts index 94ce27cea808..185c62c8333e 100644 --- a/src/shared/agent-session-record.ts +++ b/src/shared/agent-session-record.ts @@ -141,9 +141,6 @@ export type AgentSessionLease = { minimumNextFence?: number /** Null on a released lease when nothing proved its owner gone. */ deathEvidence: AgentSessionDeathEvidence | null - /** When the work the generations before this fence left unfinished was last settled, or null - * while it is owed. An index for startup only: every settlement re-derives from the journal. */ - leftoverSettledAt: number | null } export type AgentSessionRecord = { @@ -342,8 +339,7 @@ function isPersistedAgentSessionLease(value: unknown): value is PersistedAgentSe lease.claimStatus === 'conflicted' || lease.claimStatus === 'released') && typeof lease.unreconciled === 'boolean' && - (lease.deathEvidence === null || isAgentSessionDeathEvidence(lease.deathEvidence)) && - (lease.leftoverSettledAt == null || Number.isSafeInteger(lease.leftoverSettledAt)) + (lease.deathEvidence === null || isAgentSessionDeathEvidence(lease.deathEvidence)) ) } diff --git a/src/shared/agent-session-wire.ts b/src/shared/agent-session-wire.ts index e06444e6d75f..0ef3440e3539 100644 --- a/src/shared/agent-session-wire.ts +++ b/src/shared/agent-session-wire.ts @@ -153,6 +153,11 @@ export type AgentSessionHistoryPage = { * listed, since nothing can take its answer. Absent from an older host, whose clients read every * pending prompt they hold as waiting. */ actionablePromptIds?: string[] + /** Rides with `latestTurn`: whether the agent running now is working, by the host's projection of + * current work — its running turn, or a send it owes, eligible or handed over and unanswered. + * What an agent that has ended left counts for nothing. Absent from an older host, whose + * clients derive it from the rows they hold. */ + working?: boolean } export type AgentSessionHistoryResult = @@ -214,6 +219,8 @@ export type AgentSessionSubscribeEvent = latestTurn?: AgentSessionLatestTurn | null /** Rides with `latestTurn` (`AgentSessionHistoryPage.actionablePromptIds`). */ actionablePromptIds?: string[] + /** Rides with `latestTurn` (`AgentSessionHistoryPage.working`). */ + working?: boolean } & AgentSessionFrameFields) | ({ type: 'reset' diff --git a/src/shared/native-chat-types.ts b/src/shared/native-chat-types.ts index 730a6af0032d..b3efa07c0a81 100644 --- a/src/shared/native-chat-types.ts +++ b/src/shared/native-chat-types.ts @@ -151,6 +151,9 @@ export type NativeChatSubagentEntry = { startedAt?: number /** Epoch ms the entry latched terminal. */ settledAt?: number + /** The agent generation (lease fence) whose execution this entry last observed: stamped by the + * journal on a fresh observation, kept when carried unchanged. Absent: the row's own. */ + ownerFence?: number } /** One spawn group's roster, revised in place as its children report activity. @@ -195,6 +198,8 @@ export type NativeChatBackgroundTaskBlock = { startedAt?: number /** Epoch ms the row latched terminal. */ settledAt?: number + /** As `NativeChatSubagentEntry.ownerFence`. */ + ownerFence?: number } export type NativeChatBlock = diff --git a/src/shared/structured-agent-session-coalescer.ts b/src/shared/structured-agent-session-coalescer.ts index 21e11ad4b6ce..fc418c8bf5e6 100644 --- a/src/shared/structured-agent-session-coalescer.ts +++ b/src/shared/structured-agent-session-coalescer.ts @@ -1,6 +1,7 @@ import type { AgentSessionSubscribeEvent } from './agent-session-wire' import { actionablePromptIdsAfterStructuredAgentSessionBatch, + hostWorkingAfterStructuredAgentSessionBatch, latestTurnAfterStructuredAgentSessionBatch } from './structured-agent-session-live-turn' @@ -36,6 +37,7 @@ function mergeBatch( left.actionablePromptIds, right ) + const working = hostWorkingAfterStructuredAgentSessionBatch(left.working, right) return { type: 'batch', ...(right.commands !== undefined || left.commands !== undefined @@ -63,7 +65,8 @@ function mergeBatch( ? { activity: right.activity !== undefined ? right.activity : (left.activity ?? null) } : {}), ...(latestTurn !== undefined ? { latestTurn } : {}), - ...(actionablePromptIds !== undefined ? { actionablePromptIds } : {}) + ...(actionablePromptIds !== undefined ? { actionablePromptIds } : {}), + ...(working !== undefined ? { working } : {}) } } diff --git a/src/shared/structured-agent-session-live-turn.ts b/src/shared/structured-agent-session-live-turn.ts index 91f4bda9f21a..0316defad8f2 100644 --- a/src/shared/structured-agent-session-live-turn.ts +++ b/src/shared/structured-agent-session-live-turn.ts @@ -188,23 +188,48 @@ export function actionablePromptIdsAfterStructuredAgentSessionBatch( : previous } -/** Whether a frame's turn and prompts, when it names them, are what the state already holds: a - * generation's end can change them with no row, so a frame that does is never a no-op. */ +/** The host's working answer once `event` applies, under `latestTurn`'s rule. */ +export function hostWorkingAfterStructuredAgentSessionBatch( + previous: boolean | undefined, + event: Extract +): boolean | undefined { + const { items, removedItemIds, submissions } = event.batch + const carriesRows = items.length > 0 || removedItemIds.length > 0 || submissions.length > 0 + return carriesRows || event.working !== undefined ? event.working : previous +} + +/** Whether a frame's turn, prompts and working answer, when it names them, are what the state + * already holds: a generation's end can change them with no row, so a frame that does is never a + * no-op. */ export function restatesStructuredAgentSessionHostWork( event: Pick< Extract, - 'latestTurn' | 'actionablePromptIds' + 'latestTurn' | 'actionablePromptIds' | 'working' >, - state: { latestTurn?: AgentSessionLatestTurn | null; actionablePromptIds?: string[] } + state: { + latestTurn?: AgentSessionLatestTurn | null + actionablePromptIds?: string[] + working?: boolean + } ): boolean { return ( (event.latestTurn === undefined || JSON.stringify(event.latestTurn) === JSON.stringify(state.latestTurn)) && (event.actionablePromptIds === undefined || - event.actionablePromptIds.join('\n') === state.actionablePromptIds?.join('\n')) + event.actionablePromptIds.join('\n') === state.actionablePromptIds?.join('\n')) && + (event.working === undefined || event.working === state.working) ) } +/** Whether the session's own agent is working: the host's answer when it gave one, else what the + * client derives from its rows (an older host). */ +export function structuredAgentSessionHostSaysWorking( + hostWorking: boolean | undefined, + derived: () => boolean +): boolean { + return hostWorking ?? derived() +} + /** Whether a pending prompt waits on the person: the host names the ones that do, and a prompt an * agent that has ended raised is not among them. An older host names none: every pending one. */ export function isActionableStructuredAgentSessionPrompt( diff --git a/src/shared/structured-agent-session-reducer.ts b/src/shared/structured-agent-session-reducer.ts index d82950eaf1d7..f20c673e7969 100644 --- a/src/shared/structured-agent-session-reducer.ts +++ b/src/shared/structured-agent-session-reducer.ts @@ -28,6 +28,7 @@ import { readAgentJournalTurn } from './agent-session-turn-record' import { queuePublicationField } from './structured-agent-session-queue-publication-field' import { actionablePromptIdsAfterStructuredAgentSessionBatch, + hostWorkingAfterStructuredAgentSessionBatch, latestTurnAfterStructuredAgentSessionBatch, restatesStructuredAgentSessionHostWork } from './structured-agent-session-live-turn' @@ -82,6 +83,9 @@ export type StructuredAgentSessionState = { /** The prompts the host says still wait on the person (`AgentSessionHistoryPage`); absent from * an older host, whose every pending prompt in `items` is read as waiting. */ actionablePromptIds?: string[] + /** Whether the host says the session's own agent is working (`AgentSessionHistoryPage.working`); + * absent from an older host, whose answer is derived from `items` and `submissions`. */ + working?: boolean /** Bumped per live batch that leaves a turn row's newest revision outside the window * (dropped or trimmed), so a whole-journal answer derived from turn rows is asked for again. */ unloadedTurnRevisions?: number @@ -140,6 +144,7 @@ function replacePage( ...(page.actionablePromptIds !== undefined ? { actionablePromptIds: page.actionablePromptIds } : {}), + ...(page.working !== undefined ? { working: page.working } : {}), ...(backgroundTasks !== undefined ? { backgroundTasks } : page.backgroundTasks !== undefined @@ -325,6 +330,7 @@ export function reduceStructuredAgentSession( state.actionablePromptIds, event ), + working: hostWorkingAfterStructuredAgentSessionBatch(state.working, event), ...queuePublicationField(event, state), backgroundTasks, ...(activity !== undefined ? { activity } : {}), From f8de364e901d96776252690c143e68f33182c505 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:23:10 -0700 Subject: [PATCH 04/22] fix(native-chat): round-1 review fixes for settling a gone agent's leftover work - restart offer at quit scoped by the host's live child, not the saved lease - queue pause after a restart derived from where the journal opened; the reopen-mark latch is gone - re-hand of a released lease's proof removed; acquisition is not a trigger - a journal that can never load, or a rewind only its provider can prove, is not owed - quit no longer waits for the startup scan; a reader's open goes ahead of it - the startup scan reads chats as the worker's own handle: never published, never closes a reader's - the exit settles at the dead fence before the release, so late rows still land - stop-note refresh frames carry the host's current work - cancelling an ended agent's card only dismisses it; a stale cancel stops the live turn plainly - the refusal names the agent; client readers route through the host's current work --- .../scripts/vitest-sqlite-runtime-files.mjs | 2 + .../mobile-structured-agent-session-rpc.ts | 11 +- .../use-mobile-native-chat-turn-disclosure.ts | 3 +- .../use-mobile-structured-agent-mutation.ts | 9 +- .../use-mobile-structured-agent-session.ts | 3 +- ...use-mobile-structured-agent-turn-timing.ts | 7 +- .../acp/acp-structured-host-crash.test.ts | 7 +- ...acp-structured-host-restore-failed.test.ts | 10 +- ...p-timeline-grok-subagent-retention.test.ts | 11 +- .../acp/acp-timeline-grok-subagents.test.ts | 1 + ...journal-atomic-terminal-settlement.test.ts | 2 +- .../journal-open-failure.ts | 10 + .../journal-store.test.ts | 2 +- .../agent-session-journal/journal-store.ts | 26 +- ...nal-terminal-settlement-validation.test.ts | 6 +- .../queued-message-reopen-floor.ts | 16 ++ ...der-timeline-assembler-attribution.test.ts | 11 +- .../agent-session-stop-note-refresh.test.ts | 140 +++++++++ .../agent-session-stop-note-refresh.ts | 12 +- .../structured-agent-session-child-exit.ts | 35 +-- ...red-agent-session-conversation-lifetime.ts | 9 - ...uctured-agent-session-conversation-open.ts | 1 + ...-agent-session-ended-prompt-cancel.test.ts | 177 ++++++++++++ .../structured-agent-session-host-delivery.ts | 4 +- .../structured-agent-session-host-lifetime.ts | 10 +- .../structured-agent-session-host.ts | 11 +- ...ssion-leftover-settlement-triggers.test.ts | 4 +- ...red-agent-session-live-turn-window.test.ts | 2 +- ...ructured-agent-session-mutation-context.ts | 3 - .../structured-agent-session-prompt-cancel.ts | 25 ++ .../structured-agent-session-prompt-state.ts | 44 +++ ...tructured-agent-session-queued-messages.ts | 7 +- .../structured-agent-session-queued-wiring.ts | 1 - ...tured-agent-session-reconciliation-load.ts | 66 +++++ ...tured-agent-session-reconciliation-pass.ts | 50 ++-- ...ured-agent-session-reconciliation-slots.ts | 56 ++++ ...gent-session-reconciliation-worker.test.ts | 135 ++++++++- ...red-agent-session-reconciliation-worker.ts | 270 ++++++++++-------- ...tured-agent-session-restart-resume.test.ts | 12 +- .../structured-agent-session-reveal.ts | 13 +- ...ured-agent-session-send-settlement.test.ts | 21 +- ...agent-session-startup-failure-exit.test.ts | 4 +- ...gent-session-startup-scan-lifetime.test.ts | 219 ++++++++++++++ ...agent-session-startup-scan.test-fixture.ts | 162 +++++++++++ ...uctured-agent-session-startup-scan.test.ts | 200 +++++-------- ...ctured-agent-session-startup-settlement.ts | 6 +- ...d-agent-session-stop-event-binding.test.ts | 4 +- ...d-agent-session-stop-event-entries.test.ts | 2 +- ...d-agent-session-stop-event-restart.test.ts | 2 +- ...tructured-agent-session-stop-event.test.ts | 2 +- ...red-agent-session-stop-never-waits.test.ts | 2 +- ...-agent-session-stop-settle-binding.test.ts | 2 +- .../structured-agent-session-stopping.test.ts | 8 +- .../structured-agent-session-subscribers.ts | 3 +- .../structured-agent-session-turns-cancel.ts | 15 +- ...ured-agent-session-unexpected-exit.test.ts | 7 +- ...t-session-wedged-profile-migration.test.ts | 12 +- ...tured-agent-session-working-at-teardown.ts | 12 +- .../agent-session-generation-end.test.ts | 15 +- .../runtime/agent-session-generation-end.ts | 20 +- .../native-chat/NativeChatMessageList.tsx | 11 +- .../NativeChatResolutionReceipt.test.tsx | 16 ++ .../NativeChatResolutionReceipt.tsx | 12 +- .../NativeChatStructuredSession.tsx | 1 + .../native-chat/NativeChatTranscriptRow.tsx | 12 +- .../agent-session-failure-words-text.ts | 6 + .../agent-session-write-notice-text.ts | 10 +- .../native-chat-transcript-slots.ts | 6 +- .../use-structured-agent-session-mutate.ts | 25 +- ...tructured-agent-session-transport-state.ts | 15 + .../use-structured-agent-session-transport.ts | 7 +- .../use-structured-agent-session.ts | 5 +- .../src/i18n/en-runtime-required.json | 1 + src/renderer/src/i18n/locales/en.json | 1 + src/renderer/src/i18n/locales/es.json | 1 + src/renderer/src/i18n/locales/fr.json | 1 + src/renderer/src/i18n/locales/ja.json | 1 + src/renderer/src/i18n/locales/ko.json | 1 + src/renderer/src/i18n/locales/zh.json | 1 + src/shared/agent-session-failure-copy.ts | 2 + .../agent-session-failure-words.test.ts | 8 +- src/shared/agent-session-failure-words.ts | 14 +- .../agent-session-refusal-notice.test.ts | 25 +- src/shared/agent-session-refusal-notice.ts | 10 +- .../agent-session-refusal-reason-words.ts | 3 +- ...-messages-waiting-behind-live-turn.test.ts | 52 ++++ ...-chat-messages-waiting-behind-live-turn.ts | 24 +- src/shared/native-chat-turn-membership.ts | 2 + 88 files changed, 1717 insertions(+), 498 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts create mode 100644 src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts create mode 100644 src/shared/native-chat-messages-waiting-behind-live-turn.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index fe4ab072e38a..a55e2b5015d3 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -113,6 +113,8 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts', diff --git a/mobile/src/session/mobile-structured-agent-session-rpc.ts b/mobile/src/session/mobile-structured-agent-session-rpc.ts index 7b2da967d56c..4802042435b1 100644 --- a/mobile/src/session/mobile-structured-agent-session-rpc.ts +++ b/mobile/src/session/mobile-structured-agent-session-rpc.ts @@ -127,6 +127,8 @@ export async function requestStructuredAgentSessionMutation(args: { fields: Record clientOperationId?: string timeoutMs?: number + /** The chat's agent, which a refusal about it names. */ + agentName?: string }): Promise> { const { client, @@ -136,8 +138,10 @@ export async function requestStructuredAgentSessionMutation(args: { expectedRuntimeFence, fields, clientOperationId, - timeoutMs + timeoutMs, + agentName } = args + const context = agentName ? { agentName } : {} try { const result = await callAgentSession>( client, @@ -171,7 +175,8 @@ export async function requestStructuredAgentSessionMutation(args: { code: result.refusal.code, message: agentSessionRefusalNotice( result.refusal, - phoneWriteKind(fingerprintMethod, fields) + phoneWriteKind(fingerprintMethod, fields), + context ) } } catch (error) { @@ -184,7 +189,7 @@ export async function requestStructuredAgentSessionMutation(args: { return { status: 'failed', message: agentSessionWriteNoticeEnglish( - agentSessionWriteNoticeParts(answered, phoneWriteKind(fingerprintMethod, fields)) + agentSessionWriteNoticeParts(answered, phoneWriteKind(fingerprintMethod, fields), context) ) } } diff --git a/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts b/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts index ed93bda5924a..f75782b657aa 100644 --- a/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts +++ b/mobile/src/session/use-mobile-native-chat-turn-disclosure.ts @@ -145,7 +145,8 @@ export function useMobileNativeChatTurnDisclosure({ rows, turnJournal?.items, stopping, - turnJournal?.submissions + turnJournal?.submissions, + turnJournal ?? {} ) : null if (!ids?.size) { diff --git a/mobile/src/session/use-mobile-structured-agent-mutation.ts b/mobile/src/session/use-mobile-structured-agent-mutation.ts index 4eb8a203121b..d8a6a30fb7bf 100644 --- a/mobile/src/session/use-mobile-structured-agent-mutation.ts +++ b/mobile/src/session/use-mobile-structured-agent-mutation.ts @@ -22,8 +22,10 @@ export function useMobileStructuredAgentMutate(args: { enabled: boolean stateRef: { readonly current: StructuredAgentSessionState } onSendError: (message: string) => void + /** The chat's agent, which a refusal about it names. */ + agentName?: string }): MobileStructuredAgentMutate { - const { client, enabled, onSendError, sessionId, stateRef } = args + const { agentName, client, enabled, onSendError, sessionId, stateRef } = args return useCallback( async ( method: string, @@ -41,7 +43,8 @@ export function useMobileStructuredAgentMutate(args: { fingerprintMethod, sessionId, expectedRuntimeFence: targetFence, - fields + fields, + ...(agentName ? { agentName } : {}) }) if (result.status === 'accepted') { return { @@ -56,6 +59,6 @@ export function useMobileStructuredAgentMutate(args: { onSendError(result.message) return { status: 'rejected' } }, - [client, enabled, onSendError, sessionId, stateRef] + [agentName, client, enabled, onSendError, sessionId, stateRef] ) } diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index 4633519adef8..1b1aab57c176 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -91,7 +91,8 @@ export function useMobileStructuredAgentSession(args: { sessionId, enabled, stateRef, - onSendError + onSendError, + ...(agent ? { agentName: tuiAgentDisplayName(agent) ?? agent } : {}) }) const options = useMobileStructuredAgentOptions({ diff --git a/mobile/src/session/use-mobile-structured-agent-turn-timing.ts b/mobile/src/session/use-mobile-structured-agent-turn-timing.ts index a701cdbc21d1..d23bb936c545 100644 --- a/mobile/src/session/use-mobile-structured-agent-turn-timing.ts +++ b/mobile/src/session/use-mobile-structured-agent-turn-timing.ts @@ -21,11 +21,14 @@ export function useMobileStructuredAgentTurnTiming( items, submissions, latestTurn, + working, hostClock }: { items: readonly AgentJournalRenderItem[] submissions: readonly AgentJournalSubmission[] latestTurn?: AgentSessionLatestTurn | null + /** The host's working answer, which places rows waiting behind the live turn. */ + working?: boolean hostClock?: StructuredAgentHostClock | null }, turnId: string | null @@ -40,8 +43,8 @@ export function useMobileStructuredAgentTurnTiming( [items, submissions, turnId, latestTurn] ) const turnJournal = useMemo( - () => ({ items, submissions, latestTurn }), - [items, submissions, latestTurn] + () => ({ items, submissions, latestTurn, ...(working !== undefined ? { working } : {}) }), + [items, submissions, latestTurn, working] ) const [latch, setLatch] = useState(null) // Stamp during render (React's derive-from-props pattern) so the first paint of diff --git a/src/main/acp/acp-structured-host-crash.test.ts b/src/main/acp/acp-structured-host-crash.test.ts index 742e0de2b6eb..39032c7a86c6 100644 --- a/src/main/acp/acp-structured-host-crash.test.ts +++ b/src/main/acp/acp-structured-host-crash.test.ts @@ -38,14 +38,17 @@ async function midReply() { batches.push(batch) return append(batch) }) - // A settlement plans its batch at its own turn in the write queue: recorded as it is planned. + // A settlement plans its batch at its own turn in the write queue: recorded as it is planned, and + // only when it writes something (an empty plan writes no row). const appendPlanned = journal.appendPlannedLifecycleBatch vi.spyOn(journal, 'appendPlannedLifecycleBatch').mockImplementation((input) => appendPlanned({ ...input, plan: () => { const planned = input.plan() - batches.push({ ...input, mutations: planned.mutations }) + if (planned.mutations.length > 0 || planned.dispatches.length > 0) { + batches.push({ ...input, mutations: planned.mutations }) + } return planned } }) diff --git a/src/main/acp/acp-structured-host-restore-failed.test.ts b/src/main/acp/acp-structured-host-restore-failed.test.ts index 27d8b4eaa505..75ea27a91e67 100644 --- a/src/main/acp/acp-structured-host-restore-failed.test.ts +++ b/src/main/acp/acp-structured-host-restore-failed.test.ts @@ -224,11 +224,11 @@ describe('the warning row of a replacement whose attach failed', () => { const { host, store, fence, warnings } = opened expect(await host.attach(CALLER, attachParams())).toMatchObject({ ok: true }) await host.close(SESSION, 'user-close') - // The journal's open fails after the fresh session's link is durable, before its row is written: - // the start's leftover settlement meets it first and goes on, then the attach's own open. - vi.spyOn(AgentSessionJournal.prototype, 'open') - .mockRejectedValueOnce(new Error('journal path unavailable')) - .mockRejectedValueOnce(new Error('journal path unavailable')) + // The journal's open fails after the fresh session's link is durable, before its row is written. + // A start settles nothing, so the attach's own open is the only one. + vi.spyOn(AgentSessionJournal.prototype, 'open').mockRejectedValueOnce( + new Error('journal path unavailable') + ) await expect(host.attach(CALLER, attachParams(fence()))).rejects.toThrow( 'journal path unavailable' ) diff --git a/src/main/acp/acp-timeline-grok-subagent-retention.test.ts b/src/main/acp/acp-timeline-grok-subagent-retention.test.ts index edacc93ab735..44065c2843b5 100644 --- a/src/main/acp/acp-timeline-grok-subagent-retention.test.ts +++ b/src/main/acp/acp-timeline-grok-subagent-retention.test.ts @@ -153,12 +153,20 @@ describe('Grok roster ownership after more than 32 spawning groups', () => { { id: 'old', label: 'Original label', + ownerFence: 1, startedAt: 2, settledAt: 301, state: 'completed', tokens: 12 }, - { id: 'sibling', label: 'Original label 2', startedAt: 3, settledAt: 4, state: 'completed' } + { + id: 'sibling', + label: 'Original label 2', + ownerFence: 1, + startedAt: 3, + settledAt: 4, + state: 'completed' + } ]) expect( (await roster(fixture)).flatMap(({ group }) => @@ -192,6 +200,7 @@ describe('Grok roster ownership after more than 32 spawning groups', () => { { id: 'old', label: 'Original label', + ownerFence: 1, startedAt: 2, settledAt: 301, state: kind === 'output' ? 'completed' : 'stopped' diff --git a/src/main/acp/acp-timeline-grok-subagents.test.ts b/src/main/acp/acp-timeline-grok-subagents.test.ts index 2acd725cf284..f3d7adadb158 100644 --- a/src/main/acp/acp-timeline-grok-subagents.test.ts +++ b/src/main/acp/acp-timeline-grok-subagents.test.ts @@ -62,6 +62,7 @@ describe('Grok subagents through the shared timeline', () => { { id: 'subagent-1', label: 'Count note lines', + ownerFence: 1, state: 'completed', tokens: 13778, startedAt: expect.any(Number), diff --git a/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts b/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts index ca622c5b2f71..f20b24d60b3c 100644 --- a/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts @@ -209,7 +209,7 @@ describe('atomic terminal settlement', () => { expect(frames.at(-1)).toMatchObject({ type: 'batch', batch: { cursor: settled.cursor } }) const frameCount = frames.length expect(settled.submissions[0]).toMatchObject({ dispatchState: 'unknown', recovered: true }) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() const calls = settled.items.filter((item) => item.body.kind === 'tool-call') expect(calls).toHaveLength(201) expect( diff --git a/src/main/native-chat/agent-session-journal/journal-open-failure.ts b/src/main/native-chat/agent-session-journal/journal-open-failure.ts index 73349ab09391..eb259b81e437 100644 --- a/src/main/native-chat/agent-session-journal/journal-open-failure.ts +++ b/src/main/native-chat/agent-session-journal/journal-open-failure.ts @@ -88,6 +88,16 @@ export function isJournalWrittenByNewerOrca(error: unknown): boolean { return error instanceof AgentSessionJournalError && error.code === 'journal_read_only' } +/** A failed open no retry gets past in this process: damage, or a newer Orca's journal. */ +export function isJournalOpenFailurePermanent(error: unknown): boolean { + const refusal = journalOpenRefusal(error) + if (refusal.code !== 'agent_session_journal_unreadable') { + return false + } + const reason = refusal.details?.reason + return reason === 'journalCorrupt' || reason === 'journalWrittenByNewerOrca' +} + /** Why a journal open failed. A newer Orca's journal clears when this Orca is updated, so it is * not damage. */ function journalRefusalReason(error: unknown): JournalRefusalReason { diff --git a/src/main/native-chat/agent-session-journal/journal-store.test.ts b/src/main/native-chat/agent-session-journal/journal-store.test.ts index 5c5637de2dfa..6b2488c259a6 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.test.ts @@ -141,7 +141,7 @@ describe('sequences', () => { const rendered = (): string | null => activeStructuredAgentSessionTurnId(journal.snapshot().items) const bothAgreeOn = async (turnId: string | null): Promise => { - expect(journal.activeTurnId()).toBe(turnId) + expect(journal.runningTurn()?.turnId ?? null).toBe(turnId) expect(rendered()).toBe(turnId) } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 1363b12536d9..afc8960fd2e6 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -1,6 +1,7 @@ // Append-only journal store for one agent session. It owns the chat's fold and write queue, and no // connection: every statement goes through the host's one journal database. +import { queuedMessageReopenFloor } from './queued-message-reopen-floor' import type { AgentJournalDispatchRejection } from '../../../shared/agent-session-failure-words' import { randomUUID } from 'node:crypto' import type { @@ -169,17 +170,23 @@ export class AgentSessionJournal { /** Where this handle opened the journal (`wroteBeforeOpen`). */ openedAt = (): AgentJournalCursor => this.openedThrough - /** Where the reopen's pause begins when this handle could not write its mark: where the mark - * would have gone. Null once a mark is written. Per handle, so the next open marks again. */ - reopenFloor(): AgentJournalCursor | null { - return this.reopenUnmarked - } + /** Where the reopen's pause begins while no mark holds it: where this handle's mark would go. + * Null once a mark is written. Per handle, so the next open derives it again. */ + reopenFloor = (): AgentJournalCursor | null => this.reopenUnmarked async open(): Promise { await this.restore() this.openedThrough = this.cursor() } + /** A conversation's open, which always follows the chat stopping (a quit, a crash or a close): + * a card it finds waits for a turn from where this handle opened, derived with nothing written, + * unless an earlier mark already holds it. The mark that follows records the same start. */ + holdReopenFromOpen(): void { + this.reopenUnmarked = + queuedMessageReopenFloor(this.queuedMessages, this.openedThrough) ?? this.reopenUnmarked + } + /** Refuses every later write and resolves once the admitted ones have landed. Holds no * connection, so there is nothing to release and nothing that can fail. */ close(): Promise { @@ -224,12 +231,9 @@ export class AgentSessionJournal { } } - /** The turn this journal records as running, whichever generation opened it — the same read a - * client's snapshot gives, without materialising one. Whether it is current work is the host - * projection's to say (`structuredAgentSessionCurrentWork`). */ - activeTurnId = (): string | null => this.runningTurn()?.turnId ?? null - - /** That turn with its record's item. */ + /** The turn this journal records as running, with its record's item, whichever generation + * opened it — the same read a client's snapshot gives, without materialising one. Whether it is + * current work is the host projection's to say (`structuredAgentSessionCurrentWork`). */ runningTurn = (): { item: AgentJournalRenderItem; turnId: string } | null => runningStructuredAgentSessionTurnItemBySequence(this.state.items.values()) diff --git a/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts b/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts index dba12dc49d3f..3d9f3b0ce85b 100644 --- a/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-terminal-settlement-validation.test.ts @@ -117,7 +117,7 @@ it.each([ turnId: 'old', state: 'interrupted' }) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() const ended = journal.snapshot() await settle(observedAt, lastProvenAliveAt) expect(journal.snapshot()).toEqual(ended) @@ -145,7 +145,7 @@ it.each([{ kind: 'tool-call' }, { providerTurnId: '' }])( state: 'interrupted', completedAt: 1_000 }) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() await expectSendStillWritable() } ) @@ -329,7 +329,7 @@ it.each([ } expect(journal.itemBody('orca:extension-7')).toHaveProperty('laterField', { kind: 'tool-call' }) expect(journal.itemBody('orca:extension-6')).toHaveProperty('contextUsage', laterContextUsage) - expect(journal.activeTurnId()).toBeNull() + expect(journal.runningTurn()?.turnId ?? null).toBeNull() expect(journal.snapshot().items).toHaveLength(inputs.length + 1) const ended = journal.snapshot() await settle(observedAt, lastProvenAliveAt) diff --git a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts new file mode 100644 index 000000000000..5d97e175d71b --- /dev/null +++ b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts @@ -0,0 +1,16 @@ +// Where a reopened chat's queue pause starts before any reopen mark is written: derived from where +// the conversation's handle opened, so nothing has to land first (`queued-message-pause.ts`). + +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { DerivedQueuePause } from './queued-message-pause' + +/** Just past `opened` when a card waits for a reopen mark and no earlier mark holds it already; + * null otherwise. */ +export function queuedMessageReopenFloor( + queue: { awaitReopenMark: () => boolean; pauses: () => readonly DerivedQueuePause[] }, + opened: AgentJournalCursor +): AgentJournalCursor | null { + return queue.awaitReopenMark() && !queue.pauses().some((pause) => pause.reason === 'restarted') + ? { epoch: opened.epoch, sequence: opened.sequence + 1 } + : null +} diff --git a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts index c49d266e3432..82703eaf0d70 100644 --- a/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts +++ b/src/main/native-chat/agent-session-timeline/provider-timeline-assembler-attribution.test.ts @@ -215,9 +215,14 @@ describe('a background task', () => { rig.assembler.apply({ type: 'item.open', item: 'ran', body: backgroundTask('ran', 'working') }) rig.assembler.apply({ type: 'item.close', item: 'ran', body: backgroundTask('ran', 'done') }) rig.assembler.apply({ type: 'session.ended', verdict: { state: 'unverifiable' } }) - expect((await rig.row(providerItemId('item', 'bg')))?.body).toEqual( - backgroundTask('bg', 'unverifiable') - ) + // The entry keeps the generation that ran it (`ownerFence`), stamped when it was observed. + const unverifiable = backgroundTask('bg', 'unverifiable') + expect((await rig.row(providerItemId('item', 'bg')))?.body).toEqual({ + ...unverifiable, + blocks: unverifiable.blocks.map((block) => + block.type === 'background-task' ? { ...block, ownerFence: 1 } : block + ) + }) expect(await backgroundTaskState(rig, 'ran')).toBe('done') }) }) diff --git a/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts new file mode 100644 index 000000000000..1f22d20e6d23 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.test.ts @@ -0,0 +1,140 @@ +// The stop-note refresh frame carries rows, so it restates the host's current work as every rows +// frame does: a client reads a rows frame without it as an older host's and falls back to its own +// raw derivation (an ended turn as Working, an ended agent's prompt as answerable). + +import { describe, expect, it } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' +import type { + AgentSessionHistoryPage, + AgentSessionSubscribeEvent +} from '../../../shared/agent-session-wire' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession +} from '../../../shared/structured-agent-session-reducer' +import { isActionableStructuredAgentSessionPrompt } from '../../../shared/structured-agent-session-live-turn' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { refreshDerivedStopNotes } from './agent-session-stop-note-refresh' +import { structuredAgentSessionStopNoteIdentity } from './structured-agent-session-command-turn' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { Subscriber } from './structured-agent-session-subscribers' + +const staleTurn: AgentJournalRenderItem = { + itemId: 'turn-old', + revision: 1, + sequence: 1, + observedAt: 1, + body: { kind: 'turn', turnId: 't-old', state: 'running', startedAt: 1 } +} +const stalePrompt: AgentJournalRenderItem = { + itemId: 'prompt-old', + revision: 1, + sequence: 2, + observedAt: 2, + body: { + kind: 'approval', + title: 'Run ls', + detail: null, + options: [{ id: 'allow', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + } +} +const noteId = agentJournalItemKey(structuredAgentSessionStopNoteIdentity('t-old')) +const unconfirmed = { + kind: 'status' as const, + text: 'x', + failure: { kind: 'cancelUnconfirmed' as const } +} +const projectedNote: AgentJournalRenderItem = { + itemId: noteId, + revision: 2, + sequence: 3, + observedAt: 3, + body: { kind: 'status', text: 'Stopped.' } +} +const items = [staleTurn, stalePrompt, projectedNote] + +/** What a current host publishes: the ended generation's turn and prompt are not current. */ +function page(): AgentSessionHistoryPage { + return { + sessionId: 's', + epoch: 'e', + direction: 'tail', + items, + removedItemIds: [], + submissions: [], + window: { + oldest: { epoch: 'e', sequence: 1 }, + newest: { epoch: 'e', sequence: 3 }, + nextCursor: { epoch: 'e', sequence: 1 } + }, + liveCursor: { epoch: 'e', sequence: 3 }, + hasOlder: false, + hasNewer: false, + latestTurn: null, + actionablePromptIds: [], + working: false + } +} + +describe('the stop-note refresh frame', () => { + it("restates the host's current work, so a client keeps the host's answers", () => { + const emitted: AgentSessionSubscribeEvent[] = [] + const journal = { + visitItems: (visit: (itemId: string, sequence: number, body: unknown) => void) => + visit(noteId, 3, unconfirmed), + snapshot: () => ({ + sessionId: 's', + cursor: { epoch: 'e', sequence: 3 }, + items, + submissions: [] + }), + itemBody: (itemId: string) => (itemId === noteId ? unconfirmed : null) + } + // The generation live now is fence 2; both stale items came from fence 1. + const work = new StructuredAgentSessionCurrentWork( + { + runningTurn: () => ({ item: staleTurn, turnId: 't-old' }), + itemFence: () => 1, + visitItems: (visit) => { + for (const item of items) { + visit(item.itemId, item.sequence, item.body) + } + }, + submissions: () => [], + wroteBeforeOpen: () => false + }, + 2 + ) + const subscriber = { sessionId: 's', cursor: { epoch: 'e', sequence: 3 }, fence: 2 } + refreshDerivedStopNotes( + { emit: (_subscriber, event) => emitted.push(event), isActive: () => true }, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the refresh reads only the subscriber's session, cursor and fence. + subscriber as unknown as Subscriber, + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the refresh reads only visitItems, snapshot and itemBody, all supplied. + journal as unknown as AgentSessionJournal, + 10, + work + ) + + expect(emitted).toHaveLength(1) + const frame = emitted[0] + if (frame?.type !== 'batch') { + throw new Error('expected a batch') + } + expect(frame.batch.items.length).toBeGreaterThan(0) + expect(frame).toMatchObject({ working: false, actionablePromptIds: [], latestTurn: null }) + + let state = reduceStructuredAgentSession(EMPTY_STRUCTURED_AGENT_SESSION, { + type: 'event', + event: { type: 'snapshot', sessionId: 's', page: page(), fence: 2 } + }) + state = reduceStructuredAgentSession(state, { type: 'event', event: frame }) + expect(state.working).toBe(false) + expect(state.latestTurn).toBeNull() + expect(isActionableStructuredAgentSessionPrompt('prompt-old', state.actionablePromptIds)).toBe( + false + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts index 5f86048af0c1..83ce92fc7cf4 100644 --- a/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts +++ b/src/main/native-chat/agent-session-wire/agent-session-stop-note-refresh.ts @@ -6,16 +6,23 @@ import { oversizedHistoryItem } from './agent-session-history-page-bounds' import { emptyAgentSessionBatch } from './agent-session-empty-batch' +import { + agentSessionPublishedWorkFields, + type AgentSessionPublishedWork +} from './agent-session-history-page' import type { SubscriberDeliveryPort } from './agent-session-subscriber-catch-up' import type { Subscriber } from './structured-agent-session-subscribers' import { isStructuredAgentSessionStopNote } from './structured-agent-session-command-turn' -/** A stale in-flight history page can land after its note's live correction was out of window. */ +/** A stale in-flight history page can land after its note's live correction was out of window. + * `work`: the host's current work, which a frame carrying rows restates, as every such frame does + * (a client reads a rows frame without it as an older host's). */ export function refreshDerivedStopNotes( port: Pick, subscriber: Subscriber, journal: AgentSessionJournal, - hostNow: number + hostNow: number, + work?: AgentSessionPublishedWork ): void { let hasUnconfirmedStop = false journal.visitItems((itemId, _sequence, body) => { @@ -39,6 +46,7 @@ export function refreshDerivedStopNotes( sessionId: subscriber.sessionId, batch: { ...emptyAgentSessionBatch(subscriber.cursor), items: pending }, fence: subscriber.fence, + ...(work ? agentSessionPublishedWorkFields(journal, work) : {}), hostNow }) pending = [] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index e11315074089..3dd260ddf140 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -239,31 +239,19 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< ...(unrunRejection ? { unrunRejection } : {}) } } finally { - // The root's exit was observed, so the owner is released first, writing the proof and ending - // the generation; then what it left is settled at the new fence. A release or a settlement - // that cannot commit stays owed: the chat's reconciliation worker retries it. + // What the exited child left is settled first, at its own fence, as its own closing tail lands: + // a row its closed sink still delivers lands too, and wakes the chat's worker to settle it once + // the release below moves the fence. A settlement or a release that cannot commit stays owed: + // the chat's reconciliation worker retries it. let released = false let owed: StructuredAgentSessionExitSettlement | null = null - try { - await releaseStoredStructuredAgentSessionOwnerAfterExit({ - store: context.store, - sessionId, - expectedFence: child.fence, - now: context.now(), - exitObservedAt: observedAt, - exitReason: exit.reason - }) - released = true - } catch (error) { - logExitFailure(context, sessionId, 'exit-owner-release', error) - } if (settlement) { const settled = await settleStructuredAgentSessionLeftovers({ store: context.store, sessionId, journal: session.journal, exit: settlement, - // Ended whether or not the release landed: the host saw the root go. + // Ended whether or not the release lands: the host saw the root go. ended: { fence: child.fence, rootGone: true } }) if (!settled.ok) { @@ -271,6 +259,19 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< logExitFailure(context, sessionId, 'exit-settlement', settled.error) } } + try { + await releaseStoredStructuredAgentSessionOwnerAfterExit({ + store: context.store, + sessionId, + expectedFence: child.fence, + now: context.now(), + exitObservedAt: observedAt, + exitReason: exit.reason + }) + released = true + } catch (error) { + logExitFailure(context, sessionId, 'exit-owner-release', error) + } if (context.route) { const { runtimeState, acknowledgeRelease } = context.route await evictStructuredAgentSession({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index e6698a7d10b2..4211929805cc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -44,8 +44,6 @@ export function createStructuredAgentSessionConversationLifetime(host: { closeStatus: (sessionId: string, options: { listed: boolean }) => void /** The session's child records, the host's one read of them. */ readChildWork: (sessionId: string) => readonly AgentChildWorkView[] | undefined - /** `StructuredAgentSessionReconciliation.deferCloseAtRest`. */ - deferCloseAtRest: (sessionId: string) => boolean }) { let disposed = false const { sessions, serialize } = host @@ -61,10 +59,6 @@ export function createStructuredAgentSessionConversationLifetime(host: { if (await releaseLeaseOfEndedStructuredAgentSessionChild(host.context(), sessionId)) { return false } - // At rest it waits for the chat's reconciliation, which would only open it again. - if (atRest && host.deferCloseAtRest(sessionId)) { - return false - } const closed = await closeStructuredAgentSessionConversationUnderSerialize( { sessions, @@ -112,9 +106,6 @@ export function createStructuredAgentSessionConversationLifetime(host: { return { idleSweep, - /** The idle sweep's close, for a conversation only the reconciliation worker opened. */ - closeAtRest: (sessionId: string): Promise => - serialize(sessionId, () => closeConversation(sessionId, true)), stopAgent, /** A host stop of the child the expired attempt published, if it is still that one and still * starting: its end fails what is queued, as the idle sweep's stop of a start does. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts index cdee4f6f7e2f..062e3008c85c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-open.ts @@ -73,6 +73,7 @@ export async function openStructuredAgentSessionConversationJournal( }) const identity = journalIdentityFor(record, params) const journal = await openAgentSessionJournal({ identity, database: deps.journalDatabase }) + journal.holdReopenFromOpen() return { session: { journal, params, child: null } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts new file mode 100644 index 000000000000..1c9b509d6bc8 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts @@ -0,0 +1,177 @@ +// A Cancel that names a card an ended generation raised, from a client that still shows it: the +// card is dismissed in the journal alone, no provider is asked about it, and the live generation's +// turn is stopped only by a request that names it (an older phone's Stop carries the card it shows). + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + AGENT_JOURNAL_THREAD_SCOPE, + type AgentSessionJournalIdentity +} from '../../../shared/agent-session-journal-types' +import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { claudePromptCancelRoute } from '../../claude/claude-structured-prompt-replies' +import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import type { StructuredAgentSessionChatStopRun } from './structured-agent-session-chat-stop' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { cancelStructuredAgentSessionPrompt } from './structured-agent-session-prompt-cancel' +import { performCancel, type AgentSessionTurnContext } from './structured-agent-session-turns' + +const IDENTITY: AgentSessionJournalIdentity = { + sessionId: 'session-1', + workspaceId: 'workspace-1', + hostId: 'host-1', + agent: 'codex', + providerHandle: codexProviderHandle('thread-1') +} +const ENDED = 1 +const LIVE = 2 + +const journals = createTrackedJournalOpener() +let root: string | null = null + +afterEach(async () => { + await journals.closeAll() + if (root) { + await rm(root, { recursive: true, force: true }) + root = null + } +}) + +/** Generation 1 raised a card and ended without its cleanup; generation 2 runs `turn-2`. */ +async function endedCardAndLiveTurn(): Promise<{ journal: AgentSessionJournal; itemId: string }> { + root = await mkdtemp(join(tmpdir(), 'orca-ended-prompt-cancel-')) + const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) + const turn = (turnId: string, fence: number) => + journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId, ordinal: 0 }, + { kind: 'turn', turnId, state: 'running', startedAt: fence }, + { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + const old = await turn('turn-1', ENDED) + const card = await journal.appendItem( + { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 1 }, + { + kind: 'approval', + title: 'Approve?', + detail: null, + options: [{ id: 'allow', label: 'Allow' }], + resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } + }, + { fence: ENDED, turnScope: { kind: 'turn', turnItemId: old.itemId } } + ) + await turn('turn-2', LIVE) + return { journal, itemId: card.itemId } +} + +function provider(routePromptCancel?: StructuredAgentSessionAdapter['routePromptCancel']) { + return { + cancelTurn: vi.fn(async (_input: { turnId?: string; prompt?: unknown }) => ({ + cancelled: true + })), + dismissPrompt: vi.fn(async () => undefined), + ...(routePromptCancel ? { routePromptCancel } : {}) + } +} + +function context( + journal: AgentSessionJournal, + adapter: ReturnType +): AgentSessionTurnContext { + return { + logger: createStructuredAgentSessionLogger(), + sessionId: 'session-1', + journal, + fence: LIVE, + agents: NO_STRUCTURED_AGENTS, + agent: 'codex', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: a Cancel reaches only cancelTurn, dismissPrompt and routePromptCancel, all supplied. + adapter: adapter as unknown as StructuredAgentSessionAdapter, + persistOptions: async () => undefined, + resolvedBy: 'client-1', + publish: vi.fn(), + now: () => 1 + } +} + +function chatStop() { + return vi.fn(async (): Promise => ({ + outcome: { ok: true, value: { turnId: 'turn-2', cancelled: true } }, + endsSession: false + })) +} + +function resolution(journal: AgentSessionJournal, itemId: string) { + const body = journal.snapshot().items.find((item) => item.itemId === itemId)?.body + return body?.kind === 'approval' ? body.resolution.state : undefined +} + +describe('a card an ended generation raised, cancelled from a client that still shows it', () => { + it.each([ + ['naming no turn', undefined], + ["naming the ended generation's turn", 'turn-1'] + ])( + 'ACP (no card route): %s, it is dismissed and the live turn is not interrupted', + async (_name, turnId) => { + const { journal, itemId } = await endedCardAndLiveTurn() + const acp = provider() + const stop = chatStop() + const interrupt = vi.fn() + + const result = await cancelStructuredAgentSessionPrompt( + context(journal, acp), + { ...(turnId ? { turnId } : {}), prompt: { itemId, expectedRevision: 1 } }, + { stop, interrupt } + ) + + expect(result).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(resolution(journal, itemId)).toBe('cancelled') + expect(interrupt).not.toHaveBeenCalled() + expect(stop).not.toHaveBeenCalled() + expect(acp.cancelTurn).not.toHaveBeenCalled() + expect(acp.dismissPrompt).not.toHaveBeenCalled() + } + ) + + it.each([ + ['Claude (an approval routes to a dismissal)', claudePromptCancelRoute], + ['Codex (no card route)', undefined] + ])("%s: an older phone's Stop carrying it still stops the live turn", async (_name, route) => { + const { journal, itemId } = await endedCardAndLiveTurn() + const agent = provider(route) + const stop = chatStop() + const interrupt = vi.fn() + + const result = await cancelStructuredAgentSessionPrompt( + context(journal, agent), + { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } }, + { stop, interrupt } + ) + + expect(result).toMatchObject({ ok: true, value: { turnId: 'turn-2', cancelled: true } }) + expect(stop).toHaveBeenCalledOnce() + expect(interrupt).not.toHaveBeenCalled() + expect(agent.dismissPrompt).not.toHaveBeenCalled() + expect(resolution(journal, itemId)).toBe('cancelled') + }) + + it('reaching the Stop itself, the card is dropped and the live turn is stopped plainly', async () => { + const { journal, itemId } = await endedCardAndLiveTurn() + const codex = provider() + + await performCancel(context(journal, codex), { + clientOperationId: 'cancel-1', + turnId: 'turn-2', + prompt: { itemId, expectedRevision: 1 } + }) + + expect(codex.cancelTurn).toHaveBeenCalledOnce() + expect(codex.cancelTurn.mock.calls[0]?.[0]).not.toHaveProperty('prompt') + expect(codex.cancelTurn.mock.calls[0]?.[0]).toMatchObject({ turnId: 'turn-2' }) + expect(resolution(journal, itemId)).toBe('cancelled') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index c6b7cdea1995..5eac9309bcd3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -113,8 +113,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { sessionId: string, opened: OpenedStructuredAgentSessionConversation ): Promise => { - // Indexing a chat writes nothing and wakes nothing: what an earlier process left queued is - // startup's to hold, and the loop's next wake holds whatever that missed before any hand-over. + // Indexing a chat writes nothing: the loop never hands over a send an earlier process left + // (`StructuredAgentSessionCurrentWork.handsOver`), and keeping one as a card is startup's. sessions.set(sessionId, opened.session) input.clientDelivery.publishRestored(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index f2e6dff0bdd7..fed005360303 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -73,11 +73,11 @@ type ConversationCloseDeps = Pick & { /** What is still queued when the chat closes will not be handed over: a person's message is kept * as a card that waits for the chat's next turn, the rest rejected (`journal-unsent-send-hold.ts`). - * A quit is not a close: the next open settles what it left. The chat stops running, so the - * reopen mark follows (`markStructuredQueueReopen`): on every `close`, or only once it settled a - * send, for a later re-check of the same close, which must never mark past a new send. `which` narrows it to the messages a close that did not complete - * closed. Best effort, so a close never waits on it: resolves false when it failed, reported and - * never thrown. */ + * A quit is not a close: what it left, the next process's startup share keeps as cards. The chat + * stops running, so the reopen mark follows (`markStructuredQueueReopen`): on every `close`, or + * only once it settled a send, for a later re-check of the same close, which must never mark past + * a new send. `which` narrows it to the messages a close that did not complete closed. Best + * effort, so a close never waits on it: resolves false when it failed, reported and never thrown. */ export async function holdClosedStructuredAgentSessionSends( deps: ConversationCloseDeps, sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index edb957ca3172..cca51b41b70f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -65,7 +65,11 @@ export class StructuredAgentSessionHost { }, deliverSettleEdge: (id, journal) => this.conversationDelivery.afterSettleEdge(id, journal), logger: sessionLogger.deferredStructuredAgentSessionLogger(() => this.deps.logger), - onOpened: (sessionId) => this.queued.drain.schedule(sessionId), + onOpened: (sessionId) => { + this.queued.drain.schedule(sessionId) + // A reader holds it now: its settlement goes ahead of the rest of the startup scan. + this.restore.reconciliation.prioritize(sessionId) + }, now: () => this.now() }) private readonly queued = wireQueuedMessages(this.sessions, () => this.mutationContext()) @@ -135,7 +139,6 @@ export class StructuredAgentSessionHost { // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened, onUnopened: (sessionId) => this.tabs.markUnopened(sessionId), - closeAtRest: (sessionId) => this.lifetime.closeAtRest(sessionId), startup: { sessions: this.sessions, tasks: this.tasks, clientDelivery: this.clientDelivery } }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ @@ -162,8 +165,7 @@ export class StructuredAgentSessionHost { open: (sessionId) => this.conversationDelivery.open(sessionId), deliveryActive: (sessionId) => this.conversationDelivery.loop.isRunning(sessionId), closeStatus: (sessionId, options) => this.clientDelivery.closeSession(sessionId, options), - readChildWork: this.clientDelivery.readChildWork, - deferCloseAtRest: (sessionId) => this.restore.reconciliation.deferCloseAtRest(sessionId) + readChildWork: this.clientDelivery.readChildWork }) this.runtimeState.startLeaseRenewal() this.lifetime.idleSweep.start() @@ -291,7 +293,6 @@ export class StructuredAgentSessionHost { wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), stopAgent: (sessionId, ending) => this.lifetime.stopAgent(sessionId, ending), wakeQueuedDrain: (sessionId) => this.queued.drain.schedule(sessionId), - awaitingReopenMark: (sessionId) => this.restore.reconciliation.awaitingReopenMark(sessionId), acquireAborts: this.runtimeState.acquireAborts, optionRevisions: this.runtimeState.optionRevisions, now: () => this.now() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts index f0cf5e13f9f3..f8f2befc1843 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -286,14 +286,14 @@ describe('a stale prompt', () => { if (refused.ok) { throw new Error('expected the answer refused') } - // Plain words: what did not happen, that the agent stopped, and how to go on. + // One sentence: the agent stopped, the answer was not sent, and how to go on. const words = agentSessionWriteNoticeEnglish( agentSessionWriteNoticeParts(agentSessionRefusalFailure(refused.refusal), 'answer', { agentName: 'Codex' }) ) expect(words).toBe( - 'Your answer was not sent. Codex stopped while this response was in progress. You can continue in this conversation.' + 'Codex has stopped, so your answer was not sent. Send a message to continue.' ) expect(current.answerPrompt).not.toHaveBeenCalled() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts index 75c88bcf480e..895f4f39d18b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-live-turn-window.test.ts @@ -116,7 +116,7 @@ describe('a running turn whose record is older than the loaded page', () => { await runLongTurn('turn-1', 250) const state = hydrate() - expect(journal.activeTurnId()).toBe('turn-1') + expect(journal.runningTurn()?.turnId ?? null).toBe('turn-1') expect(state.hasOlder).toBe(true) // What every client read before: the record is off the page, so the loaded rows said idle. expect(activeStructuredAgentSessionTurnId(state.items)).toBeNull() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 509e5a28ccf5..e1f6f40f0b00 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -46,9 +46,6 @@ export type StructuredAgentSessionMutationContext = { * journal commit (a conversation command). Draft-table changes need no call: * the draft store notifies through the journal's own commit listener. */ wakeQueuedDrain?: (sessionId: string) => void - /** The queue sends nothing on its own before the chat's startup reopen mark - * (`StructuredAgentSessionReconciliation.awaitingReopenMark`). */ - awaitingReopenMark?: (sessionId: string) => boolean /** The provider wait each session's serialize is on (a start, an option write), which a caller * outside that serialize aborts. */ acquireAborts: Pick diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts index a92a9d036beb..2d50173ff745 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts @@ -10,6 +10,8 @@ import { AgentSessionPromptUnavailableError } from './structured-agent-session-a import type { StructuredAgentSessionChatStopRun } from './structured-agent-session-chat-stop' import { answerCancelOfSettledPrompt, + dismissEndedGenerationPrompt, + isEndedGenerationPrompt, validatePendingPrompt, type PendingPromptValidation } from './structured-agent-session-prompt-state' @@ -31,6 +33,9 @@ export async function cancelStructuredAgentSessionPrompt( if (!validated.ok) { return answerCancelOfSettledPrompt(ctx, input, validated) } + if (isEndedGenerationPrompt(ctx, validated)) { + return cancelEndedGenerationPrompt(ctx, input, validated, routes.stop) + } const route = ctx.adapter.routePromptCancel?.({ sessionId: ctx.sessionId, prompt: validated.prompt @@ -64,6 +69,26 @@ export async function cancelStructuredAgentSessionPrompt( return dismissed.ok ? cancelled : dismissed } +/** A card an ended generation raised: no live provider holds it, so it is dismissed in the journal + * alone and the live turn is not this card's. A request that also names the live turn (an older + * phone's Stop carries the card it shows) is that turn's Stop as well. */ +async function cancelEndedGenerationPrompt( + ctx: AgentSessionTurnContext, + input: { turnId?: string }, + pending: PendingPrompt, + stop: () => Promise +): Promise { + const dismissed = await dismissEndedGenerationPrompt(ctx, pending) + if (!dismissed.ok) { + return dismissed + } + const liveTurnId = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() + if (input.turnId !== undefined && input.turnId === liveTurnId) { + return (await stop()).outcome + } + return { ok: true, value: { ...(input.turnId ? { turnId: input.turnId } : {}), cancelled: true } } +} + function raisedByLiveTurn(ctx: AgentSessionTurnContext, pending: PendingPrompt): boolean { const live = contextStructuredAgentSessionCurrentWork(ctx).turnScope() const raised = pending.item.turnScope diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts index 8cc9b7e4bc92..288e99521eed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-state.ts @@ -7,7 +7,9 @@ import { type AgentSessionCancelResult, type AgentSessionWireRefusal } from '../../../shared/agent-session-wire' +import { parseAgentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import type { AgentSessionTurnContext, TurnOutcome } from './structured-agent-session-turns' +import { contextStructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' type PendingPromptBody = Extract @@ -91,3 +93,45 @@ export function validatePendingPrompt( } return { ok: true, item, prompt } } + +/** A pending prompt an ended generation raised (`StructuredAgentSessionCurrentWork.isCurrentItem`): + * no live provider holds it, so a Cancel of it reaches no provider. */ +export function isEndedGenerationPrompt( + ctx: Pick, + pending: Extract +): boolean { + return !contextStructuredAgentSessionCurrentWork(ctx).isCurrentItem(pending.item.itemId) +} + +/** Records an ended generation's prompt as cancelled by the caller, in the journal only. */ +export async function dismissEndedGenerationPrompt( + ctx: Pick, + pending: Extract +): Promise> { + const identity = parseAgentJournalItemKey(pending.item.itemId) + if (!identity) { + return { + ok: false, + refusal: refuse( + 'agent_session_operation_invalid', + { reason: 'requestMalformed' }, + `Item id ${pending.item.itemId} is not a well-formed item key.` + ) + } + } + await ctx.journal.appendItem( + identity, + { + ...pending.prompt, + resolution: { + state: 'cancelled', + selectedOptionId: null, + resolvedBy: ctx.resolvedBy, + resolvedAt: ctx.now() + } + }, + // A revision: the prompt keeps the turn it was raised in. + { fence: ctx.fence, turnScope: contextStructuredAgentSessionCurrentWork(ctx).turnScope() } + ) + return { ok: true, value: null } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index a033c718eec4..48a380142842 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -233,9 +233,6 @@ export type QueuedMessageDrainDeps = { currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void - /** Nothing sends on its own yet: the chat's startup reopen mark has not landed. Its landing - * wakes the drain again. */ - held?: (sessionId: string) => boolean logger: StructuredAgentSessionLogger } @@ -262,7 +259,7 @@ export class StructuredAgentSessionQueuedMessageDrain { schedule(sessionId: string): void { const journal = this.disposed ? undefined : this.deps.sessions.get(sessionId)?.journal - if (!journal || this.deps.held?.(sessionId)) { + if (!journal) { return } // Cheap pre-check so token streams do not pay a serialized step per delta. @@ -301,7 +298,7 @@ export class StructuredAgentSessionQueuedMessageDrain { private async step(sessionId: string): Promise { const session = this.deps.sessions.get(sessionId) - if (this.disposed || !session || this.deps.held?.(sessionId)) { + if (this.disposed || !session) { return } const journal = session.journal diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index 2fedb5e9a8f8..1c2984ffa399 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -35,7 +35,6 @@ export function wireStructuredAgentSessionQueuedMessages( currentWork: (sessionId) => hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), - held: (sessionId) => context().awaitingReopenMark?.(sessionId) === true, // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts new file mode 100644 index 000000000000..9fa1b9164d3b --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts @@ -0,0 +1,66 @@ +// The reconciliation worker's own read of a chat nobody has open: replayed outside the chat's lane, +// and never indexed as the chat's conversation, so what its pass writes reaches no reader and no +// status surface. A reader that opens the chat replays it fresh; the worker then uses theirs. + +import { setImmediate as yieldToEventLoop } from 'node:timers/promises' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + readJournalRowsAfter, + readJournalSessionEpoch +} from '../agent-session-journal/journal-row-table' +import { isJournalOpenFailurePermanent } from '../agent-session-journal/journal-open-failure' +import type { StructuredAgentSessionConversationOpenDeps } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' + +export type StructuredAgentSessionReconciliationLoad = + | { kind: 'loaded'; session: StructuredAgentSessionHostSession } + /** Opened by a reader meanwhile, or quitting: nothing read. */ + | { kind: 'skipped' } + /** Nothing owed: no journal (nothing was ever written), or one damaged or written by a newer Orca, + * which no retry reads (reported once here). */ + | { kind: 'nothing' } + +/** Replays a closed chat's journal, for a caller in a background slot. Throws only a failure that + * can clear. `skip`: read once the turn comes, so a reader's open meanwhile wins. */ +export async function loadStructuredAgentSessionForReconciliation( + deps: StructuredAgentSessionConversationOpenDeps, + sessionId: string, + skip: () => boolean +): Promise { + // A replay is synchronous SQLite: a macrotask before each keeps a send or a read from waiting + // behind a whole scan's worth of them. + await yieldToEventLoop() + if (skip()) { + return { kind: 'skipped' } + } + try { + const opened = await restoreStructuredAgentSessionRead(deps, sessionId) + return opened ? { kind: 'loaded', session: opened.session } : { kind: 'nothing' } + } catch (error) { + if (!isJournalOpenFailurePermanent(error)) { + throw error + } + deps.logger.warn("a chat's history cannot be loaded, so nothing is settled", { + scope: 'reconciliation', + sessionId, + error + }) + return { kind: 'nothing' } + } +} + +/** Whether a handle still holds the journal's newest row: another handle (a reader that opened, + * wrote and closed the chat meanwhile) leaves it stale, and a stale handle must not write. */ +export function structuredAgentSessionJournalIsCurrent( + deps: StructuredAgentSessionConversationOpenDeps, + sessionId: string, + journal: Pick +): boolean { + const { db } = deps.journalDatabase + const { epoch, sequence } = journal.cursor() + return ( + readJournalSessionEpoch(db, sessionId) === epoch && + readJournalRowsAfter(db, sessionId, epoch, sequence, 1).length === 0 + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index ca4490428122..734e5840c8f1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -5,8 +5,10 @@ // // The steps stay apart, so one never retires another's debt: // (a) lease-release repair: an exit this host observed whose release write failed; -// (b) the startup share: the cards' repair and prune, an earlier host process's unsent sends kept -// as cards, the queue's reopen mark after them, and a rewind it left in doubt; +// (b) the startup share: the cards' repair and prune, and an earlier host process's unsent sends +// kept as cards; then, best effort, the queue's reopen mark after them and a rewind it left in +// doubt (neither is owed: the pause is derived from where the handle opened, and a rewind only +// its provider can prove waits for the next acquisition); // (c) item settlement (`settleStructuredAgentSessionLeftovers`), one planned batch. // An empty item plan says nothing of (a) or (b). @@ -22,7 +24,10 @@ import { settleStructuredAgentSessionLeftovers, type StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' -import { structuredAgentSessionHostInstance } from './structured-agent-session-queued-pause' +import { + markStructuredQueueReopen, + structuredAgentSessionHostInstance +} from './structured-agent-session-queued-pause' import { recoverStructuredRewind } from './structured-rewind-recovery' export type StructuredAgentSessionReconciliationPassContext = Pick< @@ -37,7 +42,7 @@ export type StructuredAgentSessionReconciliationDebts = { * later release replaces it with its own. */ evidence?: AgentSessionDeathEvidence[] /** This process has not yet finished the chat's startup share. */ - startupShare?: { reopenMarked: boolean; leaseSettled: boolean } + startupShare?: { leaseSettled: boolean } } export type StructuredAgentSessionReconciliationPass = { @@ -45,29 +50,27 @@ export type StructuredAgentSessionReconciliationPass = { failed: unknown[] /** Rows were written, so the next pass verifies nothing is left. */ wrote: boolean - /** The conversation closed before this pass took the lane: nothing was checked. */ - absent?: true } +/** `session`: the chat's open conversation, or the worker's own read of a closed one. */ export async function runStructuredAgentSessionReconciliationPass( context: StructuredAgentSessionReconciliationPassContext, sessionId: string, + session: Pick, debts: StructuredAgentSessionReconciliationDebts ): Promise { const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false } - const session = context.sessions.get(sessionId) - if (!session) { - return { ...pass, absent: true } - } if ( structuredAgentSessionEndedChildHoldsLease(context, sessionId) && (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId)) ) { pass.failed.push(new Error('agent_session_exit_release_owed')) } - if (debts.startupShare) { + // Only once the lease's recovery was decided (the worker's step before the lane): a share signalled + // mid-attempt waits for the next one. + if (debts.startupShare?.leaseSettled) { const failedBefore = pass.failed.length - await runStartupShare(context, sessionId, session, debts.startupShare, pass) + await runStartupShare(context, sessionId, session.journal, pass) // Done only once every step landed; a failed conversion or repair is owed still. if (pass.failed.length === failedBefore) { delete debts.startupShare @@ -124,8 +127,7 @@ function heldProofs( async function runStartupShare( context: StructuredAgentSessionReconciliationPassContext, sessionId: string, - { journal }: StructuredAgentSessionHostSession, - share: NonNullable, + journal: StructuredAgentSessionHostSession['journal'], pass: StructuredAgentSessionReconciliationPass ): Promise { const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence @@ -149,14 +151,20 @@ async function runStartupShare( hold: { cause: 'hostRestarted' } }) ) - // Once, after every card the earlier process left: it starts where this handle opened, so a - // card this process queued is never held by it. - if (converted && !share.reopenMarked) { + // After every card the earlier process left, from where this handle opened, so a card this + // process queued is never held by it. Reported, never owed: the pause starts there anyway. + if (converted) { const opened = journal.openedAt() const since = opened.epoch === journal.cursor().epoch ? opened.sequence + 1 : undefined - share.reopenMarked = await attempt(() => journal.markQueueReopen(fence, since)) + await markStructuredQueueReopen(sessionId, journal, fence, context.deps.logger, since) } - // A rewind the earlier process left prepared refuses every send until settled; one only its - // provider can prove stays for the next acquisition. - await attempt(() => recoverStructuredRewind(context.deps, sessionId, journal, fence)) + // A rewind the earlier process left prepared refuses every send until settled. With no provider + // here, one only its provider can prove stays for the next acquisition, which recovers it. + await recoverStructuredRewind(context.deps, sessionId, journal, fence).catch((error: unknown) => + context.deps.logger.warn('recovering a rewind an earlier process left did not finish', { + scope: 'reconciliation-rewind', + sessionId, + error + }) + ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts new file mode 100644 index 000000000000..f651affba300 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts @@ -0,0 +1,56 @@ +// The reconciliation workers' few background slots, for a chat's replay and its startup recovery +// step: outside every chat's lane, so a send or a read never waits behind the scan. A chat a reader +// opened goes ahead of the rest; a quit ends every wait, and a task it has not started never runs. + +import { PrioritySemaphore } from '../../../shared/priority-semaphore' + +const BACKGROUND_SLOTS = 4 +const URGENT = 0 +const SCAN = 1 + +/** One chat's place in line. */ +export type StructuredAgentSessionReconciliationSlotWaiter = { + /** Goes ahead of the startup scan: a reader opened the chat. */ + urgent: boolean + /** The wait in progress, aborted to requeue it ahead or to quit. */ + slotWait?: AbortController +} + +export class StructuredAgentSessionReconciliationSlots { + private readonly slots = new PrioritySemaphore(BACKGROUND_SLOTS) + + constructor(private readonly disposed: () => boolean) {} + + /** Requeues the chat's wait ahead of the scan. */ + prioritize(waiter: StructuredAgentSessionReconciliationSlotWaiter): void { + if (!waiter.urgent) { + waiter.urgent = true + waiter.slotWait?.abort() + } + } + + /** Runs the task in a slot; null when quit came first, and the task never ran. */ + async run( + waiter: StructuredAgentSessionReconciliationSlotWaiter, + task: () => Promise + ): Promise<{ value: T } | null> { + let release: (() => void) | null = null + while (!release) { + if (this.disposed()) { + return null + } + const wait = new AbortController() + waiter.slotWait = wait + // Aborted to requeue ahead or to quit: the loop decides which. + release = await this.slots + .acquire(waiter.urgent ? URGENT : SCAN, wait.signal) + .catch(() => null) + waiter.slotWait = undefined + } + try { + return this.disposed() ? null : { value: await task() } + } finally { + release() + } + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts index d6f541242851..6d70ee65028e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts @@ -2,13 +2,30 @@ // has written anything. import { afterEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { + AGENT_JOURNAL_THREAD_SCOPE, + AGENT_SESSION_JOURNAL_SCHEMA_VERSION, + type AgentJournalItemIdentity +} from '../../../shared/agent-session-journal-types' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' -import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + liveTestJournalRows, + openTestJournalHostDatabase, + updateTestJournalRowJson +} from '../agent-session-journal/journal-host-database-test-support' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD +} from './structured-agent-session-host-test-data' import { createQueuedMessageTestRig, + eventually, type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' import { currentJournal, @@ -161,3 +178,117 @@ describe('an observed exit whose release write failed', () => { unsubscribe() }) }) + +describe('a journal no retry can load', () => { + it.each([ + ['damaged', '}{'], + ['saved by a newer Orca', null] + ] as const)( + '%s: the startup worker retires instead of replaying it again', + async (_why, json) => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.settleAccepted(await current.workingSend(), 'work') + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let opens = 0 + await current.crashReloadHostProcess(() => { + const { db } = openTestJournalHostDatabase(current.root) + const epochRow = liveTestJournalRows(db, SESSION).find((row) => row.seq === 1)! + const newer = { + ...JSON.parse(epochRow.rowJson), + v: AGENT_SESSION_JOURNAL_SCHEMA_VERSION + 1 + } + updateTestJournalRowJson(db, SESSION, 1, json ?? JSON.stringify(newer)) + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + opens += 1 + return open.call(this) + }) + }) + + expect(worker(current).owes(SESSION)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(opens).toBe(1) + } + ) +}) + +describe("the startup share's reopen mark", () => { + // Bookkeeping, reported and never owed: the pause starts where the handle opened either way. + it.each([false, true])( + 'never holds the queue past the turn that lifts it (its write fails: %s)', + async (failMark) => { + rig = await createQueuedMessageTestRig() + const current = rig + await current.workingSend() + const queued = await current.send('queued before the crash', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + const draftId = queued.value.queued.messageId + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + await current.crashRestartHostProcess(() => { + if (failMark) { + vi.spyOn(AgentSessionJournal.prototype, 'appendQueueReopen').mockRejectedValue( + new Error('persistent storage failure') + ) + } + }) + const journal = currentJournal(current) + // Held from the open, before any mark could land. + expect(structuredQueuePauses(journal).map((pause) => pause.reason)).toEqual(['restarted']) + + const next = current.send('sent after the restart') + await next.result + await current.settleAccepted(next.id, 'next') + + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + await reconciled(current) + } + ) +}) + +describe("an ended generation's closing tail", () => { + it('lands at its own fence after the exit settled, and its worker settles what it left running', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + // A running turn is left, so the exit's settlement writes rows. + await leaveUnfinishedWork(current) + const deadFence = current.store.getRecord(SESSION)?.lease.runtimeFence ?? 0 + await exitChild(current) + await reconciled(current) + const tail = (ordinal: number): AgentJournalItemIdentity => ({ + provider: 'codex', + threadId: THREAD, + turnId: 'late', + ordinal + }) + const late = { fence: deadFence, ownerFence: deadFence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + + // What the closed sink had already taken, delivered after the exit: it lands. + await currentJournal(current).appendItem( + tail(1), + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'final answer' }] }, + late + ) + await currentJournal(current).appendItem( + tail(2), + { kind: 'turn', turnId: 'late', state: 'running' }, + late + ) + + expect(currentJournal(current).itemBody(agentJournalItemKey(tail(1)))).not.toBeNull() + // Below the moved fence, so its commit woke the worker, which settles it by the exit's proof. + await vi.waitFor(() => + expect( + readAgentJournalTurn( + currentJournal(current).itemBody(agentJournalItemKey(tail(2))) ?? undefined + )?.state + ).toBe('interrupted') + ) + await reconciled(current) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts index 0d9b2bda9525..c24d25a1e070 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -7,30 +7,37 @@ // operational revision, the published view, the queued-card drain), then wakes the worker. // // The worker coalesces: a chat has at most one attempt running and one timer waiting, and every -// attempt re-derives everything owed (`runStructuredAgentSessionReconciliationPass`). An attempt -// that finds the chat closed replays its journal in one of a few background slots outside the -// chat's lane, and publishes it; only the publish and the pass take the lane, as separate steps, so -// a send that arrived meanwhile goes first. A failure backs off from 1 s to 30 s, in memory only. -// The worker retires once a pass finds nothing owed and wrote nothing, when the chat's record is -// gone, when the database is read-only, or at shutdown. A conversation it opened itself, or one the -// idle sweep left to it, it closes again on retiring. Nothing a person does (send, start, open, -// Stop, answer) ever waits on it. +// attempt re-derives everything owed (`runStructuredAgentSessionReconciliationPass`). A chat nobody +// has open is replayed in one of a few background slots outside its lane, as the worker's own read +// (`structured-agent-session-reconciliation-load.ts`): never the chat's conversation, so nothing it +// writes is published and nothing it closes was anyone's. Only the pass takes the lane, so a send +// that arrived meanwhile goes first; a chat a reader opens moves ahead of the rest of the scan. +// A failure backs off from 1 s to 30 s, in memory only, and is given up after a bounded run of +// them: the next signal or the next startup derives whatever is still owed again. The worker +// retires once a pass finds nothing owed and wrote nothing, when the chat's record is gone or its +// journal can never load, when the database is read-only, or at shutdown. Nothing a person does +// (send, start, open, Stop, answer) ever waits on it. -import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' -import { PrioritySemaphore } from '../../../shared/priority-semaphore' -import type { OpenedStructuredAgentSessionConversation } from './structured-agent-session-conversation-open' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' -import { restoreStructuredAgentSessionRead } from './structured-agent-session-read-restore' import { runStructuredAgentSessionReconciliationPass, type StructuredAgentSessionReconciliationDebts, type StructuredAgentSessionReconciliationPassContext } from './structured-agent-session-reconciliation-pass' +import { + loadStructuredAgentSessionForReconciliation, + structuredAgentSessionJournalIsCurrent +} from './structured-agent-session-reconciliation-load' +import { + StructuredAgentSessionReconciliationSlots, + type StructuredAgentSessionReconciliationSlotWaiter +} from './structured-agent-session-reconciliation-slots' -const BACKGROUND_SLOTS = 4 export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 30_000 } as const +/** Consecutive failed attempts after which the worker gives up (about three minutes of backoff). */ +export const RECONCILIATION_MAX_FAILED_ATTEMPTS = 10 export type StructuredAgentSessionReconciliationSignal = { /** Proof the lease no longer holds (`AgentSessionGenerationEnd.evidence`). */ @@ -50,17 +57,17 @@ export type StructuredAgentSessionReconciliationContext = serialize: (sessionId: string, task: () => Promise) => Promise /** Lets a quit wait for an attempt already writing. */ track: (operation: Promise) => Promise - /** Indexes and publishes a journal this worker opened. */ - adopt: (sessionId: string, opened: OpenedStructuredAgentSessionConversation) => Promise /** Publishes what is current now and wakes the queued-card drain. */ publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void /** Exits a lease latched in recovery when present-time evidence permits; a no-op otherwise. */ resolveRecovery: (sessionId: string) => Promise - /** Closes a conversation that is only a cache, as the idle sweep does; takes the lane. */ - closeAtRest: (sessionId: string) => Promise } -type ChatWorker = { +/** `done`: a pass found nothing owed, unless a signal came meanwhile; `retire`: nothing this + * worker can do, whatever came meanwhile (a later signal starts a new one). */ +type Outcome = 'done' | 'retire' | 'again' | 'failed' + +type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { debts: StructuredAgentSessionReconciliationDebts running: boolean /** A signal arrived while an attempt ran: the next attempt follows at once. */ @@ -69,15 +76,14 @@ type ChatWorker = { timer: ReturnType | 'soon' | null failures: number attempted: (() => void)[] - /** Closed at rest when this worker retires, if still open: one it opened, or one the idle sweep - * would have closed but for it. */ - closeOnRetire?: StructuredAgentSessionHostSession + /** The worker's own read of the chat while nobody has it open; closed when it retires. */ + loaded?: StructuredAgentSessionHostSession } export class StructuredAgentSessionReconciliation { private readonly workers = new Map() - private readonly slots = new PrioritySemaphore(BACKGROUND_SLOTS) private disposed = false + private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) private readonly unsubscribe: () => void constructor(private readonly context: StructuredAgentSessionReconciliationContext) { @@ -103,7 +109,7 @@ export class StructuredAgentSessionReconciliation { worker.debts.exit = signal.exit } if (signal.startup && !worker.debts.startupShare) { - worker.debts.startupShare = { reopenMarked: false, leaseSettled: false } + worker.debts.startupShare = { leaseSettled: false } } if (worker.running) { worker.dirty = true @@ -121,33 +127,23 @@ export class StructuredAgentSessionReconciliation { } } + /** A reader opened the chat: whatever its worker still waits a slot for goes ahead of the scan. */ + prioritize = (sessionId: string): void => { + const worker = this.workers.get(sessionId) + if (worker) { + this.slots.prioritize(worker) + } + } + /** Resolves once the chat's next attempt finished, or at once when it has no worker. */ attempted = (sessionId: string): Promise => { const worker = this.workers.get(sessionId) return worker ? new Promise((resolve) => worker.attempted.push(resolve)) : Promise.resolve() } - /** The chat's startup share has not yet written its reopen mark: the queue sends nothing on its - * own until it has, so every card the earlier process left waits for a turn first. */ - awaitingReopenMark = (sessionId: string): boolean => { - const share = this.workers.get(sessionId)?.debts.startupShare - return share !== undefined && !share.reopenMarked - } - /** Whether the chat has a worker: something was owed at its last attempt. */ owes = (sessionId: string): boolean => this.workers.has(sessionId) - /** The idle sweep's close of a chat this worker still owes: deferred to its retirement, so the - * worker does not open it again. False when nothing is owed and the close can go ahead. */ - deferCloseAtRest = (sessionId: string): boolean => { - const worker = this.workers.get(sessionId) - if (!worker) { - return false - } - worker.closeOnRetire = this.context.sessions.get(sessionId) - return true - } - /** Resolves once the chat's worker retired, or is waiting out a backoff. */ idle = async (sessionId: string): Promise => { for (;;) { @@ -159,11 +155,13 @@ export class StructuredAgentSessionReconciliation { } } - /** Shutdown: every timer stops; an attempt already writing finishes, tracked. */ + /** Shutdown: every timer stops and every slot wait ends; an attempt already writing finishes, + * tracked, and one that has not started work starts none. */ dispose = (): void => { this.disposed = true this.unsubscribe() for (const [sessionId, worker] of this.workers) { + worker.slotWait?.abort() this.retire(sessionId, worker) } } @@ -171,7 +169,15 @@ export class StructuredAgentSessionReconciliation { private workerFor(sessionId: string): ChatWorker { let worker = this.workers.get(sessionId) if (!worker) { - worker = { debts: {}, running: false, dirty: false, timer: null, failures: 0, attempted: [] } + worker = { + debts: {}, + running: false, + dirty: false, + timer: null, + failures: 0, + attempted: [], + urgent: false + } this.workers.set(sessionId, worker) } return worker @@ -190,111 +196,140 @@ export class StructuredAgentSessionReconciliation { .then((outcome) => this.afterAttempt(sessionId, worker, outcome)) } - private async runAttempt( - sessionId: string, - worker: ChatWorker - ): Promise<'retire' | 'again' | 'failed'> { + private async runAttempt(sessionId: string, worker: ChatWorker): Promise { try { const { store } = this.context.deps if (this.disposed || store.readOnly || !store.getRecord(sessionId)) { return 'retire' } const share = worker.debts.startupShare - // Once per startup share, before it publishes: a lease latched in recovery is decided first, - // as the restore decides it, so no reader sees a gone owner's turn as running. + // Once per startup share, before its pass: a lease latched in recovery is decided first, as + // the restore decides it, so the pass judges what is current. if (share && !share.leaseSettled) { - await this.inSlot(() => this.context.resolveRecovery(sessionId)).catch((error: unknown) => - this.warn(sessionId, error) + const resolved = await this.slots.run(worker, () => + this.context.resolveRecovery(sessionId).catch((error: unknown) => { + this.warn(sessionId, error) + }) ) + if (!resolved) { + return 'retire' + } share.leaseSettled = true } - // Replayed in a background slot outside the chat's lane, and published before the pass: a - // reader or a send never waits on the replay, and the lane is held only to publish and for - // the pass's recheck and writes. - if (!this.context.sessions.has(sessionId) && !(await this.load(sessionId, worker))) { - return 'retire' - } - const held = this.awaitingReopenMark(sessionId) - const pass = await this.context.serialize(sessionId, () => - runStructuredAgentSessionReconciliationPass(this.context, sessionId, worker.debts) - ) - if (held && !this.awaitingReopenMark(sessionId)) { - // The queue may send on its own again. - this.context.publishGenerationEnded(sessionId) - } - // Closed while this attempt waited for the lane: the next one opens it again. - if (pass.absent) { - return 'again' - } - if (pass.failed.length > 0) { - this.warn(sessionId, pass.failed[0]) - return 'failed' + if (!this.context.sessions.has(sessionId) && !worker.loaded) { + const stop = await this.load(sessionId, worker) + if (stop) { + return stop + } } - return pass.wrote ? 'again' : 'retire' + return await this.context.serialize(sessionId, () => this.runPass(sessionId, worker)) } catch (error) { this.warn(sessionId, error) return 'failed' } } - private warn(sessionId: string, error: unknown): void { - this.context.deps.logger.warn("settling a gone agent's leftover work did not finish", { - scope: 'reconciliation', + /** The pass, inside the chat's lane, on the conversation a reader holds or the worker's own. */ + private async runPass(sessionId: string, worker: ChatWorker): Promise { + if (this.disposed) { + return 'retire' + } + const session = this.sessionFor(sessionId, worker) + // Closed, or written by another handle, since this attempt began: the next one reads it again. + if (!session) { + return 'again' + } + const pass = await runStructuredAgentSessionReconciliationPass( + this.context, sessionId, - error - }) + session, + worker.debts + ) + if (pass.failed.length > 0) { + this.warn(sessionId, pass.failed[0]) + return 'failed' + } + return pass.wrote ? 'again' : 'done' } - private async inSlot(task: () => Promise): Promise { - const release = await this.slots.acquire(0) - try { - return await task() - } finally { - release() + private sessionFor( + sessionId: string, + worker: ChatWorker + ): Pick | undefined { + const open = this.context.sessions.get(sessionId) + if (open) { + this.dropLoaded(worker) + return open } + const { loaded } = worker + if ( + loaded && + !structuredAgentSessionJournalIsCurrent(this.context.deps, sessionId, loaded.journal) + ) { + this.dropLoaded(worker) + return undefined + } + return loaded } - /** Replays the journal in a background slot, then publishes it; false when it has none. */ - private async load(sessionId: string, worker: ChatWorker): Promise { - const opened = await this.inSlot(async () => { - // A replay is synchronous SQLite: a macrotask before each keeps a send or a read from waiting - // behind a whole scan's worth of them. - await yieldToEventLoop() - return this.context.sessions.has(sessionId) - ? null - : restoreStructuredAgentSessionRead(this.context.deps, sessionId) - }) - if (!opened) { - // Open already, or no journal: nothing was ever written, so nothing is owed. - return this.context.sessions.has(sessionId) - } - const replayed = opened - // Published under the lane, so it never replaces a conversation a reader or a send opened - // meanwhile; theirs is the conversation, and this replay is dropped. - await this.context.serialize(sessionId, async () => { - if (this.context.sessions.has(sessionId)) { - await replayed.session.journal.close() - return + /** Replays the journal in a background slot as the worker's own read; null to go on to the pass, + * else how the attempt ends. */ + private async load(sessionId: string, worker: ChatWorker): Promise { + const result = await this.slots.run(worker, () => + loadStructuredAgentSessionForReconciliation( + this.context.deps, + sessionId, + () => this.disposed || this.context.sessions.has(sessionId) + ) + ) + const loaded = result?.value + if (!loaded || this.disposed) { + if (loaded?.kind === 'loaded') { + void loaded.session.journal.close() } - await this.context.adopt(sessionId, replayed) - worker.closeOnRetire = this.context.sessions.get(sessionId) + return 'retire' + } + if (loaded.kind === 'loaded') { + worker.loaded = loaded.session + } + // Skipped: a reader opened it meanwhile, and the pass uses their conversation. + return loaded.kind === 'nothing' ? 'retire' : null + } + + private dropLoaded(worker: ChatWorker): void { + const journal = worker.loaded?.journal + worker.loaded = undefined + void journal?.close() + } + + private warn(sessionId: string, error: unknown): void { + this.context.deps.logger.warn("settling a gone agent's leftover work did not finish", { + scope: 'reconciliation', + sessionId, + error }) - return true } - private afterAttempt( - sessionId: string, - worker: ChatWorker, - outcome: 'retire' | 'again' | 'failed' - ): void { + private afterAttempt(sessionId: string, worker: ChatWorker, outcome: Outcome): void { worker.running = false const attempted = worker.attempted.splice(0) attempted.forEach((resolve) => resolve()) if (this.workers.get(sessionId) !== worker || this.disposed) { + // Retired while this attempt ran: its own read closes now that nothing writes through it. + this.dropLoaded(worker) return } if (outcome === 'failed') { worker.failures += 1 + if (worker.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { + this.context.deps.logger.warn("gave up settling a gone agent's leftover work for now", { + scope: 'reconciliation', + sessionId, + error: new Error(`${worker.failures} attempts failed`) + }) + this.retire(sessionId, worker) + return + } const delay = Math.min( RECONCILIATION_BACKOFF_MS.max, RECONCILIATION_BACKOFF_MS.first * 2 ** (worker.failures - 1) @@ -303,17 +338,11 @@ export class StructuredAgentSessionReconciliation { return } worker.failures = 0 - if (outcome === 'again' || worker.dirty) { + if (outcome === 'again' || (outcome === 'done' && worker.dirty)) { this.schedule(sessionId, worker, 0) return } this.retire(sessionId, worker) - // Only while it is still that conversation: a reader's reopen is theirs. - if (worker.closeOnRetire && this.context.sessions.get(sessionId) === worker.closeOnRetire) { - void this.context - .closeAtRest(sessionId) - .catch((error: unknown) => this.warn(sessionId, error)) - } } private schedule(sessionId: string, worker: ChatWorker, delay: number): void { @@ -335,6 +364,7 @@ export class StructuredAgentSessionReconciliation { worker.timer = null this.workers.delete(sessionId) if (!worker.running) { + this.dropLoaded(worker) worker.attempted.splice(0).forEach((resolve) => resolve()) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts index 25b7cd85a6f0..bc88fe1c4152 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-resume.test.ts @@ -42,17 +42,7 @@ function markersAtTeardown( } ) { return [...input.sessions].flatMap(([sessionId, session]) => { - // Teardown runs while this host's child still holds the lease. - const getRecord = (id: string) => { - const held = input.getRecord(id) - return held && { ...held, lease: { ...held.lease, claimStatus: 'live' as const } } - } - const recorded = structuredAgentSessionWorkingAtStop({ - ...input, - getRecord, - sessionId, - session - }) + const recorded = structuredAgentSessionWorkingAtStop({ ...input, sessionId, session }) return recorded ? [recorded] : [] }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index a846c5a1628a..d45feb519a58 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -74,7 +74,6 @@ export function createStructuredAgentSessionHostRestore( tasks: Pick clientDelivery: Pick } - closeAtRest: StructuredAgentSessionReconciliationContext['closeAtRest'] } ): { reconcileRestartLeases: () => Promise @@ -83,7 +82,7 @@ export function createStructuredAgentSessionHostRestore( restoreReadableSessions: (sessionIds?: readonly string[]) => Promise reconciliation: StructuredAgentSessionReconciliation } { - const { reconcileLeases, resolveRecovery, startup, closeAtRest, ...rest } = wiring + const { reconcileLeases, resolveRecovery, startup, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) const reconcile = createReaderReconcile(reconcileLeases, failures) const reconciliation = new StructuredAgentSessionReconciliation({ @@ -93,15 +92,11 @@ export function createStructuredAgentSessionHostRestore( serialize: rest.serialize, // Tracked like a start, so a quit waits for an attempt before it closes what it writes to. track: (operation) => startup.tasks.trackAttach(operation), - adopt: async (sessionId, opened) => { - await rest.onReadable(sessionId, opened) - }, publishGenerationEnded: (sessionId, options) => startup.clientDelivery.publishGenerationEnded(sessionId, options), - // A lease latched in recovery is decided before the share publishes the chat, as the restore - // decides it; a failure is the worker's to report, and the next attach or send resolves it. - resolveRecovery, - closeAtRest + // A lease latched in recovery is decided before the share's pass, as the restore decides it; + // a failure is the worker's to report, and the next attach or send resolves it. + resolveRecovery }) let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts index 677963ab3f9b..b6e7cdb4d3c5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-settlement.test.ts @@ -1,9 +1,12 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' +import { + StructuredAgentSessionSendSettlement, + type SendSettlementJournal +} from './structured-agent-session-send-settlement' -function journal(dispatchState: 'pending' | 'accepted' | 'unknown'): AgentSessionJournal { +function journal(dispatchState: 'pending' | 'accepted' | 'unknown'): SendSettlementJournal { return { + activeTurnId: () => null, cursor: () => ({ epoch: 'epoch-1', sequence: dispatchState === 'pending' ? 1 : 2 }), submissions: () => [ { @@ -17,21 +20,19 @@ function journal(dispatchState: 'pending' | 'accepted' | 'unknown'): AgentSessio resolvedAt: dispatchState === 'pending' ? null : 2 } ] - } as AgentSessionJournal + } } -function emptyJournal(): AgentSessionJournal { +function emptyJournal(): SendSettlementJournal { return { + activeTurnId: () => null, cursor: () => ({ epoch: 'epoch-1', sequence: 2 }), submissions: () => [] - } as unknown as AgentSessionJournal + } } /** A message accepted and not yet handed over, while `activeTurnId` names the running turn. */ -function queuedJournal( - activeTurnId: string | null, - handedOver = false -): Pick { +function queuedJournal(activeTurnId: string | null, handedOver = false): SendSettlementJournal { return { cursor: () => ({ epoch: 'epoch-1', sequence: handedOver ? 3 : 2 }), activeTurnId: () => activeTurnId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts index 9414a69b9fc4..cf49ea0b7934 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-failure-exit.test.ts @@ -168,8 +168,8 @@ describe('a provider that ends before it finished starting', () => { clientMessageId: 'compact-1', state: 'rejected', reason: text, - // At the fence the exit's release moved to. - fence: 8, + // At the exited child's own fence, before its release moves it. + fence: 7, recovered: true }) ], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts new file mode 100644 index 000000000000..29d6c7e4df79 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts @@ -0,0 +1,219 @@ +// The startup scan's reconciliation workers over their whole life: what a quit cuts short, what +// they never publish or close, and what ends a retry that cannot succeed. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import type { AgentSessionStatusEvent } from '../../../shared/agent-session-wire' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { + openScanHost, + scanIdle as idle, + scanRecord as record, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined +const warnings: string[] = [] + +function openHost(options: Parameters[2] = {}): StructuredAgentSessionHost { + const logger = createStructuredAgentSessionLogger() + host = openScanHost(root, store, { + logger: { ...logger, warn: (message) => warnings.push(message) }, + ...options + }) + return host +} + +async function seed(sessionIds: readonly string[], extra: Parameters[2] = {}) { + await seedTestAgentSessionRecordStore(root, { + records: sessionIds.map((id) => record(id, true, extra)) + }) + for (const sessionId of sessionIds) { + await seedScanJournal(root, sessionId) + } + store = await openTestAgentSessionRecordStore(root) +} + +/** Counts every journal open from now on. */ +function countOpens(): { opens: number } { + const counter = { opens: 0 } + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + counter.opens += 1 + return open.call(this) + }) + return counter +} + +beforeEach(async () => { + warnings.length = 0 + root = await mkdtemp(join(tmpdir(), 'orca-startup-scan-lifetime-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +describe('a quit during the startup scan', () => { + it('waits only for work already started: no chat still waiting for a slot opens or recovers', async () => { + const chats = Array.from( + { length: 10 }, + (_, index) => `chat-q${String(index).padStart(7, '0')}` + ) + await seed(chats) + const current = openHost() + let release: () => void = () => undefined + const gate = new Promise((resolve) => (release = resolve)) + const { runtimeState } = current.collaboratorsForTests() + const resolve = runtimeState.resolveRecovery.bind(runtimeState) + let recoveries = 0 + vi.spyOn(runtimeState, 'resolveRecovery').mockImplementation(async (sessionId) => { + recoveries += 1 + await gate + return resolve(sessionId) + }) + await current.reconcileRestartLeases() + await vi.waitFor(() => expect(recoveries).toBe(4)) + const counter = countOpens() + + const quit = current.flushAllStreamedEvents() + release() + await quit + host = undefined + + // The four already recovering finished that step and went no further; the six still waiting + // for a slot never started. + expect(recoveries).toBe(4) + expect(counter.opens).toBe(0) + }) +}) + +describe('a chat the startup scan settles', () => { + it('is published to no status surface unless a reader or the restorer opened it', async () => { + const visible = 'chat-visible1' + const hidden = ['chat-hidden01', 'chat-hidden02', 'chat-hidden03'] + await seed([visible, ...hidden]) + const published: string[] = [] + const current = openHost({ + statusSink: { + publish: (summary) => published.push(summary.sessionId), + forget: () => undefined + } + }) + await current.reconcileRestartLeases() + await current.restoreReadableSessions([visible]) + await current.startupSettled() + await idle(current, [visible, ...hidden]) + + const snapshots: string[][] = [] + const unsubscribe = current.subscribeStatus({ + id: 'renderer', + emit: (event: AgentSessionStatusEvent) => { + if (event.type === 'snapshot') { + snapshots.push(event.sessions.map((session) => session.sessionId)) + } + } + }) + unsubscribe() + expect(new Set(published)).toEqual(new Set([visible])) + expect(snapshots.at(-1)).toEqual([visible]) + // Settled all the same. + expect(current.collaboratorsForTests().sessions.has('chat-hidden01')).toBe(false) + expect(await turnState(current, 'chat-hidden01')).toBe('unverifiable') + }) + + it('stays open when the restorer opened it while the scan was settling it', async () => { + await seed([CHAT]) + const current = openHost() + const { sessions } = current.collaboratorsForTests() + // The restore lands between the scan's read and its pass. + let restore: Promise | null = null + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + await open.call(this) + restore ??= current.restoreReadableSessions([CHAT]) + }) + await current.reconcileRestartLeases() + await current.startupSettled() + await restore + await idle(current, [CHAT]) + await new Promise((resolve) => setTimeout(resolve, 20)) + + expect(sessions.has(CHAT)).toBe(true) + expect(await turnState(current, CHAT)).toBe('unverifiable') + }) +}) + +describe('a startup worker whose step cannot succeed', () => { + it('retires at once on a journal no retry can load, and replays it once', async () => { + await seed([CHAT]) + // The last row is one no build wrote. + openTestJournalHostDatabase(root) + .db.prepare( + `UPDATE journal_rows SET row_json = '{"garbage":true}' WHERE session_id = ? AND seq = (SELECT MAX(seq) FROM journal_rows WHERE session_id = ?)` + ) + .run(CHAT, CHAT) + const counter = countOpens() + const current = openHost() + await current.reconcileRestartLeases() + await current.startupSettled() + + expect(current.collaboratorsForTests().reconciliation.owes(CHAT)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(counter.opens).toBe(1) + expect(warnings).toContain("a chat's history cannot be loaded, so nothing is settled") + }) + + it('leaves a rewind only its provider can prove for the next acquisition, and retires', async () => { + const rewind = { + operationId: 'op-rewind', + callerKey: 'caller', + itemId: agentJournalItemKey({ + provider: 'codex', + threadId: 'thread', + turnId: 'turn-1', + ordinal: 0 + }), + expectedEpoch: 'epoch-x', + phase: 'prepared' as const, + retained: [] + } + await seed([CHAT], { rewind }) + const current = openHost() + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [CHAT]) + + expect(store.getRecord(CHAT)?.rewind?.phase).toBe('prepared') + expect(current.collaboratorsForTests().sessions.has(CHAT)).toBe(false) + expect(warnings).toEqual(['recovering a rewind an earlier process left did not finish']) + expect(await turnState(current, CHAT)).toBe('unverifiable') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts new file mode 100644 index 000000000000..c3e7cad4e159 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts @@ -0,0 +1,162 @@ +// Chats an earlier process left, for the startup-scan suites: records on disk, each journal with a +// running turn at fence 13, and a host over them with no provider. + +import { vi, expect } from 'vitest' +import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' +import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' +import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' +import { + createStructuredAgentSessionLogger, + type StructuredAgentSessionLogger +} from './structured-agent-session-logger' +import type { StructuredAgentSessionStatusSink } from './structured-agent-session-status-ownership' + +export const SCAN_NOW = 1_800_000_000_000 +export const SCAN_CHATS = [ + 'chat-aaaaaaa1', + 'chat-aaaaaaa2', + 'chat-aaaaaaa3', + 'chat-aaaaaaa4', + 'chat-aaaaaaa5' +] +export const SCAN_LATE = 'chat-aaaaaaa6' +/** Released long ago with nothing proving its owner gone: no restart moves its fence. */ +export const SCAN_RELEASED = 'chat-released1' + +export function scanRecord( + sessionId: string, + released: boolean, + extra: Partial = {} +): AgentSessionRecord { + const linkId = `${sessionId}-link` + return { + schemaVersion: 2, + sessionId, + location: LOCATION, + provider: 'claude', + providerHandleChain: [ + { + linkId, + handle: claudeProviderHandle(`provider-${sessionId}`, null), + origin: 'created', + mintedAtFence: 13, + observedAt: SCAN_NOW - 60_000 + } + ], + accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, + createdAt: SCAN_NOW - 60_000, + updatedAt: SCAN_NOW, + lease: { + sessionId, + runtimeKind: 'native', + runtimeFence: released ? 14 : 13, + handoffStage: null, + provenHandleLinkId: linkId, + ownerProcess: released + ? null + : { + hostId: 'local', + pid: 12_000, + processStartTimeMs: SCAN_NOW - 60_000, + spawnToken: 'spawn' + }, + reservedSpawnToken: released ? null : 'spawn', + leaseDeadlineAt: SCAN_NOW + 30_000, + lastRenewedAt: SCAN_NOW, + handoffOperationId: null, + journalCheckpoint: null, + claimKeyId: 'key-1', + claimStatus: released ? 'released' : 'live', + unreconciled: false, + deathEvidence: null + }, + ...extra + } +} + +/** The chat's journal as the crashed process left it: a running turn at fence 13, and, when + * asked, a send it accepted and never handed over. */ +export async function seedScanJournal( + root: string, + sessionId: string, + options: { queued?: true } = {} +): Promise { + const journal = await openAgentSessionJournal({ + identity: { + sessionId, + workspaceId: LOCATION.workspaceId, + hostId: LOCATION.executionHostId, + agent: 'claude', + providerHandle: claudeProviderHandle(`provider-${sessionId}`, null) + }, + database: openTestJournalHostDatabase(root), + now: () => SCAN_NOW + }) + await journal.appendItem( + { provider: 'claude', sessionId: `provider-${sessionId}`, uuid: 'turn' }, + { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: SCAN_NOW - 30_000 }, + { fence: 13, turnScope: AGENT_JOURNAL_THREAD_SCOPE } + ) + if (options.queued) { + await journal.appendSubmission({ + clientMessageId: `${sessionId}-queued`, + payloadFingerprint: '0'.repeat(64), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, + fence: 14, + handoverRecorded: true, + origin: 'client', + source: { kind: 'user' } + }) + } + await journal.close() +} + +export function openScanHost( + root: string, + store: AgentSessionRecordStore, + options: { + logger?: StructuredAgentSessionLogger + statusSink?: StructuredAgentSessionStatusSink + } = {} +): StructuredAgentSessionHost { + return new StructuredAgentSessionHost({ + agents: NO_STRUCTURED_AGENTS, + logger: options.logger ?? createStructuredAgentSessionLogger(), + store, + adapter: { + acquire: vi.fn(), + dispatch: vi.fn(), + cancelTurn: vi.fn(), + answerPrompt: vi.fn(), + setOption: vi.fn(), + supportsCreate: () => true + }, + journalDatabase: openTestJournalHostDatabase(root), + claimKeyId: 'key-1', + ...(options.statusSink ? { statusSink: options.statusSink } : {}), + probeOwner: async () => ({ outcome: 'pid-absent' }), + now: () => SCAN_NOW + }) +} + +/** The turn's state as a reader sees it; the read opens the chat. */ +export async function scanTurnState(current: StructuredAgentSessionHost, sessionId: string) { + return (await current.journalSnapshot(sessionId)).items + .map((item) => readAgentJournalTurn(item.body)) + .find(Boolean)?.state +} + +export async function scanIdle(current: StructuredAgentSessionHost, sessionIds: readonly string[]) { + const { reconciliation } = current.collaboratorsForTests() + await vi.waitFor( + () => expect(sessionIds.filter((sessionId) => reconciliation.owes(sessionId))).toEqual([]), + { timeout: 10_000 } + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts index fecf478fd3e6..965cc5cd019d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts @@ -5,10 +5,6 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { AGENT_JOURNAL_THREAD_SCOPE } from '../../../shared/agent-session-journal-types' -import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { openTestAgentSessionRecordStore, @@ -19,129 +15,30 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' -import { StructuredAgentSessionHost } from './structured-agent-session-host' -import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' -import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' - -const NOW = 1_800_000_000_000 -const CHATS = ['chat-aaaaaaa1', 'chat-aaaaaaa2', 'chat-aaaaaaa3', 'chat-aaaaaaa4', 'chat-aaaaaaa5'] -const LATE = 'chat-aaaaaaa6' -/** Released long ago with nothing proving its owner gone: no restart moves its fence. */ -const RELEASED = 'chat-released1' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + SCAN_CHATS as CHATS, + SCAN_LATE as LATE, + SCAN_RELEASED as RELEASED, + scanIdle as idle, + scanRecord as record, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' let root: string let store: AgentSessionRecordStore let host: StructuredAgentSessionHost | undefined -function record(sessionId: string, released: boolean): AgentSessionRecord { - const linkId = `${sessionId}-link` - return { - schemaVersion: 2, - sessionId, - location: LOCATION, - provider: 'claude', - providerHandleChain: [ - { - linkId, - handle: claudeProviderHandle(`provider-${sessionId}`, null), - origin: 'created', - mintedAtFence: 13, - observedAt: NOW - 60_000 - } - ], - accountHome: { variable: 'CLAUDE_CONFIG_DIR', path: '/home/dev/.claude' }, - createdAt: NOW - 60_000, - updatedAt: NOW, - lease: { - sessionId, - runtimeKind: 'native', - runtimeFence: released ? 14 : 13, - handoffStage: null, - provenHandleLinkId: linkId, - ownerProcess: released - ? null - : { hostId: 'local', pid: 12_000, processStartTimeMs: NOW - 60_000, spawnToken: 'spawn' }, - reservedSpawnToken: released ? null : 'spawn', - leaseDeadlineAt: NOW + 30_000, - lastRenewedAt: NOW, - handoffOperationId: null, - journalCheckpoint: null, - claimKeyId: 'key-1', - claimStatus: released ? 'released' : 'live', - unreconciled: false, - deathEvidence: null - } - } -} - -/** The chat's journal as the crashed process left it: a running turn at fence 13, and, when - * asked, a send it accepted and never handed over. */ -async function seedJournal(sessionId: string, options: { queued?: true } = {}): Promise { - const journal = await openAgentSessionJournal({ - identity: { - sessionId, - workspaceId: LOCATION.workspaceId, - hostId: LOCATION.executionHostId, - agent: 'claude', - providerHandle: claudeProviderHandle(`provider-${sessionId}`, null) - }, - database: openTestJournalHostDatabase(root), - now: () => NOW - }) - await journal.appendItem( - { provider: 'claude', sessionId: `provider-${sessionId}`, uuid: 'turn' }, - { kind: 'turn', turnId: 'turn-1', state: 'running', startedAt: NOW - 30_000 }, - { fence: 13, turnScope: AGENT_JOURNAL_THREAD_SCOPE } - ) - if (options.queued) { - await journal.appendSubmission({ - clientMessageId: `${sessionId}-queued`, - payloadFingerprint: '0'.repeat(64), - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, - fence: 14, - handoverRecorded: true, - origin: 'client', - source: { kind: 'user' } - }) - } - await journal.close() -} - function openHost(): StructuredAgentSessionHost { - host = new StructuredAgentSessionHost({ - agents: NO_STRUCTURED_AGENTS, - logger: createStructuredAgentSessionLogger(), - store, - adapter: { - acquire: vi.fn(), - dispatch: vi.fn(), - cancelTurn: vi.fn(), - answerPrompt: vi.fn(), - setOption: vi.fn(), - supportsCreate: () => true - }, - journalDatabase: openTestJournalHostDatabase(root), - claimKeyId: 'key-1', - probeOwner: async () => ({ outcome: 'pid-absent' }), - now: () => NOW - }) + host = openScanHost(root, store) return host } -async function turnState(current: StructuredAgentSessionHost, sessionId: string) { - return (await current.journalSnapshot(sessionId)).items - .map((item) => readAgentJournalTurn(item.body)) - .find(Boolean)?.state -} - -async function idle(current: StructuredAgentSessionHost, sessionIds: readonly string[]) { - const { reconciliation } = current.collaboratorsForTests() - await vi.waitFor( - () => expect(sessionIds.filter((sessionId) => reconciliation.owes(sessionId))).toEqual([]), - { timeout: 10_000 } - ) +/** Whose journal a call ran on: the scan's own read is no conversation the host indexed. */ +function journalSession(journal: AgentSessionJournal): string { + return journal.snapshot().sessionId } beforeEach(async () => { @@ -157,11 +54,11 @@ afterEach(async () => { }) describe('the startup scan', () => { - it('never puts a chat late in the scan behind its own share: a read and the lane go first', async () => { + it('never puts a chat late in the scan behind its own share, and settles it without publishing it', async () => { const all = [...CHATS, LATE] await seedTestAgentSessionRecordStore(root, { records: all.map((id) => record(id, false)) }) for (const sessionId of all) { - await seedJournal(sessionId) + await seedScanJournal(root, sessionId) } store = await openTestAgentSessionRecordStore(root) // The first opens the scan makes hold every background slot until the test lets them go. @@ -179,11 +76,11 @@ describe('the startup scan', () => { return open.call(this) }) const current = openHost() - const events: string[] = [] + const indexed: string[] = [] const { sessions, serialize } = current.collaboratorsForTests() const set = sessions.set.bind(sessions) vi.spyOn(sessions, 'set').mockImplementation((sessionId, session) => { - events.push(`publish:${sessionId}`) + indexed.push(sessionId) return set(sessionId, session) }) const shares: string[] = [] @@ -192,9 +89,7 @@ describe('the startup scan', () => { this: AgentSessionJournal, ...args ) { - const owner = [...sessions].find(([, session]) => session.journal === this)?.[0] - shares.push(owner ?? 'unknown') - events.push(`share:${owner}`) + shares.push(journalSession(this)) return mark.apply(this, args) }) @@ -214,20 +109,53 @@ describe('the startup scan', () => { await current.startupSettled() await idle(current, all) expect(order).toEqual(['lane', 'read']) + expect(new Set(shares)).toEqual(new Set(all)) + // Only the chat a reader opened is a conversation; the scan's own reads published nothing. + expect(indexed).toEqual([LATE]) for (const sessionId of all) { expect(await turnState(current, sessionId)).toBe('interrupted') - // Published before its share wrote anything. - expect(events.indexOf(`publish:${sessionId}`)).toBeGreaterThanOrEqual(0) - expect(events.indexOf(`publish:${sessionId}`)).toBeLessThan( - events.indexOf(`share:${sessionId}`) - ) } - expect(new Set(shares)).toEqual(new Set(all)) + }) + + it('takes a chat a reader opens ahead of the rest of the scan', async () => { + const all = [...CHATS, LATE] + // Released: the restart reconcile ends nothing, so each chat's first attempt is its share. + await seedTestAgentSessionRecordStore(root, { records: all.map((id) => record(id, true)) }) + for (const sessionId of all) { + await seedScanJournal(root, sessionId) + } + store = await openTestAgentSessionRecordStore(root) + const current = openHost() + // The first four chats' recovery holds every background slot until the test lets it go. + let release: () => void = () => undefined + const gate = new Promise((resolve) => (release = resolve)) + const { runtimeState } = current.collaboratorsForTests() + const resolve = runtimeState.resolveRecovery.bind(runtimeState) + const resolved: string[] = [] + vi.spyOn(runtimeState, 'resolveRecovery').mockImplementation(async (sessionId) => { + resolved.push(sessionId) + if (resolved.length <= 4) { + await gate + } + return resolve(sessionId) + }) + + await current.reconcileRestartLeases() + await vi.waitFor(() => expect(resolved).toHaveLength(4)) + expect(resolved).not.toContain(LATE) + // The person opens the late chat while two chats still wait for a slot, it among them. + expect(await turnState(current, LATE)).toBe('running') + + release() + await current.startupSettled() + await idle(current, all) + expect(resolved.slice(4)).toEqual([LATE, CHATS[4]]) + expect(await turnState(current, LATE)).toBe('unverifiable') }) it('finds what an earlier process left with no fence move and no stored mark, and keeps its send as a card', async () => { await seedTestAgentSessionRecordStore(root, { records: [record(RELEASED, true)] }) - await seedJournal(RELEASED, { queued: true }) + await seedScanJournal(root, RELEASED, { queued: true }) store = await openTestAgentSessionRecordStore(root) const current = openHost() const ended = vi.fn() @@ -251,7 +179,7 @@ describe('the startup scan', () => { it('retries a conversion that failed instead of marking it done', async () => { await seedTestAgentSessionRecordStore(root, { records: [record(RELEASED, true)] }) - await seedJournal(RELEASED, { queued: true }) + await seedScanJournal(root, RELEASED, { queued: true }) store = await openTestAgentSessionRecordStore(root) vi.spyOn(console, 'warn').mockImplementation(() => undefined) const database = openTestJournalHostDatabase(root) @@ -265,14 +193,14 @@ describe('the startup scan', () => { await current.reconcileRestartLeases() await current.startupSettled() - const { reconciliation, sessions } = current.collaboratorsForTests() + const { reconciliation } = current.collaboratorsForTests() expect(reconciliation.owes(RELEASED)).toBe(true) - const journal = sessions.get(RELEASED)!.journal - expect(journal.queuedMessages.list()).toEqual([]) database.db.exec('DROP TRIGGER reject_conversion') await idle(current, [RELEASED]) + expect(await turnState(current, RELEASED)).toBe('unverifiable') + const { journal } = current.collaboratorsForTests().sessions.get(RELEASED)! expect(journal.submission(`${RELEASED}-queued`)).toMatchObject({ dispatchState: 'rejected' }) expect(journal.queuedMessages.list()).toHaveLength(1) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts index 6dfe2f58fa30..ee3607bc6f0a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -4,9 +4,11 @@ // // Each chat goes to its reconciliation worker with its startup share owed // (`structured-agent-session-reconciliation-pass.ts`): the worker takes a background slot outside -// the chat's action lane, opens and publishes the journal, and enters the lane only for the short +// the chat's action lane, replays a closed chat's journal as its own read (published to no status +// surface: only a chat a reader or the restorer opens is), and enters the lane only for the short // recheck and its writes. So a send, a start or a read of a chat late in the scan never waits -// behind its share. A share that fails is not done: the worker retries it. +// behind its share, and a chat a reader opens moves ahead of the rest. A share that fails is not +// done: the worker retries it, up to its bound. import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionReconciliation } from './structured-agent-session-reconciliation-worker' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts index 2a91867ae446..56a1da6f5dd1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts @@ -248,7 +248,7 @@ describe('a Stop pressed before its send opened a turn binds only the turn it st expect(laterTurnEndRows()).toEqual([ expect.objectContaining({ state: 'interrupted', outcome: 'cancellation' }) ]) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() }) it('binds no turn that opens after a Stop that stopped nothing', async () => { @@ -346,7 +346,7 @@ describe("a Stop's settle that ends the turn its interrupt took", () => { expect(await rig.stop()).toMatchObject({ ok: true, value: { cancelled: true } }) expect(await laterTurn()).toMatchObject({ state: 'interrupted', outcome: 'cancellation' }) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts index 17a274d6f5fc..9bba98fcf3da 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-entries.test.ts @@ -269,7 +269,7 @@ describe('every Stop entry writes its event, with its reason, before it ends the { kind: 'turn', turnId: 'turn-1', state: 'interrupted', completedAt: Date.now() }, { fence: 1, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() const sink = rig.host['runtimeState'].eventSinkFor(HOST_TEST_SESSION) const drained = sink.drained.bind(sink) vi.spyOn(sink, 'drained') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts index 50b25503d7ec..17c9350dd750 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-restart.test.ts @@ -364,7 +364,7 @@ describe('a restart between a Stop and its turn end', () => { return new Promise(() => undefined) }) void rig.stop() - await expect.poll(() => journal().activeTurnId()).toBe(TURN) + await expect.poll(() => journal().runningTurn()?.turnId ?? null).toBe(TURN) expect(journal().stopMarks.latest()?.event.turnId).toBeUndefined() await restartAndSettle('exit-observed') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts index ea0dae59199e..f243af3ba3c6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event.test.ts @@ -258,7 +258,7 @@ describe("a Stop's event", () => { await rig.settleAccepted(working, 'working') await turnRow('turn-1', 'interrupted') await eventually(async () => expect((await rig.handoff(next))?.handedOverAt).toBeDefined()) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() const fields = { turnId: 'turn-1' } await rig.host.cancel(QUEUED_RIG_CALLER, { envelope: rig.envelope(fields, 'agentSession.cancel', hostTestOperationId()), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts index 4cbc39073c17..526c82de0b25 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-never-waits.test.ts @@ -45,7 +45,7 @@ async function runningTurn(): Promise { throw new Error('no open journal') } // The turn row landed when the provider's event was handed over. - expect(journal.activeTurnId()).toBe('turn-1') + expect(journal.runningTurn()?.turnId ?? null).toBe('turn-1') return journal } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts index 26e5a0475678..ed528949a952 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-settle-binding.test.ts @@ -66,7 +66,7 @@ function openedTurn() { function nothingRuns(): boolean { return !isStructuredAgentSessionMainAgentWorking( - journal().activeTurnId(), + journal().runningTurn()?.turnId ?? null, journal().submissions() ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts index 479874892264..e760de42f86f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stopping.test.ts @@ -274,7 +274,7 @@ describe('a Stop that failed before the turn it meant to stop opened', () => { expect(await rig.stop()).toMatchObject({ ok: true }) // A Stop that ends its session ends the child on the session's next step. await rig.host['tasks'].serialize(HOST_TEST_SESSION, async () => {}) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() return { sent, status } } @@ -289,7 +289,7 @@ describe('a Stop that failed before the turn it meant to stop opened', () => { await rig.settleAccepted(sent, 'sent') await turn('turn-1', sent, 'running') - await eventually(() => expect(journal().activeTurnId()).toBe('turn-1')) + await eventually(() => expect(journal().runningTurn()?.turnId ?? null).toBe('turn-1')) expect(status()).toMatchObject({ status: 'working', stopping: true }) await turn('turn-1', sent, 'interrupted') await eventually(() => expect(status()?.status).toBe('idle')) @@ -357,7 +357,7 @@ describe('a Stop pressed before its send opened a turn', () => { it('ends the turn its interrupt took at the settle, and Stopping with it', async () => { const { status } = await stopAsTheTurnOpens({ cancelled: true, turnId: 'turn-1' }) - expect(journal().activeTurnId()).toBeNull() + expect(journal().runningTurn()?.turnId ?? null).toBeNull() await eventually(() => expect(status()?.status).toBe('idle')) expect(status()).not.toHaveProperty('stopping') }) @@ -396,7 +396,7 @@ describe('a Stop pressed before its send opened a turn', () => { expect(status()).not.toHaveProperty('stopping') await rig.settleAccepted(sent, 'sent') await turn('turn-1', sent, 'running') - await eventually(() => expect(journal().activeTurnId()).toBe('turn-1')) + await eventually(() => expect(journal().runningTurn()?.turnId ?? null).toBe('turn-1')) expect(status()).not.toHaveProperty('stopping') }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index f41865aab51a..b3b961a63446 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -110,7 +110,8 @@ export class AgentSessionSubscribers { }, subscriber, input.journal, - hostNow + hostNow, + this.hooks.readCurrentWork?.(input.sessionId, input.journal) ) } else { const work = this.hooks.readCurrentWork?.(input.sessionId, input.journal) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index df94fa7a34f3..fb89bdc96565 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -16,6 +16,8 @@ import { } from './structured-agent-session-command-turn' import { answerCancelOfSettledPrompt, + dismissEndedGenerationPrompt, + isEndedGenerationPrompt, validatePendingPrompt } from './structured-agent-session-prompt-state' import { @@ -96,13 +98,24 @@ type StopSettleBinding = { export async function performCancel( ctx: AgentSessionTurnContext, - input: PerformCancelInput + given: PerformCancelInput ): Promise> { + let input = given if (input.prompt) { const validated = validatePendingPrompt(ctx, input.prompt) if (!validated.ok) { return answerCancelOfSettledPrompt(ctx, { ...input, prompt: input.prompt }, validated) } + // A card an ended generation raised reaches no provider: dismissed in the journal, and the + // call is a plain Stop of what runs now, so an older client's Stop still stops it. + if (isEndedGenerationPrompt(ctx, validated)) { + const dismissed = await dismissEndedGenerationPrompt(ctx, validated) + if (!dismissed.ok) { + return dismissed + } + input = { ...input } + delete input.prompt + } } // A person's Stop that named no turn binds what ends while it settles (`beginJournalStopSettle`). const settle = input.opensSettle ? ctx.journal.stopMarks.beginSettle() : null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index 1ceb551108a8..df4d5fb28c31 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -370,16 +370,17 @@ describe('provider-exit settlement', () => { acquisitionGeneration: GENERATION }) - // Written at the fence the exit's release moved to, as is every row its settlement writes. + // Written at the exited child's own fence, before its release moves it, as is every row its + // settlement writes. expect(appendLifecycleBatch).toHaveBeenCalledWith( expect.objectContaining({ - fence: 8, + fence: 7, dispatches: [ { clientMessageId: 'client-1', state: 'unknown', reason: 'provider_exited_before_acknowledgement', - fence: 8, + fence: 7, recovered: true } ], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index eaa82e6d7175..7b00c0fca222 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -422,7 +422,7 @@ describe('already-wedged profiles become usable on load', () => { }) }) - it('settles an observed-exit latch through attach before the boot sweep', async () => { + it('an attach that beats the boot sweep clears the latch; the sweep then settles the turn unverifiable', async () => { const record = wedgedRecord({ claimStatus: 'released', handoffStage: 'recovering' }) // The settlement latch an older build wrote; this build derives the settlement instead. const olderBuildLatch = { @@ -443,17 +443,17 @@ describe('already-wedged profiles become usable on load', () => { openHost() expect(await host.attach(CALLER, hostTestAttachParams(13))).toMatchObject({ ok: true }) - // The reservation cleared the release's proof and handed it to the chat's worker. + // Acquiring a released lease ends nothing (design §9.8): the boot sweep settles what it left. + host.collaboratorsForTests().reconciliation.signal(SESSION, { startup: true }) await reconciled() expect(acquire).toHaveBeenCalledOnce() expect(activeStructuredAgentSessionTurnId(restoredJournal().snapshot().items)).toBe(null) - // The exit was observed, so its proof is the turn's end. - expect(turnLifecycle('turn-1')).toEqual({ + // The reservation cleared the release's proof before the sweep ran: less specific, never wrong. + expect(turnLifecycle('turn-1')).toMatchObject({ turnId: 'turn-1', - state: 'interrupted', + state: 'unverifiable', startedAt: NOW - 5_000, - completedAt: NOW - 1_000, recovered: true }) expect(store.getRecord(SESSION)?.lease).toMatchObject({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts index ba16a34b58ca..329f4405597f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-working-at-teardown.ts @@ -52,10 +52,7 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { isUnansweredHandedOverSubmission } from '../agent-session-journal/journal-unsent-send-hold' import { structuredAgentSessionShownStatus } from './structured-agent-session-shown-work' -import { - structuredAgentSessionCurrentWork, - type StructuredAgentSessionWorkEvidence -} from './structured-agent-session-current-work' +import { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' /** A send Orca journaled that the provider has neither opened a turn for nor refused. Mirrors the @@ -180,7 +177,6 @@ type WorkingCandidateSession = { journal: AgentSessionJournal /** Only this host generation's own child counts. A restored-for-reading journal has none. */ child: { fence: number } | null - lastEndedChild?: StructuredAgentSessionWorkEvidence['ended'] } /** The offer one session is owed, taken right before teardown stops its provider child; null when @@ -214,10 +210,8 @@ export function structuredAgentSessionWorkingAtStop(input: { ) const childWork = input.childWork(sessionId) const record = input.getRecord(sessionId) - const { scope } = structuredAgentSessionCurrentWork(session.journal, { - record, - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) - }) + // The live generation is this host's own child, read first-hand like the rest, never the lease. + const { scope } = new StructuredAgentSessionCurrentWork(session.journal, session.child.fence) const status = structuredAgentSessionShownStatus( { items: current.items, submissions: handedOver }, childWork, diff --git a/src/main/runtime/agent-session-generation-end.test.ts b/src/main/runtime/agent-session-generation-end.test.ts index 047f333fdaa0..30ef76df9fda 100644 --- a/src/main/runtime/agent-session-generation-end.test.ts +++ b/src/main/runtime/agent-session-generation-end.test.ts @@ -245,7 +245,9 @@ describe('a write that ends nothing', () => { expect(ended).toEqual([]) }) - it('hands on, once, the proof a reservation clears from a released lease, for the generation it judges', async () => { + // Design §9.8: that generation was told at its release; the reservation's clearing of the + // release's proof is no second end. + it('tells no one for a reservation over a lease an observed exit released with its proof', async () => { const first = await liveOwner() await releaseStoredStructuredAgentSessionOwnerAfterExit({ store, @@ -256,16 +258,9 @@ describe('a write that ends nothing', () => { }) ended = [] - const { fence } = await liveOwner() + await liveOwner() - expect(fence).toBe(first.fence + 2) expect(store.getRecord(SESSION)?.lease.deathEvidence).toBeNull() - expect(ended).toEqual([ - { - sessionId: SESSION, - endedFence: first.fence, - evidence: expect.objectContaining({ kind: 'exit-observed', ownerFence: first.fence }) - } - ]) + expect(ended).toEqual([]) }) }) diff --git a/src/main/runtime/agent-session-generation-end.ts b/src/main/runtime/agent-session-generation-end.ts index f5f4355d1f83..e7468e9420ed 100644 --- a/src/main/runtime/agent-session-generation-end.ts +++ b/src/main/runtime/agent-session-generation-end.ts @@ -2,9 +2,7 @@ // supersedes a generation that could still act. A release of any kind (exit, surface, failed // acquisition, eviction, unproven, restart adjudication) moves the fence and leaves no owner; a // reservation granted over an owner a probe proved gone supersedes it directly. Reserving a lease -// already released ends nothing new, but when the write drops the proof the release kept, that -// proof is handed on once more for the generation it judges, so nothing the store forgets is lost -// to the settlement still owed for it. +// already released ends nothing new: that generation was told at its release. import { agentSessionDeathEvidenceFromProbe, @@ -61,13 +59,8 @@ export function agentSessionGenerationEnds( if (!prior || prior === lease) { continue } + // A released lease's generation ended at its release; acquiring it again ends nothing. if (prior.claimStatus === 'released') { - const proof = prior.deathEvidence - if (proof && !sameProof(proof, lease.deathEvidence)) { - // A proof from before fences were recorded on it judges everything the lease fenced. - const ownerFence = proof.ownerFence ?? prior.runtimeFence - ends.push({ sessionId, endedFence: ownerFence, evidence: { ...proof, ownerFence } }) - } continue } const released = lease.claimStatus === 'released' @@ -82,12 +75,3 @@ export function agentSessionGenerationEnds( } return ends } - -function sameProof(proof: AgentSessionDeathEvidence, other: AgentSessionDeathEvidence | null) { - return ( - other !== null && - other.kind === proof.kind && - other.observedAt === proof.observedAt && - other.ownerFence === proof.ownerFence - ) -} diff --git a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx index 8ff0e413f445..9c11a9e2053f 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx @@ -27,6 +27,7 @@ import { NativeChatWaitingTranscriptItems } from './NativeChatTranscriptItems' import type { NativeChatTranscriptRowContext } from './NativeChatTranscriptRow' +import type { StructuredAgentSessionHostWork } from './use-structured-agent-session-transport-state' import type { NativeChatDeliveryNotice } from './NativeChatMessageRow' import { splitNativeChatSlotsWaitingBehindLiveTurn, @@ -77,6 +78,7 @@ export function NativeChatMessageList({ journalItems, journalSubmissions, journalLatestTurn: latestTurn, + journalHostWork: hostWork, subagentRoster, railOutline = null, isVisible = true, @@ -99,6 +101,8 @@ export function NativeChatMessageList({ journalSubmissions?: readonly AgentJournalSubmission[] /** The host's newest turn record, which places a live turn whose record is not loaded. */ journalLatestTurn?: AgentSessionLatestTurn | null + /** The host's prompts still waiting on the person and its working answer (an older host: none). */ + journalHostWork?: StructuredAgentSessionHostWork /** Every subagent the session's rosters named, whether or not its roster row is loaded. */ subagentRoster?: Parameters[2] /** User messages older than the loaded window, from the host's outline. */ @@ -218,9 +222,10 @@ export function NativeChatMessageList({ allSlots, journalItems, stopping, - journalSubmissions + journalSubmissions, + { latestTurn, ...hostWork } ), - [allSlots, journalItems, stopping, journalSubmissions] + [allSlots, journalItems, stopping, journalSubmissions, latestTurn, hostWork] ) const transcriptWindow = useNativeChatTranscriptWindow({ scrollRef, @@ -328,6 +333,7 @@ export function NativeChatMessageList({ const rowContext = useMemo( () => ({ agentName: structuredAgentLabel(session.agent), + actionablePromptIds: hostWork?.actionablePromptIds, expandSignal, revealedDiff, taskListPredecessors, @@ -343,6 +349,7 @@ export function NativeChatMessageList({ }), [ session.agent, + hostWork, allowFileUriLinks, expandSignal, expandedTurnIds, diff --git a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx index 72f1f62c21d3..00c9c4b94e05 100644 --- a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx +++ b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.test.tsx @@ -218,6 +218,22 @@ describe('resolution receipts', () => { expect(screen.queryByText('1 grouped question from Claude')).toBeNull() }) + it('reports, without breathing, a question whose agent ended (the host says nobody waits)', () => { + const body: AgentJournalQuestionItem = { + kind: 'question', + question: 'Which branch?', + options: [], + resolution: { ...approval.resolution, state: 'pending', selectedOptionId: null } + } + + const { rerender } = render() + expect(screen.getByText('Awaiting user input:')).toHaveClass('animate-pulse') + rerender() + expect(screen.queryByText('Awaiting user input:')).toBeNull() + expect(screen.getByText('Asked:')).not.toHaveClass('animate-pulse') + expect(screen.getByText('Which branch?')).toBeInTheDocument() + }) + it('keeps an opened question open once it is answered', () => { const scrollWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'scrollWidth') Object.defineProperty(HTMLElement.prototype, 'scrollWidth', { diff --git a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx index f3e585ae5f9f..4d65f6400d1a 100644 --- a/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx +++ b/src/renderer/src/components/native-chat/NativeChatResolutionReceipt.tsx @@ -9,11 +9,15 @@ import { export function NativeChatResolutionReceipt({ body, - disclosureId + disclosureId, + awaiting = true }: { body: NativeChatResolvedPrompt /** Message this receipt stands in for; keys the question row's disclosure. */ disclosureId?: string + /** False when the host says nobody waits on this pending prompt: the agent that raised it ended, + * so its row reports what was asked instead of breathing. */ + awaiting?: boolean }): React.JSX.Element | null { const askDisclosureKey = disclosureId === undefined ? undefined : `ask:${disclosureId}` const subject: NativeChatAskRowSubject | null = @@ -35,7 +39,11 @@ export function NativeChatResolutionReceipt({ } if (body.resolution.state === 'pending') { return body.kind === 'question' ? ( - + ) : null } const { resolution } = body diff --git a/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx b/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx index 546273874887..2d112f9fab3c 100644 --- a/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx +++ b/src/renderer/src/components/native-chat/NativeChatStructuredSession.tsx @@ -284,6 +284,7 @@ export function NativeChatStructuredSession( journalItems={controller.journalItems} journalSubmissions={controller.submissions} journalLatestTurn={controller.latestTurn} + journalHostWork={controller.hostWork} subagentRoster={controller.subagentRoster} railOutline={controller.railOutline} isVisible={props.isVisible} diff --git a/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx b/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx index 446deba8da5c..aac778310a4d 100644 --- a/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx +++ b/src/renderer/src/components/native-chat/NativeChatTranscriptRow.tsx @@ -14,11 +14,14 @@ import type { NativeChatTranscriptSlot } from './native-chat-transcript-slots' import type { NativeChatDiffReveal, NativeChatDiffTarget } from './native-chat-turn-diffs' import { NativeChatRewindContext } from './native-chat-rewind-context' import { nativeChatRowOffersRewind } from './native-chat-rewind-eligibility' +import { isActionableStructuredAgentSessionPrompt } from '../../../../shared/structured-agent-session-live-turn' /** Everything a row needs that is the same for every row. Held as one memoized * object so a row's props change only when that row's own slot does. */ export type NativeChatTranscriptRowContext = { agentName?: string + /** The prompts the host says still wait on the person; absent from an older host. */ + actionablePromptIds?: readonly string[] | undefined expandSignal: boolean revealedDiff: NativeChatDiffReveal | null taskListPredecessors: ReadonlyMap @@ -98,7 +101,14 @@ export const NativeChatTranscriptRow = memo(function NativeChatTranscriptRow({ {/* A turn with no user bubble carries its bar above its first row. */} {slot.statusAbove ? statusRow : null} {!slot.drawsMessage ? null : receipt ? ( - + ) : ( + translate( + 'components.native-chat.failureWords.providerExitedAnswer', + COPY.providerExitedAnswer, + values + ), providerRejected: () => translate('components.native-chat.failureWords.providerRejected', COPY.providerRejected), providerRejectedQuoted: (values) => diff --git a/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts b/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts index 3589b1084a0b..97248e1be30c 100644 --- a/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts +++ b/src/renderer/src/components/native-chat/agent-session-write-notice-text.ts @@ -2,7 +2,10 @@ // shared English as its fallback so desktop and mobile never say it differently. import { translate } from '@/i18n/i18n' -import { agentSessionFailureSentence } from '../../../../shared/agent-session-failure-words' +import { + agentSessionFailureSentence, + type AgentSessionFailureWordsContext +} from '../../../../shared/agent-session-failure-words' import { agentSessionWriteNoticeParts } from '../../../../shared/agent-session-refusal-notice' import { AGENT_SESSION_WRITE_NOTICE_COPY as COPY, @@ -178,7 +181,8 @@ export function agentSessionWriteNoticeText(parts: readonly AgentSessionWriteNot export function agentSessionWriteFailureText( failure: AgentSessionWriteFailure, - write: AgentSessionWriteKind + write: AgentSessionWriteKind, + context: AgentSessionFailureWordsContext = {} ): string { - return agentSessionWriteNoticeText(agentSessionWriteNoticeParts(failure, write)) + return agentSessionWriteNoticeText(agentSessionWriteNoticeParts(failure, write, context)) } diff --git a/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts b/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts index ebae5d79e213..8e956d78e653 100644 --- a/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts +++ b/src/renderer/src/components/native-chat/native-chat-transcript-slots.ts @@ -320,13 +320,15 @@ export function splitNativeChatSlotsWaitingBehindLiveTurn( slots: readonly NativeChatTranscriptSlot[], journalItems: readonly AgentJournalRenderItem[] | undefined, stopping = false, - journalSubmissions?: readonly AgentJournalSubmission[] + journalSubmissions?: readonly AgentJournalSubmission[], + host: Parameters[4] = {} ): { slots: NativeChatTranscriptSlot[]; waitingSlots: NativeChatTranscriptSlot[] } { const waiting = nativeChatMessagesWaitingBehindLiveTurn( slots.flatMap((slot) => (slot.kind === 'message' ? [slot.message] : [])), journalItems, stopping, - journalSubmissions + journalSubmissions, + host ) const isWaiting = (slot: NativeChatTranscriptSlot): boolean => slot.kind === 'message' && diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts index bfccacad0933..55b02cf3ccc5 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-mutate.ts @@ -57,11 +57,16 @@ function namesWhatItStops(fingerprintMethod: string, fields: Record + fields: Record, + agentName: string | undefined ): { kind: 'not-done'; notice: string; failure: AgentSessionWriteFailure } { return { kind: 'not-done', - notice: agentSessionWriteFailureText(failure, writeKind(fingerprintMethod, fields)), + notice: agentSessionWriteFailureText( + failure, + writeKind(fingerprintMethod, fields), + agentName ? { agentName } : {} + ), failure } } @@ -73,11 +78,13 @@ export function useStructuredAgentSessionMutate(args: { /** Read at settle time, not at call time: the fence can move while a request * is in flight, and a result from the previous fence is not this session's. */ stateRef: { current: { fence: number | null } } + /** The chat's agent, which a refusal about it names. */ + agentName?: string }): { write: StructuredAgentSessionWrite mutate: StructuredAgentSessionMutate } { - const { enabled = true, sessionId, stateRef, target } = args + const { agentName, enabled = true, sessionId, stateRef, target } = args const inFlightStops = useRef( new Map>>() ) @@ -144,13 +151,19 @@ export function useStructuredAgentSessionMutate(args: { error instanceof RuntimeRpcCallError ? error.code : undefined ), fingerprintMethod, - fields + fields, + agentName ) : { kind: 'dropped' } } if (!result.ok) { return settlesHere() - ? notDone(agentSessionRefusalFailure(result.refusal), fingerprintMethod, fields) + ? notDone( + agentSessionRefusalFailure(result.refusal), + fingerprintMethod, + fields, + agentName + ) : { kind: 'dropped' } } if (!settlesHere()) { @@ -158,7 +171,7 @@ export function useStructuredAgentSessionMutate(args: { } return { kind: 'done', value: result.value } }, - [enabled, sessionId, stateRef, target] + [agentName, enabled, sessionId, stateRef, target] ) const write = useCallback( diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts index 3fd00f9a1302..84f1eec124a0 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport-state.ts @@ -16,6 +16,13 @@ const NO_JOURNAL_ITEMS: StructuredAgentSessionState['items'] = [] const NO_SUBMISSIONS: StructuredAgentSessionState['submissions'] = [] const NO_SUBAGENT_ROSTER: StructuredAgentSubagentRoster = new Map() +/** The host's prompts still waiting on the person and its working answer; empty from an older + * host, whose readers derive both from the rows. */ +export type StructuredAgentSessionHostWork = { + actionablePromptIds?: readonly string[] + working?: boolean +} + export function useStructuredAgentSessionTransportState( state: StructuredAgentSessionState, enabled: boolean @@ -64,10 +71,18 @@ export function useStructuredAgentSessionTransportState( structuredSessionForegroundCommands(roster, { items: journalItems, latestTurn }) ) }, [enabled, journalItems, latestTurn, state.backgroundTasks, turnId]) + const hostWork = useMemo( + () => ({ + ...(actionablePromptIds ? { actionablePromptIds } : {}), + ...(hostWorking !== undefined ? { working: hostWorking } : {}) + }), + [actionablePromptIds, hostWorking] + ) return { journalItems, latestTurn, actionablePromptIds, + hostWork, subagentRoster, submissions, fence, diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts index 47ca68a5db76..0eb220affd2e 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-transport.ts @@ -9,8 +9,10 @@ export function useStructuredAgentSessionTransport(args: { target: RuntimeClientTarget isVisible: boolean enabled: boolean + /** The chat's agent, which a refused write about it names. */ + agentName?: string }) { - const { enabled, isVisible, sessionId, target } = args + const { agentName, enabled, isVisible, sessionId, target } = args const providerVisible = isVisible && enabled useStructuredAgentSessionHold({ sessionId, @@ -25,7 +27,8 @@ export function useStructuredAgentSessionTransport(args: { sessionId, target, stateRef, - enabled + enabled, + ...(agentName ? { agentName } : {}) }) useEffect(() => { stateRef.current = read.state diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session.ts b/src/renderer/src/components/native-chat/use-structured-agent-session.ts index 3a0e144138ac..6a8f1e015eea 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session.ts @@ -89,7 +89,8 @@ export function useStructuredAgentSession(args: { sessionId, target, isVisible, - enabled: transportEnabled + enabled: transportEnabled, + agentName: structuredAgentLabel(agent) }) const commandPending = useRef(false) const transportState = useStructuredAgentSessionTransportState(state, transportEnabled) @@ -243,6 +244,8 @@ export function useStructuredAgentSession(args: { journalItems: transcriptItems, /** The host's newest turn record, which places a live turn whose record is not loaded. */ latestTurn: transportState.latestTurn, + /** The host's prompts still waiting on the person and its working answer. */ + hostWork: transportState.hostWork, subagentRoster: transportState.subagentRoster, messages, status: transportEnabled ? state.status : 'ready', diff --git a/src/renderer/src/i18n/en-runtime-required.json b/src/renderer/src/i18n/en-runtime-required.json index ef80824aa94b..91771f91d159 100644 --- a/src/renderer/src/i18n/en-runtime-required.json +++ b/src/renderer/src/i18n/en-runtime-required.json @@ -2912,6 +2912,7 @@ "notSignedIn": "{{agent}} is not signed in.", "previousExitUnverifiable": "Couldn't stop {{agent}} from before.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", + "providerExitedAnswer": "{{agent}} has stopped, so your answer was not sent. Send a message to continue.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerRateLimited": "{{agent}} is rate-limited and retrying.", "providerRejected": "The provider did not accept this message.", diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index fa1afe01cbc6..ac7ba1bfefcc 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -18035,6 +18035,7 @@ "chooseClaudeAccountThenSend": "Choose or add one in Claude Accounts settings, then send your message again.", "providerExitedRow": "{{agent}} stopped while this response was in progress. You can continue in this conversation.", "providerExitedRejection": "{{agent}} stopped before this message was sent.", + "providerExitedAnswer": "{{agent}} has stopped, so your answer was not sent. Send a message to continue.", "providerRejected": "The provider did not accept this message.", "providerRejectedQuoted": "The provider did not accept this message: {{detail}}.", "attachmentEmpty": "An image on this message is empty, so the message was not sent.", diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index f1456c9c6781..298bbdd01e66 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -17813,6 +17813,7 @@ "chooseClaudeAccountThenSend": "Elija o agregue una en la configuración de Cuentas de Claude y vuelva a enviar su mensaje.", "providerExitedRow": "{{agent}} se detuvo mientras esta respuesta estaba en curso. Puede continuar en esta conversación.", "providerExitedRejection": "{{agent}} se detuvo antes de que se enviara este mensaje.", + "providerExitedAnswer": "{{agent}} se detuvo, así que su respuesta no se envió. Envíe un mensaje para continuar.", "providerRejected": "El proveedor no aceptó este mensaje.", "providerRejectedQuoted": "El proveedor no aceptó este mensaje: {{detail}}.", "attachmentEmpty": "Una imagen de este mensaje está vacía, así que el mensaje no se envió.", diff --git a/src/renderer/src/i18n/locales/fr.json b/src/renderer/src/i18n/locales/fr.json index 74954b2510cf..2a2db90fd74b 100644 --- a/src/renderer/src/i18n/locales/fr.json +++ b/src/renderer/src/i18n/locales/fr.json @@ -17914,6 +17914,7 @@ "chooseClaudeAccountThenSend": "Choisissez-en un ou ajoutez-en un dans les paramètres Comptes Claude, puis renvoyez votre message.", "providerExitedRow": "{{agent}} s'est arrêté pendant que cette réponse était en cours. Vous pouvez continuer dans cette conversation.", "providerExitedRejection": "{{agent}} s'est arrêté avant l'envoi de ce message.", + "providerExitedAnswer": "{{agent}} s'est arrêté, donc votre réponse n'a pas été envoyée. Envoyez un message pour continuer.", "providerRejected": "Le fournisseur n'a pas accepté ce message.", "providerRejectedQuoted": "Le fournisseur n'a pas accepté ce message : {{detail}}.", "attachmentEmpty": "Une image de ce message est vide, le message n'a donc pas été envoyé.", diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index 71a8d66f1648..cffb37d84a71 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -17848,6 +17848,7 @@ "chooseClaudeAccountThenSend": "Claude アカウントの設定で選択または追加してから、メッセージをもう一度送信してください。", "providerExitedRow": "この応答の途中で {{agent}} が停止しました。この会話で続けることができます。", "providerExitedRejection": "このメッセージが送信される前に {{agent}} が停止しました。", + "providerExitedAnswer": "{{agent}} が停止したため、回答は送信されませんでした。続けるにはメッセージを送信してください。", "providerRejected": "プロバイダーはこのメッセージを受け付けませんでした。", "providerRejectedQuoted": "プロバイダーはこのメッセージを受け付けませんでした: {{detail}}。", "attachmentEmpty": "このメッセージの画像が空のため、メッセージは送信されませんでした。", diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 2a4cee1663bd..3ebc1bfd1db0 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -17848,6 +17848,7 @@ "chooseClaudeAccountThenSend": "Claude 계정 설정에서 선택하거나 추가한 다음 메시지를 다시 보내세요.", "providerExitedRow": "이 응답이 진행되는 동안 {{agent}}이(가) 중지되었습니다. 이 대화에서 계속할 수 있습니다.", "providerExitedRejection": "이 메시지가 전송되기 전에 {{agent}}이(가) 중지되었습니다.", + "providerExitedAnswer": "{{agent}}이(가) 중지되어 답변이 전송되지 않았습니다. 계속하려면 메시지를 보내세요.", "providerRejected": "공급자가 이 메시지를 받아들이지 않았습니다.", "providerRejectedQuoted": "공급자가 이 메시지를 받아들이지 않았습니다: {{detail}}.", "attachmentEmpty": "이 메시지의 이미지가 비어 있어 메시지가 전송되지 않았습니다.", diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index b1306527066e..6b092f41ada4 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -17813,6 +17813,7 @@ "chooseClaudeAccountThenSend": "请在 Claude 账户设置中选择或添加一个，然后重新发送您的消息。", "providerExitedRow": "{{agent}} 在生成此回复时停止了。您可以在此对话中继续。", "providerExitedRejection": "{{agent}} 在发送此消息前已停止。", + "providerExitedAnswer": "{{agent}} 已停止，因此您的回答未发送。发送消息以继续。", "providerRejected": "提供商未接受此消息。", "providerRejectedQuoted": "提供商未接受此消息：{{detail}}。", "attachmentEmpty": "此消息中的一张图片为空，因此消息未发送。", diff --git a/src/shared/agent-session-failure-copy.ts b/src/shared/agent-session-failure-copy.ts index 8f17594db219..0daed644a8fc 100644 --- a/src/shared/agent-session-failure-copy.ts +++ b/src/shared/agent-session-failure-copy.ts @@ -69,6 +69,8 @@ export const AGENT_SESSION_FAILURE_COPY = { providerExitedRow: '{{agent}} stopped while this response was in progress. You can continue in this conversation.', providerExitedRejection: '{{agent}} stopped before this message was sent.', + providerExitedAnswer: + '{{agent}} has stopped, so your answer was not sent. Send a message to continue.', providerRejected: 'The provider did not accept this message.', providerRejectedQuoted: 'The provider did not accept this message: {{detail}}.', attachmentEmpty: 'An image on this message is empty, so the message was not sent.', diff --git a/src/shared/agent-session-failure-words.test.ts b/src/shared/agent-session-failure-words.test.ts index bfdc644b8bee..f0eff4cdbffa 100644 --- a/src/shared/agent-session-failure-words.test.ts +++ b/src/shared/agent-session-failure-words.test.ts @@ -24,7 +24,7 @@ import { /** What Orca's own text looks like: a code or marker, a uuid, a path, an exception. */ const ORCA_INTERNAL = /\b[a-z]+_[a-z_]+\b|[0-9a-f]{8}-[0-9a-f]{4}-|[/\\][\w.-]+[/\\]|Error:|ENOENT/ -const SURFACES: readonly AgentSessionFailureSurface[] = ['row', 'rejection'] +const SURFACES: readonly AgentSessionFailureSurface[] = ['row', 'rejection', 'answer'] const LEGACY_MARKER_KINDS: ReadonlySet = new Set([ 'cancelled', 'writeFailed', @@ -76,7 +76,11 @@ describe('the words written beside a failure fact', () => { expect(sentence).not.toMatch(ORCA_INTERNAL) if (surface === 'row') { expect(agentSessionFailureWords(fact, { ...context, surface }).text).toBe(sentence) - } else if (isSubmissionRejectionFact(fact) && !LEGACY_MARKER_KINDS.has(fact.kind)) { + } else if ( + surface === 'rejection' && + isSubmissionRejectionFact(fact) && + !LEGACY_MARKER_KINDS.has(fact.kind) + ) { expect(agentSessionFailureWords(fact, { ...context, surface }).reason).toBe(sentence) } } diff --git a/src/shared/agent-session-failure-words.ts b/src/shared/agent-session-failure-words.ts index 79133b2389b3..f9aea71cc261 100644 --- a/src/shared/agent-session-failure-words.ts +++ b/src/shared/agent-session-failure-words.ts @@ -54,8 +54,9 @@ export type AgentJournalDispatchRejection = { rejection: SubmissionRejectionFact } -/** `row`: a status row, about the chat. `rejection`: a rejected message's reason, about it. */ -export type AgentSessionFailureSurface = 'row' | 'rejection' +/** `row`: a status row, about the chat. `rejection`: a rejected message's reason, about it. + * `answer`: a refused answer to an approval or question, about it. */ +export type AgentSessionFailureSurface = 'row' | 'rejection' | 'answer' export type AgentSessionFailureWordsContext = { /** The chat's agent, when the writer knows it. */ @@ -233,7 +234,14 @@ const FAILURE_SENTENCES = { : say('chooseClaudeAccountThenSend') ]), providerExited: (context, _fact, surface, say) => - say(surface === 'row' ? 'providerExitedRow' : 'providerExitedRejection', agent(say, context)), + say( + surface === 'row' + ? 'providerExitedRow' + : surface === 'answer' + ? 'providerExitedAnswer' + : 'providerExitedRejection', + agent(say, context) + ), restartFailed: couldNot('couldNotRestart'), providerRejected: (_context, fact, _surface, say) => quotingPersonDetail(say, 'providerRejected', 'providerRejectedQuoted', fact.detail), diff --git a/src/shared/agent-session-refusal-notice.test.ts b/src/shared/agent-session-refusal-notice.test.ts index 4063a95aaea5..2b10084a297f 100644 --- a/src/shared/agent-session-refusal-notice.test.ts +++ b/src/shared/agent-session-refusal-notice.test.ts @@ -451,6 +451,12 @@ describe('the notice for every reason a host names', () => { } const notDone = parts.filter((part) => typeof part === 'string' && part.startsWith('notDone')) const answeredAway = write === 'answer' && parts.includes('questionChanged') + // "...so your answer was not sent" is that sentence's own. + const ownerStopped = + write === 'answer' && + parts.some( + (part) => typeof part === 'object' && 'surface' in part && part.surface === 'answer' + ) const unsupported = failure.code === 'structured_agent_session_unsupported' && failure.details?.reason !== 'hostUnsupported' && @@ -467,7 +473,9 @@ describe('the notice for every reason a host names', () => { ] as const ).some((sentence) => parts.includes(sentence)) expect(notDone, cell).toEqual( - answeredAway || unsupported || saysNotDone || clearWhileWorking ? [] : [NOT_DONE[write]] + answeredAway || ownerStopped || unsupported || saysNotDone || clearWhileWorking + ? [] + : [NOT_DONE[write]] ) } }) @@ -530,7 +538,7 @@ describe('the notice for every reason a host names', () => { 'agent_session_operation_invalid', 'promptOwnerEnded', 'answer', - 'Your answer was not sent. The agent stopped while this response was in progress. You can continue in this conversation.' + 'The agent has stopped, so your answer was not sent. Send a message to continue.' ], [ 'agent_session_checkpoint_stale', @@ -545,6 +553,19 @@ describe('the notice for every reason a host names', () => { }) }) +describe('an answer to a prompt whose agent stopped', () => { + it('names the agent the client knows, on the phone and on desktop alike', () => { + const refusal = { + code: 'agent_session_operation_invalid' as const, + message: HOST_TEXT, + details: { reason: 'promptOwnerEnded' as const } + } + expect(agentSessionRefusalNotice(refusal, 'answer', { agentName: 'Codex' })).toBe( + 'Codex has stopped, so your answer was not sent. Send a message to continue.' + ) + }) +}) + describe('a refusal from a host that names no reason this build knows', () => { it.each([ ['an older host', undefined], diff --git a/src/shared/agent-session-refusal-notice.ts b/src/shared/agent-session-refusal-notice.ts index 67c686f9b720..c429eb689bb1 100644 --- a/src/shared/agent-session-refusal-notice.ts +++ b/src/shared/agent-session-refusal-notice.ts @@ -126,8 +126,9 @@ function reasonParts( } if ('fact' in words) { if (write === 'answer') { - // An answer to an agent that stopped: its row's sentence, which names it and says go on. - return [NOT_DONE[write], { failure: { kind: words.fact }, surface: 'row', context }] + // An approval or question was waiting on the person when its agent stopped: one sentence + // that says the answer was not sent, names the agent, and says how to go on. + return [{ failure: { kind: words.fact }, surface: 'answer', context }] } return write === 'send' || write === 'composer-send' ? [ @@ -256,10 +257,11 @@ export function agentSessionWriteNoticeEnglish( * message is not read. */ export function agentSessionRefusalNotice( refusal: Pick, - write: AgentSessionWriteKind + write: AgentSessionWriteKind, + context: AgentSessionFailureWordsContext = {} ): string { return agentSessionWriteNoticeEnglish( - agentSessionWriteNoticeParts(agentSessionRefusalFailure(refusal), write) + agentSessionWriteNoticeParts(agentSessionRefusalFailure(refusal), write, context) ) } diff --git a/src/shared/agent-session-refusal-reason-words.ts b/src/shared/agent-session-refusal-reason-words.ts index bbf30cd0d27f..1a3ced1b103b 100644 --- a/src/shared/agent-session-refusal-reason-words.ts +++ b/src/shared/agent-session-refusal-reason-words.ts @@ -88,7 +88,8 @@ const REASON_WORDS = { rewindRefused: codeWords('hostFinding'), rewindUnconfirmed: codeWords('hostFinding'), promptGone: causeWords('questionChanged', 'nothingLeft'), - // Said as that agent's own stop, naming it, with the step on from there. + // An approval or question waited on the person when its agent stopped: said naming the agent, + // with sending a message as the step on. promptOwnerEnded: { fact: 'providerExited', action: 'nothingLeft' }, optionRejected: causeWords('optionRejected', 'retry'), providerStarting: AGENT_STARTING, diff --git a/src/shared/native-chat-messages-waiting-behind-live-turn.test.ts b/src/shared/native-chat-messages-waiting-behind-live-turn.test.ts new file mode 100644 index 000000000000..91fecb6b5195 --- /dev/null +++ b/src/shared/native-chat-messages-waiting-behind-live-turn.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import type { AgentJournalRenderItem } from './agent-session-journal-types' +import { nativeChatMessagesWaitingBehindLiveTurn } from './native-chat-messages-waiting-behind-live-turn' + +/** A /compact whose turn record still reads running. */ +const compactRunning: AgentJournalRenderItem[] = [ + { + itemId: 'compact-entry', + revision: 0, + sequence: 1, + observedAt: 1, + body: { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: '/compact' }], + command: { name: 'compact' } + } + }, + { + itemId: 'compact-turn', + revision: 0, + sequence: 2, + observedAt: 2, + body: { kind: 'turn', turnId: 'compact-turn', userItemId: 'compact-entry', state: 'running' } + } +] +const queued = [{ id: 'card-1', role: 'user' as const, queued: true as const }] + +describe('a queued card behind a running /compact', () => { + it('waits behind it while the host says it runs, or an older host says nothing', () => { + expect(nativeChatMessagesWaitingBehindLiveTurn(queued, compactRunning)).toEqual( + new Set(['card-1']) + ) + const latestTurn = { + itemId: 'compact-turn', + observedAt: 2, + turn: { turnId: 'compact-turn', state: 'running' as const } + } + expect( + nativeChatMessagesWaitingBehindLiveTurn(queued, compactRunning, false, [], { latestTurn }) + ).toEqual(new Set(['card-1'])) + }) + + it('waits behind nothing once the host says no turn runs: the agent that ran it ended', () => { + expect( + nativeChatMessagesWaitingBehindLiveTurn(queued, compactRunning, false, [], { + latestTurn: null, + working: false + }) + ).toEqual(new Set()) + }) +}) diff --git a/src/shared/native-chat-messages-waiting-behind-live-turn.ts b/src/shared/native-chat-messages-waiting-behind-live-turn.ts index 767bb1e8c826..6c42a8029dbc 100644 --- a/src/shared/native-chat-messages-waiting-behind-live-turn.ts +++ b/src/shared/native-chat-messages-waiting-behind-live-turn.ts @@ -1,10 +1,11 @@ // Which rows wait at the tail, after the live turn's activity line, rather than in the transcript. import type { AgentJournalRenderItem, AgentJournalSubmission } from './agent-session-journal-types' +import type { AgentSessionLatestTurn } from './agent-session-wire' import { readAgentJournalTurn } from './agent-session-turn-record' import type { NativeChatRole } from './native-chat-types' import { isStructuredAgentSessionCommandTurn } from './structured-agent-session-command-entry' -import { liveStructuredAgentSessionTurnScope } from './structured-agent-session-live-turn' +import { runningStructuredAgentSessionTurnScope } from './structured-agent-session-live-turn' import { structuredAgentSessionOpeningSendIn } from './structured-agent-session-opening-send' /** @@ -30,7 +31,10 @@ export function nativeChatMessagesWaitingBehindLiveTurn( }[], items: readonly AgentJournalRenderItem[] | null | undefined, stopping = false, - submissions?: readonly AgentJournalSubmission[] + submissions?: readonly AgentJournalSubmission[], + /** The host's view of current work (`latestTurn`, `working`): an ended generation's running turn + * or unanswered send holds nothing. Absent from an older host, whose rows are read instead. */ + host: { latestTurn?: AgentSessionLatestTurn | null; working?: boolean } = {} ): ReadonlySet { const waits = (message: (typeof messages)[number]): boolean => message.queued === true || (stopping && message.sentWhileStopping === true) @@ -45,9 +49,11 @@ export function nativeChatMessagesWaitingBehindLiveTurn( if (candidates.length === 0 || !items) { return new Set() } - const behindHold = stopping || commandTurnRunning(items) + const behindHold = stopping || commandTurnRunning(items, host.latestTurn) const behindOpening = - submissions !== undefined && structuredAgentSessionOpeningSendIn(items, submissions) !== null + host.working !== false && + submissions !== undefined && + structuredAgentSessionOpeningSendIn(items, submissions) !== null return new Set( candidates .filter( @@ -59,8 +65,14 @@ export function nativeChatMessagesWaitingBehindLiveTurn( ) } -function commandTurnRunning(items: readonly AgentJournalRenderItem[]): boolean { - const running = liveStructuredAgentSessionTurnScope(items) +function commandTurnRunning( + items: readonly AgentJournalRenderItem[], + latestTurn: AgentSessionLatestTurn | null | undefined +): boolean { + const running = runningStructuredAgentSessionTurnScope({ + items, + ...(latestTurn !== undefined ? { latestTurn } : {}) + }) const bodyOf = (itemId: string) => items.find((item) => item.itemId === itemId)?.body return ( running.kind === 'turn' && diff --git a/src/shared/native-chat-turn-membership.ts b/src/shared/native-chat-turn-membership.ts index 7eecbe0d3ea8..191da871be54 100644 --- a/src/shared/native-chat-turn-membership.ts +++ b/src/shared/native-chat-turn-membership.ts @@ -45,6 +45,8 @@ export type NativeChatTurnJournal = { * record is not loaded, so the turn's loaded rows still draw under its bar. Absent from older * hosts. */ latestTurn?: AgentSessionLatestTurn | null + /** The host's answer to whether the session's own agent is working; absent from older hosts. */ + working?: boolean } /** From edd0c638bcade23c0f4728480a37f14f930a1257 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:29:34 -0700 Subject: [PATCH 05/22] fix(native-chat): startup settles at main's scope; an earlier process's share is derived - startup no longer resolves recovery: only the visible-tab restore, start and attach decide a recovering lease (as before this change), and their release signals the worker - the startup share is derived on every pass from where this process first opened the chat, instead of an in-memory debt that a recovering lease could strand --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + .../journal-unsent-send-hold.ts | 12 +- ...-agent-session-accept-then-deliver.test.ts | 5 +- .../structured-agent-session-conversations.ts | 4 +- .../structured-agent-session-current-work.ts | 2 +- .../structured-agent-session-host-lifetime.ts | 2 +- .../structured-agent-session-host.ts | 3 +- ...session-queued-message-rig.test-fixture.ts | 4 +- ...tured-agent-session-reconciliation-pass.ts | 103 +++++++++-------- ...gent-session-reconciliation-worker.test.ts | 8 +- ...red-agent-session-reconciliation-worker.ts | 49 ++++---- .../structured-agent-session-reveal.ts | 7 +- ...uctured-agent-session-start-failure-row.ts | 2 +- ...ent-session-startup-recovery-scope.test.ts | 107 ++++++++++++++++++ ...gent-session-startup-scan-lifetime.test.ts | 30 +++-- ...agent-session-startup-scan.test-fixture.ts | 6 +- ...uctured-agent-session-startup-scan.test.ts | 88 +++++++------- ...ctured-agent-session-startup-settlement.ts | 17 ++- ...t-session-wedged-profile-migration.test.ts | 8 +- .../structured-rewind-recovery.ts | 11 ++ 20 files changed, 306 insertions(+), 163 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index a55e2b5015d3..5b81edb72e97 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -114,6 +114,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts index fd5a774c7bf2..037f167b9474 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts @@ -23,8 +23,9 @@ import type { QueuedMessagePositionMove } from './queued-message-positions' /** Why the host can no longer hand a send over, which also says which sends it is. */ export type UnsentSendHold = - /** At open, and the delivery loop's first step: what an earlier host process accepted. */ - | { cause: 'hostRestarted' } + /** What an earlier host process accepted: by default, what was on disk when this handle opened; + * `which` names it for a handle opened after this process accepted sends of its own. */ + | { cause: 'hostRestarted'; which?: (submission: AgentJournalSubmission) => boolean } /** A close of the chat; `which` narrows it to what a close that did not complete closed. */ | { cause: 'chatClosed'; which?: (submission: AgentJournalSubmission) => boolean } @@ -96,9 +97,10 @@ export async function holdUnsentSends( input.unrun ? isUnansweredHandedOverSubmission(entry) : isQueuedAgentJournalSubmission(entry) && - (hold.cause === 'hostRestarted' - ? journal.wroteBeforeOpen(entry.acceptedSequence) - : (hold.which?.(entry) ?? true)) + (hold.which?.(entry) ?? + (hold.cause === 'hostRestarted' + ? journal.wroteBeforeOpen(entry.acceptedSequence) + : true)) ) .sort((a, b) => (a.acceptedSequence ?? 0) - (b.acceptedSequence ?? 0)) if (unsent.length === 0) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts index 965925432a5e..aaefa5a457d9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-accept-then-deliver.test.ts @@ -253,6 +253,9 @@ async function writeAsEarlierProcess( }) await write(journal, record.lease.runtimeFence) await journal.close() + // The next host is a new process, which has not opened the chat yet. + await host.flushAllStreamedEvents() + await startHost() } function earlierSubmission(id: string, text: string, handoverRecorded?: true) { @@ -582,8 +585,6 @@ describe('what an earlier host process left behind', () => { turnScope: AGENT_JOURNAL_THREAD_SCOPE }) }) - await host.flushAllStreamedEvents() - await startHost() await startUpHost(host) expect(await submission('legacy')).toMatchObject({ dispatchState: 'unknown', recovered: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts index 745c33f8d9dd..9d7fc468a655 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversations.ts @@ -31,7 +31,7 @@ export class StructuredAgentSessionConversations extends Map< deliverSettleEdge?: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger /** A conversation became held: state that waited on it (queued drafts) re-derives. */ - onOpened?: (sessionId: string) => void + onOpened?: (sessionId: string, journal: AgentSessionJournal) => void now: () => number } ) { @@ -86,7 +86,7 @@ export class StructuredAgentSessionConversations extends Map< }) this.activity.set(sessionId, this.delivery.now()) const adopted = super.set(sessionId, session) - this.delivery.onOpened?.(sessionId) + this.delivery.onOpened?.(sessionId, session.journal) return adopted } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index 2edd4704c04f..03d3e42168d4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -102,7 +102,7 @@ export class StructuredAgentSessionCurrentWork { /** Whether this process may hand this send over: accepted here, never by an earlier host process. * Derived from where the row sits, so nothing has to be written first; keeping an earlier - * process's send as a card is the startup share's bookkeeping (`holdUnsentSends`). */ + * process's send as a card is the reconciliation pass's bookkeeping (`holdUnsentSends`). */ handsOver(submission: AgentJournalSubmission): boolean { return structuredAgentSessionHandsOver(this.journal, submission) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts index fed005360303..76a37199f5d3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-lifetime.ts @@ -73,7 +73,7 @@ type ConversationCloseDeps = Pick & { /** What is still queued when the chat closes will not be handed over: a person's message is kept * as a card that waits for the chat's next turn, the rest rejected (`journal-unsent-send-hold.ts`). - * A quit is not a close: what it left, the next process's startup share keeps as cards. The chat + * A quit is not a close: what it left, the next process's reconciliation pass keeps as cards. The chat * stops running, so the reopen mark follows (`markStructuredQueueReopen`): on every `close`, or * only once it settled a send, for a later re-check of the same close, which must never mark past * a new send. `which` narrows it to the messages a close that did not complete closed. Best diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index cca51b41b70f..d20dee5218ff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -65,8 +65,9 @@ export class StructuredAgentSessionHost { }, deliverSettleEdge: (id, journal) => this.conversationDelivery.afterSettleEdge(id, journal), logger: sessionLogger.deferredStructuredAgentSessionLogger(() => this.deps.logger), - onOpened: (sessionId) => { + onOpened: (sessionId, journal) => { this.queued.drain.schedule(sessionId) + this.restore.reconciliation.noteOpened(sessionId, journal) // A reader holds it now: its settlement goes ahead of the rest of the startup scan. this.restore.reconciliation.prioritize(sessionId) }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts index 316a79400a46..0a8fd8434606 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts @@ -233,11 +233,13 @@ export async function createQueuedMessageTestRig( /** A host process that dies with no close: a new process loads the same state directory, its * records with it, and runs its startup as the runtime does before any client reaches it: the - * restart reconcile and its settlement, then the restore of the chat's tab. */ + * restart reconcile and its settlement, then the restore of the chat's tab, which decides a + * lease latched in recovery; that decision's release wakes the chat's worker once more. */ async function crashRestartHostProcess(beforeStartup?: () => unknown): Promise { await crashReloadHostProcess(beforeStartup) await host.restoreReadableSessions([SESSION]) await host.startupSettled() + await host.collaboratorsForTests().reconciliation.idle(SESSION) } /** The same crash, with only the restart reconcile and its settlement: no tab is restored. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index 734e5840c8f1..bab81658ebf3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -1,18 +1,20 @@ // One attempt of a chat's reconciliation worker, for a caller inside the chat's action lane: the // short recheck and the writes. Everything owed is re-derived here from the lease, this host's -// sight of an exit and the journal; only what cannot be derived rides in from the signals (an -// exit's own account, a proof the lease no longer holds, and the startup share not yet done). +// sight of an exit and the journal, so a pass that finds nothing owed writes nothing; only what +// cannot be derived rides in from the signals (an exit's own account, a proof the lease no longer +// holds). A lease latched in recovery owes nothing until its decision releases it. // // The steps stay apart, so one never retires another's debt: // (a) lease-release repair: an exit this host observed whose release write failed; -// (b) the startup share: the cards' repair and prune, and an earlier host process's unsent sends -// kept as cards; then, best effort, the queue's reopen mark after them and a rewind it left in -// doubt (neither is owed: the pause is derived from where the handle opened, and a rewind only -// its provider can prove waits for the next acquisition); +// (b) what an earlier host process left: the cards' repair and prune, and the unsent sends it +// accepted kept as cards; then, best effort, the queue's reopen mark and a rewind left in +// doubt (neither is owed: the pause is derived from where the handle opened, and a rewind +// only its provider can prove waits for an acquisition); // (c) item settlement (`settleStructuredAgentSessionLeftovers`), one planned batch. // An empty item plan says nothing of (a) or (b). import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -28,7 +30,10 @@ import { markStructuredQueueReopen, structuredAgentSessionHostInstance } from './structured-agent-session-queued-pause' -import { recoverStructuredRewind } from './structured-rewind-recovery' +import { + recoverStructuredRewind, + structuredRewindNeedsProvider +} from './structured-rewind-recovery' export type StructuredAgentSessionReconciliationPassContext = Pick< StructuredAgentSessionLifetimeContext, @@ -41,8 +46,6 @@ export type StructuredAgentSessionReconciliationDebts = { /** Each ended generation's proof, as its end carried it: a reservation clears the lease's, and a * later release replaces it with its own. */ evidence?: AgentSessionDeathEvidence[] - /** This process has not yet finished the chat's startup share. */ - startupShare?: { leaseSettled: boolean } } export type StructuredAgentSessionReconciliationPass = { @@ -52,30 +55,28 @@ export type StructuredAgentSessionReconciliationPass = { wrote: boolean } -/** `session`: the chat's open conversation, or the worker's own read of a closed one. */ +/** `session`: the chat's open conversation, or the worker's own read of a closed one. + * `processOpened`: where this host first opened the chat's journal; a send at or before it was an + * earlier process's. A later handle's own open cannot tell: this process may have accepted sends + * through an earlier handle, which its close or its delivery loop still owns. */ export async function runStructuredAgentSessionReconciliationPass( context: StructuredAgentSessionReconciliationPassContext, sessionId: string, session: Pick, - debts: StructuredAgentSessionReconciliationDebts + debts: StructuredAgentSessionReconciliationDebts, + processOpened: AgentJournalCursor ): Promise { const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false } + if (context.deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering') { + return pass + } if ( structuredAgentSessionEndedChildHoldsLease(context, sessionId) && (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId)) ) { pass.failed.push(new Error('agent_session_exit_release_owed')) } - // Only once the lease's recovery was decided (the worker's step before the lane): a share signalled - // mid-attempt waits for the next one. - if (debts.startupShare?.leaseSettled) { - const failedBefore = pass.failed.length - await runStartupShare(context, sessionId, session.journal, pass) - // Done only once every step landed; a failed conversion or repair is owed still. - if (pass.failed.length === failedBefore) { - delete debts.startupShare - } - } + await settleEarlierProcess(context, sessionId, session.journal, processOpened, pass) const settle = (proof?: AgentSessionDeathEvidence & { ownerFence: number }) => settleStructuredAgentSessionLeftovers({ store: context.deps.store, @@ -124,16 +125,24 @@ function heldProofs( .sort((left, right) => left.ownerFence - right.ownerFence) } -async function runStartupShare( +/** Each step finds its own debt in memory and writes only what it finds, so it is safe on every + * pass: the conversion, sends still queued from an earlier process; the mark, a converted send or + * a reopen floor this handle has not marked; the rewind, a record still in doubt. */ +async function settleEarlierProcess( context: StructuredAgentSessionReconciliationPassContext, sessionId: string, journal: StructuredAgentSessionHostSession['journal'], + processOpened: AgentJournalCursor, pass: StructuredAgentSessionReconciliationPass ): Promise { - const fence = context.deps.store.getRecord(sessionId)?.lease.runtimeFence - if (fence === undefined) { + const record = context.deps.store.getRecord(sessionId) + if (!record) { return } + const fence = record.lease.runtimeFence + const sameEpoch = processOpened.epoch === journal.cursor().epoch + const earlier = (sequence: number | undefined): boolean => + sameEpoch && sequence !== undefined && sequence <= processOpened.sequence const attempt = async (step: () => Promise): Promise => step().then( () => true, @@ -144,27 +153,31 @@ async function runStartupShare( ) await attempt(() => journal.queuedMessages.repairAndPrune()) // Before a Stop can withdraw one: a Stop never withdraws a card. - const converted = await attempt(() => - holdUnsentSends(journal, { - fence, - hostInstance: structuredAgentSessionHostInstance(), - hold: { cause: 'hostRestarted' } - }) - ) - // After every card the earlier process left, from where this handle opened, so a card this - // process queued is never held by it. Reported, never owed: the pause starts there anyway. - if (converted) { - const opened = journal.openedAt() - const since = opened.epoch === journal.cursor().epoch ? opened.sequence + 1 : undefined + let converted = false + await attempt(async () => { + converted = + (await holdUnsentSends(journal, { + fence, + hostInstance: structuredAgentSessionHostInstance(), + hold: { cause: 'hostRestarted', which: (entry) => earlier(entry.acceptedSequence) } + })) !== null + }) + // After every card the earlier process left, from where this process first opened the chat, so + // a card this one queued is never held by it. Reported, never owed: the pause starts there + // anyway, and a failed mark leaves the floor set, so the next pass marks again. + if (converted || journal.reopenFloor() !== null) { + const since = sameEpoch ? processOpened.sequence + 1 : undefined await markStructuredQueueReopen(sessionId, journal, fence, context.deps.logger, since) } - // A rewind the earlier process left prepared refuses every send until settled. With no provider - // here, one only its provider can prove stays for the next acquisition, which recovers it. - await recoverStructuredRewind(context.deps, sessionId, journal, fence).catch((error: unknown) => - context.deps.logger.warn('recovering a rewind an earlier process left did not finish', { - scope: 'reconciliation-rewind', - sessionId, - error - }) - ) + // A rewind left prepared refuses every send until settled. With no provider here, one only its + // provider can prove stays, quietly, for the next acquisition, which recovers it. + if (!structuredRewindNeedsProvider(record.rewind)) { + await recoverStructuredRewind(context.deps, sessionId, journal, fence).catch((error: unknown) => + context.deps.logger.warn('recovering a rewind an earlier process left did not finish', { + scope: 'reconciliation-rewind', + sessionId, + error + }) + ) + } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts index 6d70ee65028e..f843e38ee4fa 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts @@ -186,7 +186,11 @@ describe('a journal no retry can load', () => { ] as const)( '%s: the startup worker retires instead of replaying it again', async (_why, json) => { - rig = await createQueuedMessageTestRig({ restartable: true }) + // The owner is proven gone at the restart, so no recovery waits on a person's attach. + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => ({ outcome: 'pid-absent' }) + }) const current = rig await current.settleAccepted(await current.workingSend(), 'work') vi.spyOn(console, 'warn').mockImplementation(() => undefined) @@ -215,7 +219,7 @@ describe('a journal no retry can load', () => { ) }) -describe("the startup share's reopen mark", () => { +describe('the reopen mark after a restart', () => { // Bookkeeping, reported and never owed: the pause starts where the handle opened either way. it.each([false, true])( 'never holds the queue past the turn that lifts it (its write fails: %s)', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts index c24d25a1e070..32e3932b06bd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -4,7 +4,12 @@ // Signals: every committed revocation or supersession of a generation (the record store's // `onGenerationEnded`), an exit this host observed (whether or not its release landed), a row an // ended generation committed late, and startup. Each one re-derives what readers see at once (the -// operational revision, the published view, the queued-card drain), then wakes the worker. +// operational revision, the published view, the queued-card drain), then wakes the worker. Startup +// carries no debt: it only wakes every chat, since every pass derives what is owed. +// +// A lease latched in recovery is left alone: deciding it signals a process that may still run, so +// only a person's view or action on the chat does (the visible-tab restore, a start, an attach), +// and that decision's release is itself the signal that settles what the owner left. // // The worker coalesces: a chat has at most one attempt running and one timer waiting, and every // attempt re-derives everything owed (`runStructuredAgentSessionReconciliationPass`). A chat nobody @@ -18,7 +23,9 @@ // journal can never load, when the database is read-only, or at shutdown. Nothing a person does // (send, start, open, Stop, answer) ever waits on it. +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' import { @@ -46,8 +53,6 @@ export type StructuredAgentSessionReconciliationSignal = { exit?: StructuredAgentSessionExitSettlement /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ restate?: boolean - /** This process owes the chat its startup share. */ - startup?: true } export type StructuredAgentSessionReconciliationContext = @@ -59,8 +64,6 @@ export type StructuredAgentSessionReconciliationContext = track: (operation: Promise) => Promise /** Publishes what is current now and wakes the queued-card drain. */ publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void - /** Exits a lease latched in recovery when present-time evidence permits; a no-op otherwise. */ - resolveRecovery: (sessionId: string) => Promise } /** `done`: a pass found nothing owed, unless a signal came meanwhile; `retire`: nothing this @@ -82,6 +85,9 @@ type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { export class StructuredAgentSessionReconciliation { private readonly workers = new Map() + /** Where this host first opened each chat's journal: every send at or before it was accepted + * by an earlier host process, and every one after it by this one. */ + private readonly firstOpened = new Map() private disposed = false private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) private readonly unsubscribe: () => void @@ -108,9 +114,6 @@ export class StructuredAgentSessionReconciliation { if (signal.exit) { worker.debts.exit = signal.exit } - if (signal.startup && !worker.debts.startupShare) { - worker.debts.startupShare = { leaseSettled: false } - } if (worker.running) { worker.dirty = true } else if (worker.failures === 0 && !worker.timer) { @@ -127,6 +130,13 @@ export class StructuredAgentSessionReconciliation { } } + /** A handle this host opened on the chat's journal; only its first one marks the boundary. */ + noteOpened = (sessionId: string, journal: Pick): void => { + if (!this.firstOpened.has(sessionId)) { + this.firstOpened.set(sessionId, journal.openedAt()) + } + } + /** A reader opened the chat: whatever its worker still waits a slot for goes ahead of the scan. */ prioritize = (sessionId: string): void => { const worker = this.workers.get(sessionId) @@ -199,22 +209,13 @@ export class StructuredAgentSessionReconciliation { private async runAttempt(sessionId: string, worker: ChatWorker): Promise { try { const { store } = this.context.deps - if (this.disposed || store.readOnly || !store.getRecord(sessionId)) { + const record = store.getRecord(sessionId) + if (this.disposed || store.readOnly || !record) { return 'retire' } - const share = worker.debts.startupShare - // Once per startup share, before its pass: a lease latched in recovery is decided first, as - // the restore decides it, so the pass judges what is current. - if (share && !share.leaseSettled) { - const resolved = await this.slots.run(worker, () => - this.context.resolveRecovery(sessionId).catch((error: unknown) => { - this.warn(sessionId, error) - }) - ) - if (!resolved) { - return 'retire' - } - share.leaseSettled = true + // Nothing owed until its recovery is decided, whose release signals again: no journal read. + if (record.lease.handoffStage === 'recovering') { + return 'done' } if (!this.context.sessions.has(sessionId) && !worker.loaded) { const stop = await this.load(sessionId, worker) @@ -243,7 +244,8 @@ export class StructuredAgentSessionReconciliation { this.context, sessionId, session, - worker.debts + worker.debts, + this.firstOpened.get(sessionId) ?? session.journal.openedAt() ) if (pass.failed.length > 0) { this.warn(sessionId, pass.failed[0]) @@ -291,6 +293,7 @@ export class StructuredAgentSessionReconciliation { } if (loaded.kind === 'loaded') { worker.loaded = loaded.session + this.noteOpened(sessionId, loaded.session.journal) } // Skipped: a reader opened it meanwhile, and the pass uses their conversation. return loaded.kind === 'nothing' ? 'retire' : null diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index d45feb519a58..e32d55a85997 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -60,7 +60,7 @@ export async function revealStructuredAgentSession( /** The host's startup and its reconciliation: reconcile, then hand every chat to its reconciliation * worker in the background (no chat's tab or send waits on it), and the readable-restore sweep - * that resolves and opens each chat. Its lease bookkeeping is a reader's, which never fails a read + * that resolves and opens each visible chat. Its lease bookkeeping is a reader's, which never fails a read * or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, @@ -93,10 +93,7 @@ export function createStructuredAgentSessionHostRestore( // Tracked like a start, so a quit waits for an attempt before it closes what it writes to. track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => - startup.clientDelivery.publishGenerationEnded(sessionId, options), - // A lease latched in recovery is decided before the share's pass, as the restore decides it; - // a failure is the worker's to report, and the next attach or send resolves it. - resolveRecovery + startup.clientDelivery.publishGenerationEnded(sessionId, options) }) let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts index 6674d5345a2a..1ca8d381e323 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-start-failure-row.ts @@ -62,7 +62,7 @@ export async function recordStructuredAgentSessionStartFailure( recovered: true, mutations: [structuredAgentSessionStartFailureRow(startKey, failure)], rejectsQueued: { reason: failure.reason, rejection: failure.rejection }, - // The start was for this process's sends; an earlier process's is the startup share's to keep. + // The start was for this process's sends; an earlier process's is the reconciliation pass's to keep. rejectsQueuedOnly: (submission) => structuredAgentSessionHandsOver(journal, submission) }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts new file mode 100644 index 000000000000..1c7aae878ff4 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts @@ -0,0 +1,107 @@ +// What startup decides about a lease latched in recovery: nothing, for a chat nobody looks at. +// Deciding it signals a process that may still run, so only the visible-tab restore, a start or an +// attach does; that decision's release then wakes the chat's worker, which settles what it left. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + scanIdle as idle, + scanRecord as record, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' +const OWNER_PID = 12_000 + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +/** The earlier process's agent outlived it, until something stops it. */ +async function openWithSurvivingOwner() { + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) + await seedScanJournal(root, CHAT) + store = await openTestAgentSessionRecordStore(root) + let alive = true + const stopOwnerProcess = vi.fn((_pid: number, _signal: 'SIGTERM' | 'SIGKILL') => { + alive = false + }) + const probeOwner = async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + return { current: host, stopOwnerProcess } +} + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-startup-recovery-scope-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +describe('a hidden chat whose agent outlived the earlier process', () => { + it('is left alone at startup, and settled once an attach decides its recovery', async () => { + const { current, stopOwnerProcess } = await openWithSurvivingOwner() + const open = vi.spyOn(AgentSessionJournal.prototype, 'open') + + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [CHAT]) + + // No signal to a process that may still run, no lease change, and not even a read. + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease).toMatchObject({ + handoffStage: 'recovering', + runtimeFence: 13, + ownerProcess: { pid: OWNER_PID } + }) + expect(open).not.toHaveBeenCalled() + + // The person attaches: recovery is decided first, as on every build, stopping the owner. + const params = attachParamsForRecord(store.getRecord(CHAT)!, { + clientOperationId: 'attach-1', + expectedRuntimeFence: 13 + }) + await current.attach({ callerKey: 'client-1' }, params).catch(() => undefined) + + expect(stopOwnerProcess).toHaveBeenCalledWith(OWNER_PID, 'SIGTERM') + expect(store.getRecord(CHAT)?.lease).toMatchObject({ handoffStage: null, ownerProcess: null }) + // Its release woke the worker, which settles the turn the owner left by that proof. + await idle(current, [CHAT]) + expect(await turnState(current, CHAT)).toBe('interrupted') + }) +}) + +describe('a visible chat whose agent outlived the earlier process', () => { + it('has its owner stopped by the visible-tab restore, as before', async () => { + const { current, stopOwnerProcess } = await openWithSurvivingOwner() + + await current.reconcileRestartLeases() + await current.restoreReadableSessions([CHAT]) + + expect(stopOwnerProcess).toHaveBeenCalledWith(OWNER_PID, 'SIGTERM') + expect(store.getRecord(CHAT)?.lease).toMatchObject({ handoffStage: null, ownerProcess: null }) + await current.startupSettled() + await idle(current, [CHAT]) + expect(await turnState(current, CHAT)).toBe('interrupted') + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts index 29d6c7e4df79..f199a855d9b1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts @@ -17,6 +17,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' import type { StructuredAgentSessionHost } from './structured-agent-session-host' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { @@ -53,14 +54,15 @@ async function seed(sessionIds: readonly string[], extra: Parameters Promise = async () => undefined): { opens: number } { const counter = { opens: 0 } const open = AgentSessionJournal.prototype.open vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( this: AgentSessionJournal ) { counter.opens += 1 + await hold() return open.call(this) }) return counter @@ -89,27 +91,20 @@ describe('a quit during the startup scan', () => { const current = openHost() let release: () => void = () => undefined const gate = new Promise((resolve) => (release = resolve)) - const { runtimeState } = current.collaboratorsForTests() - const resolve = runtimeState.resolveRecovery.bind(runtimeState) - let recoveries = 0 - vi.spyOn(runtimeState, 'resolveRecovery').mockImplementation(async (sessionId) => { - recoveries += 1 - await gate - return resolve(sessionId) - }) + const counter = countOpens(() => gate) + const passes = vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune') await current.reconcileRestartLeases() - await vi.waitFor(() => expect(recoveries).toBe(4)) - const counter = countOpens() + await vi.waitFor(() => expect(counter.opens).toBe(4)) const quit = current.flushAllStreamedEvents() release() await quit host = undefined - // The four already recovering finished that step and went no further; the six still waiting + // The four already reading finished that read and went no further; the six still waiting // for a slot never started. - expect(recoveries).toBe(4) - expect(counter.opens).toBe(0) + expect(counter.opens).toBe(4) + expect(passes).not.toHaveBeenCalled() }) }) @@ -191,7 +186,7 @@ describe('a startup worker whose step cannot succeed', () => { expect(warnings).toContain("a chat's history cannot be loaded, so nothing is settled") }) - it('leaves a rewind only its provider can prove for the next acquisition, and retires', async () => { + it('leaves a rewind only its provider can prove for the next acquisition, quietly, and retires', async () => { const rewind = { operationId: 'op-rewind', callerKey: 'caller', @@ -213,7 +208,8 @@ describe('a startup worker whose step cannot succeed', () => { expect(store.getRecord(CHAT)?.rewind?.phase).toBe('prepared') expect(current.collaboratorsForTests().sessions.has(CHAT)).toBe(false) - expect(warnings).toEqual(['recovering a rewind an earlier process left did not finish']) + // Not tried here at all: without its provider it cannot succeed. + expect(warnings).toEqual([]) expect(await turnState(current, CHAT)).toBe('unverifiable') }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts index c3e7cad4e159..3390069806b0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts @@ -11,6 +11,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-ho import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' import { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' import { createStructuredAgentSessionLogger, @@ -124,7 +125,7 @@ export function openScanHost( options: { logger?: StructuredAgentSessionLogger statusSink?: StructuredAgentSessionStatusSink - } = {} + } & Partial> = {} ): StructuredAgentSessionHost { return new StructuredAgentSessionHost({ agents: NO_STRUCTURED_AGENTS, @@ -141,7 +142,8 @@ export function openScanHost( journalDatabase: openTestJournalHostDatabase(root), claimKeyId: 'key-1', ...(options.statusSink ? { statusSink: options.statusSink } : {}), - probeOwner: async () => ({ outcome: 'pid-absent' }), + ...(options.stopOwnerProcess ? { stopOwnerProcess: options.stopOwnerProcess } : {}), + probeOwner: options.probeOwner ?? (async () => ({ outcome: 'pid-absent' })), now: () => SCAN_NOW }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts index 965cc5cd019d..6403eae1f1c9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts @@ -15,6 +15,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' import type { StructuredAgentSessionHost } from './structured-agent-session-host' import { openScanHost, @@ -36,9 +37,39 @@ function openHost(): StructuredAgentSessionHost { return host } -/** Whose journal a call ran on: the scan's own read is no conversation the host indexed. */ -function journalSession(journal: AgentSessionJournal): string { - return journal.snapshot().sessionId +/** The chats each pass ran on, in order: every pass repairs the cards an earlier handle left. */ +function watchPasses(): string[] { + const passes: string[] = [] + const repair = JournalQueuedMessages.prototype.repairAndPrune + vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune').mockImplementation(function ( + this: JournalQueuedMessages + ) { + passes.push(this.sessionId) + return repair.call(this) + }) + return passes +} + +/** Holds the first four journal opens (every background slot); `opened` lists each chat whose + * open began, in order. */ +function gateFirstOpens(): { opened: string[]; releaseOne: () => void; release: () => void } { + const held: (() => void)[] = [] + const opened: string[] = [] + const open = AgentSessionJournal.prototype.open + vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( + this: AgentSessionJournal + ) { + opened.push(this.queuedMessages.sessionId) + if (opened.length <= 4) { + await new Promise((resolve) => held.push(resolve)) + } + return open.call(this) + }) + return { + opened, + releaseOne: () => held.shift()?.(), + release: () => held.splice(0).forEach((resolve) => resolve()) + } } beforeEach(async () => { @@ -61,20 +92,7 @@ describe('the startup scan', () => { await seedScanJournal(root, sessionId) } store = await openTestAgentSessionRecordStore(root) - // The first opens the scan makes hold every background slot until the test lets them go. - let release: () => void = () => undefined - const gate = new Promise((resolve) => (release = resolve)) - const open = AgentSessionJournal.prototype.open - let gated = 0 - vi.spyOn(AgentSessionJournal.prototype, 'open').mockImplementation(async function ( - this: AgentSessionJournal - ) { - gated += 1 - if (gated <= 4) { - await gate - } - return open.call(this) - }) + const { opened, release } = gateFirstOpens() const current = openHost() const indexed: string[] = [] const { sessions, serialize } = current.collaboratorsForTests() @@ -83,18 +101,10 @@ describe('the startup scan', () => { indexed.push(sessionId) return set(sessionId, session) }) - const shares: string[] = [] - const mark = AgentSessionJournal.prototype.markQueueReopen - vi.spyOn(AgentSessionJournal.prototype, 'markQueueReopen').mockImplementation(async function ( - this: AgentSessionJournal, - ...args - ) { - shares.push(journalSession(this)) - return mark.apply(this, args) - }) + const shares = watchPasses() await current.reconcileRestartLeases() - await vi.waitFor(() => expect(gated).toBe(4)) + await vi.waitFor(() => expect(opened).toHaveLength(4)) // The person's own actions on the late chat run now, ahead of its share. const order: string[] = [] @@ -125,31 +135,23 @@ describe('the startup scan', () => { await seedScanJournal(root, sessionId) } store = await openTestAgentSessionRecordStore(root) + const { opened, releaseOne, release } = gateFirstOpens() const current = openHost() - // The first four chats' recovery holds every background slot until the test lets it go. - let release: () => void = () => undefined - const gate = new Promise((resolve) => (release = resolve)) - const { runtimeState } = current.collaboratorsForTests() - const resolve = runtimeState.resolveRecovery.bind(runtimeState) - const resolved: string[] = [] - vi.spyOn(runtimeState, 'resolveRecovery').mockImplementation(async (sessionId) => { - resolved.push(sessionId) - if (resolved.length <= 4) { - await gate - } - return resolve(sessionId) - }) + const passes = watchPasses() await current.reconcileRestartLeases() - await vi.waitFor(() => expect(resolved).toHaveLength(4)) - expect(resolved).not.toContain(LATE) + await vi.waitFor(() => expect(opened).toHaveLength(4)) + expect(opened).not.toContain(LATE) // The person opens the late chat while two chats still wait for a slot, it among them. expect(await turnState(current, LATE)).toBe('running') + // One slot comes free: the late chat takes it ahead of the chat queued before it. + releaseOne() + await vi.waitFor(() => expect(passes).toContain(LATE)) release() await current.startupSettled() await idle(current, all) - expect(resolved.slice(4)).toEqual([LATE, CHATS[4]]) + expect(passes.indexOf(LATE)).toBeLessThan(passes.indexOf(CHATS[4]!)) expect(await turnState(current, LATE)).toBe('unverifiable') }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts index ee3607bc6f0a..7c4330118e2d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -1,14 +1,13 @@ // What the host owes its chats when it comes up as a new instance: what the earlier process left. // Run after the whole-host restart reconcile, in the background, for EVERY chat on record: no -// stored "settled" mark says which are owed, so each is derived from its journal. +// stored "settled" mark says which are owed, so each chat's worker derives it from its journal +// (`structured-agent-session-reconciliation-pass.ts`). The scan only wakes the workers. // -// Each chat goes to its reconciliation worker with its startup share owed -// (`structured-agent-session-reconciliation-pass.ts`): the worker takes a background slot outside -// the chat's action lane, replays a closed chat's journal as its own read (published to no status -// surface: only a chat a reader or the restorer opens is), and enters the lane only for the short -// recheck and its writes. So a send, a start or a read of a chat late in the scan never waits -// behind its share, and a chat a reader opens moves ahead of the rest. A share that fails is not -// done: the worker retries it, up to its bound. +// It never decides a lease latched in recovery: that signals a process that may still run, which +// only the visible-tab restore, a start or an attach does, as on every build before this one. +// Their release then wakes the worker again. A worker replays a closed chat in a background slot as +// its own read, published to no status surface, and takes the chat's lane only to write, so a send, +// start or read of a chat late in the scan never waits behind it, and an opened chat goes first. import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionReconciliation } from './structured-agent-session-reconciliation-worker' @@ -20,7 +19,7 @@ export function scanStructuredAgentSessionsAtStartup( ): Promise { const sessionIds = store.listRecords().map((record) => record.sessionId) for (const sessionId of sessionIds) { - reconciliation.signal(sessionId, { startup: true }) + reconciliation.signal(sessionId) } return Promise.all(sessionIds.map((sessionId) => reconciliation.attempted(sessionId))).then( () => undefined diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index 7b00c0fca222..f8bc1afa6716 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -150,14 +150,16 @@ function openHost(overrides: Partial = {}): void }) } -/** Startup as the runtime runs it: the restart reconcile, every chat's share in the background, - * and the boot sweep that opens each chat for its readers once that share closed it again. */ +/** Startup as the runtime runs it: the restart reconcile, every chat's worker in the background, + * and the boot sweep that opens each chat for its readers once that worker closed it again. A + * lease latched in recovery is decided by the sweep, whose release wakes the worker again. */ async function startUp(): Promise { await host.reconcileRestartLeases() await host.startupSettled() await reconciled() await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) await host.restoreReadableSessions() + await reconciled() } /** The chat's reconciliation worker has retired, or waits out a backoff. */ @@ -444,7 +446,7 @@ describe('already-wedged profiles become usable on load', () => { expect(await host.attach(CALLER, hostTestAttachParams(13))).toMatchObject({ ok: true }) // Acquiring a released lease ends nothing (design §9.8): the boot sweep settles what it left. - host.collaboratorsForTests().reconciliation.signal(SESSION, { startup: true }) + host.collaboratorsForTests().reconciliation.signal(SESSION) await reconciled() expect(acquire).toHaveBeenCalledOnce() diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts index b0320ffdcbcf..ca5f9ca393e3 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts @@ -66,6 +66,17 @@ async function settleUnsupportedClaudeRewind( } } +/** A rewind in doubt that only its provider's history can settle: an unverified Codex revert. */ +export function structuredRewindNeedsProvider( + rewind: AgentSessionRewindRecord | undefined +): boolean { + if (rewind?.phase !== 'provider-succeeded' && rewind?.phase !== 'prepared') { + return false + } + const target = parseAgentJournalItemKey(rewind.providerItemId ?? rewind.itemId) + return target?.provider === 'codex' && !rewind.hydrationVerified +} + /** Recovery observes provider state; it never repeats an ambiguous native mutation. */ export async function recoverStructuredRewind( deps: RewindRecoveryDeps, From f4ca43eb8ffe6f6a74b464b889f375b11dc03829 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:46:18 -0700 Subject: [PATCH 06/22] chore(native-chat): name the first-open cursor as process-scoped --- .../structured-agent-session-reconciliation-worker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts index 32e3932b06bd..78cbc241061c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -85,8 +85,8 @@ type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { export class StructuredAgentSessionReconciliation { private readonly workers = new Map() - /** Where this host first opened each chat's journal: every send at or before it was accepted - * by an earlier host process, and every one after it by this one. */ + /** Process-scoped, never persisted: where this host process first opened each chat's journal. + * Every send at or before it was accepted by an earlier host process, every one after by this. */ private readonly firstOpened = new Map() private disposed = false private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) From a0723052589ee9c241870ec857086cee561e9030 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 14:54:35 -0700 Subject: [PATCH 07/22] fix(native-chat): an earlier process's share runs whatever the lease; a stale card's Cancel matches main - while a lease is in recovery only the release repair and the item settlement wait; unsent sends from an earlier process become cards at once, as on open before this change - the reopen mark starts at this handle's floor in the current epoch - a card an ended generation raised: a dismiss route dismisses it only; with no route a Cancel naming the live turn interrupts that turn, never the chat-wide Stop - the answer refusal names the agent only when its agent stopped --- .../queued-message-reopen-floor.test.ts | 33 ++++ .../queued-message-reopen-floor.ts | 14 ++ ...-agent-session-ended-prompt-cancel.test.ts | 145 ++++++++++++------ ...tured-agent-session-leftover-settlement.ts | 3 +- .../structured-agent-session-prompt-cancel.ts | 44 ++++-- ...tured-agent-session-reconciliation-pass.ts | 24 +-- ...red-agent-session-reconciliation-worker.ts | 23 +-- ...ent-session-startup-recovery-scope.test.ts | 42 ++++- ...agent-session-startup-scan.test-fixture.ts | 7 +- .../structured-agent-session-turns-cancel.ts | 15 +- .../agent-session-refusal-notice.test.ts | 14 ++ src/shared/agent-session-refusal-notice.ts | 2 +- 12 files changed, 259 insertions(+), 107 deletions(-) create mode 100644 src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts diff --git a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts new file mode 100644 index 000000000000..c9d81ccdb13c --- /dev/null +++ b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.test.ts @@ -0,0 +1,33 @@ +// Where the reopen mark of what an earlier host process left starts. + +import { describe, expect, it } from 'vitest' +import { queuedMessageReopenMarkStart } from './queued-message-reopen-floor' + +const CURRENT = 'epoch-2' + +describe('the reopen mark after a restart', () => { + it("starts at this handle's floor when it is in the current epoch", () => { + expect( + queuedMessageReopenMarkStart( + { epoch: CURRENT, sequence: 9 }, + { epoch: CURRENT, sequence: 4 }, + CURRENT + ) + ).toBe(9) + }) + + it('starts just past where this process first opened the chat when no floor is set', () => { + expect(queuedMessageReopenMarkStart(null, { epoch: CURRENT, sequence: 4 }, CURRENT)).toBe(5) + }) + + it('is not written from a floor an older epoch left, so it never holds cards queued since', () => { + // A failed startup mark, then a rewind that moved the epoch: both cursors are the old epoch's. + expect( + queuedMessageReopenMarkStart( + { epoch: 'epoch-1', sequence: 9 }, + { epoch: 'epoch-1', sequence: 8 }, + CURRENT + ) + ).toBeNull() + }) +}) diff --git a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts index 5d97e175d71b..bfce00c8efac 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts @@ -14,3 +14,17 @@ export function queuedMessageReopenFloor( ? { epoch: opened.epoch, sequence: opened.sequence + 1 } : null } + +/** Where the reopen mark of what an earlier host process left starts: this handle's unmarked + * floor, else just past where this process first opened the chat; null when neither is in the + * journal's current epoch, since a mark from now would hold cards this process queued since. */ +export function queuedMessageReopenMarkStart( + floor: AgentJournalCursor | null, + processOpened: AgentJournalCursor, + epoch: string +): number | null { + if (floor?.epoch === epoch) { + return floor.sequence + } + return processOpened.epoch === epoch ? processOpened.sequence + 1 : null +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts index 1c9b509d6bc8..e0157bf6d8cc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts @@ -1,6 +1,8 @@ -// A Cancel that names a card an ended generation raised, from a client that still shows it: the -// card is dismissed in the journal alone, no provider is asked about it, and the live generation's -// turn is stopped only by a request that names it (an older phone's Stop carries the card it shows). +// A Cancel that names a card an ended generation raised, from a client that still shows it. As on +// main: the card is dismissed in the journal alone, and a provider whose card Cancel routes +// (Claude, Pi) stops nothing; one with no route (Codex, ACP) interrupts the live turn the request +// names through that turn's own cancel, never the chat's Stop, so queued messages stay and no Stop +// event pauses the queue. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -17,6 +19,7 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' import type { StructuredAgentSessionChatStopRun } from './structured-agent-session-chat-stop' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { cancelStructuredAgentSessionPrompt } from './structured-agent-session-prompt-cancel' import { performCancel, type AgentSessionTurnContext } from './structured-agent-session-turns' @@ -42,8 +45,11 @@ afterEach(async () => { } }) -/** Generation 1 raised a card and ended without its cleanup; generation 2 runs `turn-2`. */ -async function endedCardAndLiveTurn(): Promise<{ journal: AgentSessionJournal; itemId: string }> { +/** Generation 1 raised a card and ended without its cleanup; generation 2 runs `turn-2`, with a + * message queued behind it. */ +async function endedCardAndLiveTurn( + kind: 'approval' | 'question' = 'approval' +): Promise<{ journal: AgentSessionJournal; itemId: string }> { root = await mkdtemp(join(tmpdir(), 'orca-ended-prompt-cancel-')) const journal = await journals.open({ identity: IDENTITY, stateDirectory: root }) const turn = (turnId: string, fence: number) => @@ -53,21 +59,48 @@ async function endedCardAndLiveTurn(): Promise<{ journal: AgentSessionJournal; i { fence, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) const old = await turn('turn-1', ENDED) + const pending = { + state: 'pending' as const, + selectedOptionId: null, + resolvedBy: null, + resolvedAt: null + } const card = await journal.appendItem( { provider: 'codex', threadId: 'thread-1', turnId: 'turn-1', ordinal: 1 }, - { - kind: 'approval', - title: 'Approve?', - detail: null, - options: [{ id: 'allow', label: 'Allow' }], - resolution: { state: 'pending', selectedOptionId: null, resolvedBy: null, resolvedAt: null } - }, + kind === 'approval' + ? { + kind: 'approval', + title: 'Approve?', + detail: null, + options: [{ id: 'allow', label: 'Allow' }], + resolution: pending + } + : { + kind: 'question', + question: 'Which?', + options: [{ id: 'a', label: 'A' }], + resolution: pending + }, { fence: ENDED, turnScope: { kind: 'turn', turnItemId: old.itemId } } ) await turn('turn-2', LIVE) + await journal.queuedMessages.insert({ + messageId: 'queued-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, + fingerprint: 'queued-1', + hostInstance: 'host-1' + }) return { journal, itemId: card.itemId } } +/** The queue as the live turn left it: its message waiting, and nothing pausing it. */ +function queueUntouched(journal: AgentSessionJournal): void { + expect(journal.queuedMessages.list().map((card) => [card.messageId, card.state])).toEqual([ + ['queued-1', 'waiting'] + ]) + expect(structuredQueuePauses(journal)).toEqual([]) +} + function provider(routePromptCancel?: StructuredAgentSessionAdapter['routePromptCancel']) { return { cancelTurn: vi.fn(async (_input: { turnId?: string; prompt?: unknown }) => ({ @@ -107,71 +140,83 @@ function chatStop() { function resolution(journal: AgentSessionJournal, itemId: string) { const body = journal.snapshot().items.find((item) => item.itemId === itemId)?.body - return body?.kind === 'approval' ? body.resolution.state : undefined + return body?.kind === 'approval' || body?.kind === 'question' ? body.resolution.state : undefined } describe('a card an ended generation raised, cancelled from a client that still shows it', () => { + const pi = () => ({ kind: 'dismiss' as const }) it.each([ - ['naming no turn', undefined], - ["naming the ended generation's turn", 'turn-1'] - ])( - 'ACP (no card route): %s, it is dismissed and the live turn is not interrupted', - async (_name, turnId) => { - const { journal, itemId } = await endedCardAndLiveTurn() - const acp = provider() + ['Claude approval', 'approval', claudePromptCancelRoute], + ['Claude question', 'question', claudePromptCancelRoute], + ['Pi (every card dismisses)', 'approval', pi] + ] as const)( + '%s: dismissed only, even when the request names the live turn', + async (_name, kind, route) => { + const { journal, itemId } = await endedCardAndLiveTurn(kind) + const agent = provider(route) const stop = chatStop() const interrupt = vi.fn() const result = await cancelStructuredAgentSessionPrompt( - context(journal, acp), - { ...(turnId ? { turnId } : {}), prompt: { itemId, expectedRevision: 1 } }, + context(journal, agent), + { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } }, { stop, interrupt } ) - expect(result).toMatchObject({ ok: true, value: { cancelled: true } }) + expect(result).toMatchObject({ ok: true, value: { turnId: 'turn-2', cancelled: true } }) expect(resolution(journal, itemId)).toBe('cancelled') + expect(stop).not.toHaveBeenCalled() expect(interrupt).not.toHaveBeenCalled() + expect(agent.cancelTurn).not.toHaveBeenCalled() + expect(agent.dismissPrompt).not.toHaveBeenCalled() + queueUntouched(journal) + } + ) + + it.each(['Codex', 'ACP'])( + '%s (no card route): the named live turn is interrupted through its own cancel', + async () => { + const { journal, itemId } = await endedCardAndLiveTurn() + const agent = provider() + const stop = chatStop() + const ctx = context(journal, agent) + const input = { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } } + + const result = await cancelStructuredAgentSessionPrompt(ctx, input, { + stop, + // What the host's plan runs: the turn's cancel, with the card, as on main. + interrupt: () => performCancel(ctx, { clientOperationId: 'cancel-1', ...input }) + }) + + expect(result).toMatchObject({ ok: true }) + expect(agent.cancelTurn).toHaveBeenCalledOnce() + expect(agent.cancelTurn.mock.calls[0]?.[0]).toMatchObject({ turnId: 'turn-2' }) expect(stop).not.toHaveBeenCalled() - expect(acp.cancelTurn).not.toHaveBeenCalled() - expect(acp.dismissPrompt).not.toHaveBeenCalled() + expect(resolution(journal, itemId)).toBe('cancelled') + queueUntouched(journal) } ) it.each([ - ['Claude (an approval routes to a dismissal)', claudePromptCancelRoute], - ['Codex (no card route)', undefined] - ])("%s: an older phone's Stop carrying it still stops the live turn", async (_name, route) => { + ['naming no turn', undefined], + ["naming the ended generation's turn", 'turn-1'] + ])('Codex or ACP, %s: dismissed only, and the live turn runs on', async (_name, turnId) => { const { journal, itemId } = await endedCardAndLiveTurn() - const agent = provider(route) + const agent = provider() const stop = chatStop() const interrupt = vi.fn() const result = await cancelStructuredAgentSessionPrompt( context(journal, agent), - { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } }, + { ...(turnId ? { turnId } : {}), prompt: { itemId, expectedRevision: 1 } }, { stop, interrupt } ) - expect(result).toMatchObject({ ok: true, value: { turnId: 'turn-2', cancelled: true } }) - expect(stop).toHaveBeenCalledOnce() - expect(interrupt).not.toHaveBeenCalled() - expect(agent.dismissPrompt).not.toHaveBeenCalled() - expect(resolution(journal, itemId)).toBe('cancelled') - }) - - it('reaching the Stop itself, the card is dropped and the live turn is stopped plainly', async () => { - const { journal, itemId } = await endedCardAndLiveTurn() - const codex = provider() - - await performCancel(context(journal, codex), { - clientOperationId: 'cancel-1', - turnId: 'turn-2', - prompt: { itemId, expectedRevision: 1 } - }) - - expect(codex.cancelTurn).toHaveBeenCalledOnce() - expect(codex.cancelTurn.mock.calls[0]?.[0]).not.toHaveProperty('prompt') - expect(codex.cancelTurn.mock.calls[0]?.[0]).toMatchObject({ turnId: 'turn-2' }) + expect(result).toMatchObject({ ok: true, value: { cancelled: true } }) expect(resolution(journal, itemId)).toBe('cancelled') + expect(interrupt).not.toHaveBeenCalled() + expect(stop).not.toHaveBeenCalled() + expect(agent.cancelTurn).not.toHaveBeenCalled() + queueUntouched(journal) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts index 9071328594da..eb0eb957e981 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts @@ -100,7 +100,8 @@ export async function settleStructuredAgentSessionLeftovers( sessionId, fence, acquisitionGeneration: null, - // Read with the rows: the evidence the release before this settlement wrote. + // Read with the rows: the proof the lease holds now. On an observed exit this settlement + // runs before the release, so the exit's own account (`exit`) judges instead. deathEvidence: proof ?? input.store.getRecord(sessionId)?.lease.deathEvidence ?? null, failureTextContext, below, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts index 2d50173ff745..6a51e0781b4e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts @@ -33,13 +33,13 @@ export async function cancelStructuredAgentSessionPrompt( if (!validated.ok) { return answerCancelOfSettledPrompt(ctx, input, validated) } - if (isEndedGenerationPrompt(ctx, validated)) { - return cancelEndedGenerationPrompt(ctx, input, validated, routes.stop) - } const route = ctx.adapter.routePromptCancel?.({ sessionId: ctx.sessionId, prompt: validated.prompt }) + if (isEndedGenerationPrompt(ctx, validated)) { + return cancelEndedGenerationPrompt(ctx, input, validated, route ? null : routes.interrupt) + } if (!route) { return routes.interrupt() } @@ -70,23 +70,41 @@ export async function cancelStructuredAgentSessionPrompt( } /** A card an ended generation raised: no live provider holds it, so it is dismissed in the journal - * alone and the live turn is not this card's. A request that also names the live turn (an older - * phone's Stop carries the card it shows) is that turn's Stop as well. */ + * alone. As on main, a provider whose card Cancel routes (a dismissal, or a Stop of the turn that + * raised it) stops nothing, since the live turn did not raise it; one with no route interrupts the + * turn the request names when that turn is live, through the turn's own cancel, never the chat's + * Stop, so queued messages stay and the queue is not paused. */ async function cancelEndedGenerationPrompt( ctx: AgentSessionTurnContext, input: { turnId?: string }, pending: PendingPrompt, - stop: () => Promise + interrupt: (() => Promise) | null ): Promise { - const dismissed = await dismissEndedGenerationPrompt(ctx, pending) - if (!dismissed.ok) { - return dismissed - } const liveTurnId = contextStructuredAgentSessionCurrentWork(ctx).activeTurnId() - if (input.turnId !== undefined && input.turnId === liveTurnId) { - return (await stop()).outcome + const interrupted = + interrupt && input.turnId !== undefined && input.turnId === liveTurnId + ? await interrupt() + : null + if (interrupted && !interrupted.ok) { + return interrupted } - return { ok: true, value: { ...(input.turnId ? { turnId: input.turnId } : {}), cancelled: true } } + // The interrupt's provider may have let go of it already; what is still pending is dismissed. + const still = validatePendingPrompt(ctx, { + itemId: pending.item.itemId, + expectedRevision: pending.item.revision + }) + if (still.ok) { + const dismissed = await dismissEndedGenerationPrompt(ctx, still) + if (!dismissed.ok) { + return dismissed + } + } + return ( + interrupted ?? { + ok: true, + value: { ...(input.turnId ? { turnId: input.turnId } : {}), cancelled: true } + } + ) } function raisedByLiveTurn(ctx: AgentSessionTurnContext, pending: PendingPrompt): boolean { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index bab81658ebf3..79abde80c42a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -2,7 +2,8 @@ // short recheck and the writes. Everything owed is re-derived here from the lease, this host's // sight of an exit and the journal, so a pass that finds nothing owed writes nothing; only what // cannot be derived rides in from the signals (an exit's own account, a proof the lease no longer -// holds). A lease latched in recovery owes nothing until its decision releases it. +// holds). A lease latched in recovery still gets (b), which touches no process and no lease, as +// every open did on main; (a) and (c) wait for its decision, whose release signals again. // // The steps stay apart, so one never retires another's debt: // (a) lease-release repair: an exit this host observed whose release write failed; @@ -14,6 +15,7 @@ // An empty item plan says nothing of (a) or (b). import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import { queuedMessageReopenMarkStart } from '../agent-session-journal/queued-message-reopen-floor' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -67,16 +69,19 @@ export async function runStructuredAgentSessionReconciliationPass( processOpened: AgentJournalCursor ): Promise { const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false } - if (context.deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering') { - return pass - } + const recovering = context.deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' if ( + !recovering && structuredAgentSessionEndedChildHoldsLease(context, sessionId) && (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId)) ) { pass.failed.push(new Error('agent_session_exit_release_owed')) } await settleEarlierProcess(context, sessionId, session.journal, processOpened, pass) + if (recovering) { + // What ended is not known until its recovery is decided; the debts wait for that signal. + return pass + } const settle = (proof?: AgentSessionDeathEvidence & { ownerFence: number }) => settleStructuredAgentSessionLeftovers({ store: context.deps.store, @@ -162,11 +167,12 @@ async function settleEarlierProcess( hold: { cause: 'hostRestarted', which: (entry) => earlier(entry.acceptedSequence) } })) !== null }) - // After every card the earlier process left, from where this process first opened the chat, so - // a card this one queued is never held by it. Reported, never owed: the pause starts there - // anyway, and a failed mark leaves the floor set, so the next pass marks again. - if (converted || journal.reopenFloor() !== null) { - const since = sameEpoch ? processOpened.sequence + 1 : undefined + // After every card the earlier process left, so a card this one queued is never held by it. + // Reported, never owed: the pause starts there anyway, and a failed mark leaves the floor set, + // so the next pass marks again. + const floor = journal.reopenFloor() + const since = queuedMessageReopenMarkStart(floor, processOpened, journal.cursor().epoch) + if ((converted || floor !== null) && since !== null) { await markStructuredQueueReopen(sessionId, journal, fence, context.deps.logger, since) } // A rewind left prepared refuses every send until settled. With no provider here, one only its diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts index 78cbc241061c..7605c53f36a2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -7,9 +7,10 @@ // operational revision, the published view, the queued-card drain), then wakes the worker. Startup // carries no debt: it only wakes every chat, since every pass derives what is owed. // -// A lease latched in recovery is left alone: deciding it signals a process that may still run, so -// only a person's view or action on the chat does (the visible-tab restore, a start, an attach), -// and that decision's release is itself the signal that settles what the owner left. +// The worker never decides a lease latched in recovery: that signals a process that may still +// run, which only a person's view or action on the chat does (the visible-tab restore, a start, an +// attach). It still keeps what the earlier process left as cards; the decision's release is the +// signal that settles the rest, so a worker retiring meanwhile parks the debts it held for it. // // The worker coalesces: a chat has at most one attempt running and one timer waiting, and every // attempt re-derives everything owed (`runStructuredAgentSessionReconciliationPass`). A chat nobody @@ -85,6 +86,8 @@ type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { export class StructuredAgentSessionReconciliation { private readonly workers = new Map() + /** Debts a retired worker still held (a lease in recovery, or a run given up), for the next. */ + private readonly parked = new Map() /** Process-scoped, never persisted: where this host process first opened each chat's journal. * Every send at or before it was accepted by an earlier host process, every one after by this. */ private readonly firstOpened = new Map() @@ -180,7 +183,7 @@ export class StructuredAgentSessionReconciliation { let worker = this.workers.get(sessionId) if (!worker) { worker = { - debts: {}, + debts: this.parked.get(sessionId) ?? {}, running: false, dirty: false, timer: null, @@ -188,6 +191,7 @@ export class StructuredAgentSessionReconciliation { attempted: [], urgent: false } + this.parked.delete(sessionId) this.workers.set(sessionId, worker) } return worker @@ -209,14 +213,9 @@ export class StructuredAgentSessionReconciliation { private async runAttempt(sessionId: string, worker: ChatWorker): Promise { try { const { store } = this.context.deps - const record = store.getRecord(sessionId) - if (this.disposed || store.readOnly || !record) { + if (this.disposed || store.readOnly || !store.getRecord(sessionId)) { return 'retire' } - // Nothing owed until its recovery is decided, whose release signals again: no journal read. - if (record.lease.handoffStage === 'recovering') { - return 'done' - } if (!this.context.sessions.has(sessionId) && !worker.loaded) { const stop = await this.load(sessionId, worker) if (stop) { @@ -366,6 +365,10 @@ export class StructuredAgentSessionReconciliation { } worker.timer = null this.workers.delete(sessionId) + const { evidence, exit } = worker.debts + if (!this.disposed && (evidence || exit) && this.context.deps.store.getRecord(sessionId)) { + this.parked.set(sessionId, worker.debts) + } if (!worker.running) { this.dropLoaded(worker) worker.attempted.splice(0).forEach((resolve) => resolve()) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts index 1c7aae878ff4..a04f69c93655 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts @@ -1,6 +1,7 @@ // What startup decides about a lease latched in recovery: nothing, for a chat nobody looks at. // Deciding it signals a process that may still run, so only the visible-tab restore, a start or an // attach does; that decision's release then wakes the chat's worker, which settles what it left. +// What the earlier process left unsent is kept as a card meanwhile, as every open did on main. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -13,7 +14,6 @@ import { seedTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { attachParamsForRecord } from './structured-agent-session-conversation-open' import type { StructuredAgentSessionHost } from './structured-agent-session-host' import { @@ -32,9 +32,9 @@ let store: AgentSessionRecordStore let host: StructuredAgentSessionHost | undefined /** The earlier process's agent outlived it, until something stops it. */ -async function openWithSurvivingOwner() { +async function openWithSurvivingOwner(options: Parameters[2] = {}) { await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) - await seedScanJournal(root, CHAT) + await seedScanJournal(root, CHAT, options) store = await openTestAgentSessionRecordStore(root) let alive = true const stopOwnerProcess = vi.fn((_pid: number, _signal: 'SIGTERM' | 'SIGKILL') => { @@ -61,20 +61,18 @@ afterEach(async () => { describe('a hidden chat whose agent outlived the earlier process', () => { it('is left alone at startup, and settled once an attach decides its recovery', async () => { const { current, stopOwnerProcess } = await openWithSurvivingOwner() - const open = vi.spyOn(AgentSessionJournal.prototype, 'open') await current.reconcileRestartLeases() await current.startupSettled() await idle(current, [CHAT]) - // No signal to a process that may still run, no lease change, and not even a read. + // No signal to a process that may still run, and no lease change. expect(stopOwnerProcess).not.toHaveBeenCalled() expect(store.getRecord(CHAT)?.lease).toMatchObject({ handoffStage: 'recovering', runtimeFence: 13, ownerProcess: { pid: OWNER_PID } }) - expect(open).not.toHaveBeenCalled() // The person attaches: recovery is decided first, as on every build, stopping the owner. const params = attachParamsForRecord(store.getRecord(CHAT)!, { @@ -91,6 +89,38 @@ describe('a hidden chat whose agent outlived the earlier process', () => { }) }) +describe("a hidden chat's send the earlier process never handed over, while its lease recovers", () => { + it('is a card once the chat is read, with no attach and no decision about the owner', async () => { + // Accepted at the owner's own fence, which the lease still holds. + const { current, stopOwnerProcess } = await openWithSurvivingOwner({ + queued: true, + queuedFence: 13 + }) + + await current.reconcileRestartLeases() + await current.startupSettled() + await idle(current, [CHAT]) + // Desktop opens a chat by reading it. + const page = await current.history({ sessionId: CHAT, direction: 'tail' }) + const { submissions } = await current.journalSnapshot(CHAT) + + expect(page.ok && page.page.queuedMessages?.map((card) => card.messageId)).toEqual([ + `${CHAT}-queued` + ]) + expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-queued`)).toMatchObject({ + dispatchState: 'rejected' + }) + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease).toMatchObject({ + handoffStage: 'recovering', + runtimeFence: 13, + ownerProcess: { pid: OWNER_PID } + }) + // What the owner left running waits for the decision: no verdict yet. + expect(await turnState(current, CHAT)).toBe('running') + }) +}) + describe('a visible chat whose agent outlived the earlier process', () => { it('has its owner stopped by the visible-tab restore, as before', async () => { const { current, stopOwnerProcess } = await openWithSurvivingOwner() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts index 3390069806b0..7e146dbcf694 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts @@ -83,11 +83,12 @@ export function scanRecord( } /** The chat's journal as the crashed process left it: a running turn at fence 13, and, when - * asked, a send it accepted and never handed over. */ + * asked, a send it accepted and never handed over (at `queuedFence`, the released lease's 14 by + * default). */ export async function seedScanJournal( root: string, sessionId: string, - options: { queued?: true } = {} + options: { queued?: true; queuedFence?: number } = {} ): Promise { const journal = await openAgentSessionJournal({ identity: { @@ -110,7 +111,7 @@ export async function seedScanJournal( clientMessageId: `${sessionId}-queued`, payloadFingerprint: '0'.repeat(64), body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'after this' }] }, - fence: 14, + fence: options.queuedFence ?? 14, handoverRecorded: true, origin: 'client', source: { kind: 'user' } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts index fb89bdc96565..df94fa7a34f3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns-cancel.ts @@ -16,8 +16,6 @@ import { } from './structured-agent-session-command-turn' import { answerCancelOfSettledPrompt, - dismissEndedGenerationPrompt, - isEndedGenerationPrompt, validatePendingPrompt } from './structured-agent-session-prompt-state' import { @@ -98,24 +96,13 @@ type StopSettleBinding = { export async function performCancel( ctx: AgentSessionTurnContext, - given: PerformCancelInput + input: PerformCancelInput ): Promise> { - let input = given if (input.prompt) { const validated = validatePendingPrompt(ctx, input.prompt) if (!validated.ok) { return answerCancelOfSettledPrompt(ctx, { ...input, prompt: input.prompt }, validated) } - // A card an ended generation raised reaches no provider: dismissed in the journal, and the - // call is a plain Stop of what runs now, so an older client's Stop still stops it. - if (isEndedGenerationPrompt(ctx, validated)) { - const dismissed = await dismissEndedGenerationPrompt(ctx, validated) - if (!dismissed.ok) { - return dismissed - } - input = { ...input } - delete input.prompt - } } // A person's Stop that named no turn binds what ends while it settles (`beginJournalStopSettle`). const settle = input.opensSettle ? ctx.journal.stopMarks.beginSettle() : null diff --git a/src/shared/agent-session-refusal-notice.test.ts b/src/shared/agent-session-refusal-notice.test.ts index 2b10084a297f..c784fecbee3f 100644 --- a/src/shared/agent-session-refusal-notice.test.ts +++ b/src/shared/agent-session-refusal-notice.test.ts @@ -564,6 +564,20 @@ describe('an answer to a prompt whose agent stopped', () => { 'Codex has stopped, so your answer was not sent. Send a message to continue.' ) }) + + it.each(['providerStartFailed', 'notSignedIn', 'cliMissing', 'historyTooLarge'] as const)( + 'keeps that sentence its own: %s still reads "Your answer was not sent."', + (reason) => { + const refusal = { + code: 'agent_session_operation_invalid' as const, + message: HOST_TEXT, + details: { reason } + } + expect(agentSessionRefusalNotice(refusal, 'answer', { agentName: 'Codex' })).toBe( + 'Your answer was not sent.' + ) + } + ) }) describe('a refusal from a host that names no reason this build knows', () => { diff --git a/src/shared/agent-session-refusal-notice.ts b/src/shared/agent-session-refusal-notice.ts index c429eb689bb1..dfaaca943741 100644 --- a/src/shared/agent-session-refusal-notice.ts +++ b/src/shared/agent-session-refusal-notice.ts @@ -125,7 +125,7 @@ function reasonParts( return undefined } if ('fact' in words) { - if (write === 'answer') { + if (write === 'answer' && words.fact === 'providerExited') { // An approval or question was waiting on the person when its agent stopped: one sentence // that says the answer was not sent, names the agent, and says how to go on. return [{ failure: { kind: words.fact }, surface: 'answer', context }] From 86687f137bb4f531ad8cc0a89a1b3b132c90e94e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:41:54 -0700 Subject: [PATCH 08/22] fix(native-chat): the worker retries a failed restart reconcile; the drain waits out storage contention - a read that had to found its journal's epoch owes nothing; a read gone stale backs off instead of re-running at once - an unreconciled lease is the worker's to reconcile again, with its backoff, so a failed startup reconcile no longer leaves a dead turn working until the next send - the automatic drain starts nothing while an ended child still holds the lease, and retries a contended write quietly before main's failed-send hold - parked debts keep only death evidence and die with the chat's record --- .../scripts/vitest-sqlite-runtime-files.mjs | 2 + .../structured-agent-session-child-close.ts | 10 +- ...-agent-session-ended-prompt-cancel.test.ts | 56 ++++--- .../structured-agent-session-prompt-cancel.ts | 7 +- ...d-agent-session-queued-drain-retry.test.ts | 125 +++++++++++++++ ...ctured-agent-session-queued-drain-retry.ts | 81 ++++++++++ ...tructured-agent-session-queued-messages.ts | 31 +++- .../structured-agent-session-queued-wiring.ts | 3 + ...ed-agent-session-reconciliation-backoff.ts | 14 ++ ...tured-agent-session-reconciliation-load.ts | 17 +- ...red-agent-session-reconciliation-memory.ts | 45 ++++++ ...ession-reconciliation-parked-debts.test.ts | 150 ++++++++++++++++++ ...red-agent-session-reconciliation-worker.ts | 68 ++++---- ...uctured-agent-session-restart-reconcile.ts | 9 +- ...tructured-agent-session-restart-restore.ts | 3 +- .../structured-agent-session-reveal.ts | 3 +- ...ent-session-startup-recovery-scope.test.ts | 28 ++++ ...gent-session-startup-scan-lifetime.test.ts | 36 +++++ ...agent-session-startup-scan.test-fixture.ts | 23 ++- 19 files changed, 633 insertions(+), 78 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 5b81edb72e97..b560ffc59288 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -115,6 +115,8 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index 524b4fd4d3f2..be3ccd7645e7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -10,7 +10,10 @@ import { refuse } from '../../../shared/agent-session-wire-refusals' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' -import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' +import type { + StructuredAgentSessionHostSession, + StructuredAgentSessionProviderChild +} from './structured-agent-session-host-types' import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' import { releaseStoredStructuredAgentSessionOwnerAfterExit } from './structured-agent-session-lease-release' import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' @@ -102,7 +105,10 @@ function closeProviderRoot( /** Whether the lease still names the child this host last proved gone, unreleased: only that * in-memory proof lets this host release it without a probe, so the handle carrying it stays. */ export function structuredAgentSessionEndedChildHoldsLease( - context: Pick, + context: { + deps: Pick + sessions: ReadonlyMap> + }, sessionId: string ): boolean { const ended = context.sessions.get(sessionId)?.lastEndedChild diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts index e0157bf6d8cc..351d48db55d8 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts @@ -1,8 +1,8 @@ // A Cancel that names a card an ended generation raised, from a client that still shows it. As on // main: the card is dismissed in the journal alone, and a provider whose card Cancel routes -// (Claude, Pi) stops nothing; one with no route (Codex, ACP) interrupts the live turn the request -// names through that turn's own cancel, never the chat's Stop, so queued messages stay and no Stop -// event pauses the queue. +// (Claude, Pi) stops nothing. One with no route gets the turn's own cancel with the card, never the +// chat's Stop, so queued messages stay and no Stop event pauses the queue: ACP interrupts the live +// turn the request names; Codex finds no such card in its live child and interrupts nothing. import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -173,29 +173,33 @@ describe('a card an ended generation raised, cancelled from a client that still } ) - it.each(['Codex', 'ACP'])( - '%s (no card route): the named live turn is interrupted through its own cancel', - async () => { - const { journal, itemId } = await endedCardAndLiveTurn() - const agent = provider() - const stop = chatStop() - const ctx = context(journal, agent) - const input = { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } } - - const result = await cancelStructuredAgentSessionPrompt(ctx, input, { - stop, - // What the host's plan runs: the turn's cancel, with the card, as on main. - interrupt: () => performCancel(ctx, { clientOperationId: 'cancel-1', ...input }) - }) - - expect(result).toMatchObject({ ok: true }) - expect(agent.cancelTurn).toHaveBeenCalledOnce() - expect(agent.cancelTurn.mock.calls[0]?.[0]).toMatchObject({ turnId: 'turn-2' }) - expect(stop).not.toHaveBeenCalled() - expect(resolution(journal, itemId)).toBe('cancelled') - queueUntouched(journal) - } - ) + it.each([ + ['ACP: the named live turn is interrupted', true], + ['Codex: its live child holds no such card, so nothing is interrupted, as on main', false] + ] as const)("no card route, through the turn's own cancel. %s", async (_name, interrupts) => { + const { journal, itemId } = await endedCardAndLiveTurn() + const agent = provider() + agent.cancelTurn.mockResolvedValueOnce({ cancelled: interrupts }) + const stop = chatStop() + const ctx = context(journal, agent) + const input = { turnId: 'turn-2', prompt: { itemId, expectedRevision: 1 } } + + const result = await cancelStructuredAgentSessionPrompt(ctx, input, { + stop, + // What the host's plan runs: the turn's cancel, with the card, as on main. + interrupt: () => performCancel(ctx, { clientOperationId: 'cancel-1', ...input }) + }) + + expect(result).toMatchObject({ ok: true, value: { cancelled: interrupts } }) + expect(agent.cancelTurn).toHaveBeenCalledOnce() + expect(agent.cancelTurn.mock.calls[0]?.[0]).toMatchObject({ + turnId: 'turn-2', + prompt: { itemId } + }) + expect(stop).not.toHaveBeenCalled() + expect(resolution(journal, itemId)).toBe('cancelled') + queueUntouched(journal) + }) it.each([ ['naming no turn', undefined], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts index 6a51e0781b4e..aaf7885b6326 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-prompt-cancel.ts @@ -71,9 +71,10 @@ export async function cancelStructuredAgentSessionPrompt( /** A card an ended generation raised: no live provider holds it, so it is dismissed in the journal * alone. As on main, a provider whose card Cancel routes (a dismissal, or a Stop of the turn that - * raised it) stops nothing, since the live turn did not raise it; one with no route interrupts the - * turn the request names when that turn is live, through the turn's own cancel, never the chat's - * Stop, so queued messages stay and the queue is not paused. */ + * raised it) stops nothing, since the live turn did not raise it. One with no route gets the + * named live turn's own cancel with the card, as main's plan does, never the chat's Stop, so + * queued messages stay and the queue is not paused: ACP interrupts that turn, and Codex, finding + * no such card in its live child, interrupts nothing. */ async function cancelEndedGenerationPrompt( ctx: AgentSessionTurnContext, input: { turnId?: string }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts new file mode 100644 index 000000000000..e57706b3cc27 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts @@ -0,0 +1,125 @@ +// The queue's automatic send when storage fails: another connection holding the database is +// retried with nothing shown, and only a refusal, or contention past its retries, holds the card +// with "Couldn't send". An exited child's lease not yet released keeps the drain from trying. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { exitChild, reconciled } from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + isTransientStorageFailure, + StructuredAgentSessionQueuedDrainRetry +} from './structured-agent-session-queued-drain-retry' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + vi.useRealTimers() + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** A card queued behind a working turn. */ +async function queuedBehindWork(): Promise<{ + current: QueuedMessageTestRig + draftId: string + working: string +}> { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + return { current, draftId: queued.value.queued.messageId, working } +} + +async function pausedReason(current: QueuedMessageTestRig): Promise { + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + return page.ok ? page.page.queuedMessages?.[0]?.pausedReason : undefined +} + +const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) + +describe("the queue's automatic send", () => { + it("waits for an exited child's release, then sends on its own, never marked", async () => { + const { current, draftId } = await queuedBehindWork() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const release = vi + .spyOn(current.store, 'transitionHandoff') + .mockRejectedValue(new Error('database is locked')) + + await exitChild(current) + await new Promise((resolve) => setTimeout(resolve, 250)) + // The host reads nothing working, yet the drain does not try while the release is owed. + expect(await current.handoff(draftId)).toBeUndefined() + expect(await pausedReason(current)).toBeUndefined() + + release.mockRestore() + await reconciled(current) + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + }) + + it('retries a send that meets a locked database, and sends it with nothing shown', async () => { + const { current, draftId, working } = await queuedBehindWork() + const append = vi + .spyOn(AgentSessionJournal.prototype, 'appendSubmission') + .mockRejectedValueOnce(locked()) + .mockRejectedValueOnce(locked()) + + await current.settleAccepted(working, 'a') + await vi.waitFor(() => expect(append).toHaveBeenCalledTimes(1)) + expect(await pausedReason(current)).toBeUndefined() + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + expect(append.mock.calls.length).toBeGreaterThanOrEqual(3) + }) + + it('holds the card with "Couldn\'t send" at once for a refusal', async () => { + const { current, working } = await queuedBehindWork() + vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockRejectedValueOnce( + new Error('disk full') + ) + + await current.settleAccepted(working, 'a') + await eventually(async () => + expect(await pausedReason(current)).toBe(QUEUED_MESSAGE_PAUSED_SEND_FAILED) + ) + }) +}) + +describe('a send that keeps meeting a locked database', () => { + it('is given up after a bounded run, and only then held', () => { + vi.useFakeTimers() + const wake = vi.fn() + const retry = new StructuredAgentSessionQueuedDrainRetry(wake) + for (let attempt = 1; attempt < 10; attempt += 1) { + expect(retry.retryLater('chat', 'card')).toBe(true) + expect(retry.waiting('chat', 'card')).toBe(true) + vi.advanceTimersByTime(30_000) + expect(retry.waiting('chat', 'card')).toBe(false) + } + expect(wake).toHaveBeenCalledTimes(9) + // The tenth failure falls back to the held card. + expect(retry.retryLater('chat', 'card')).toBe(false) + }) + + it('starts over for a different card, and tells contention from other failures', () => { + vi.useFakeTimers() + const retry = new StructuredAgentSessionQueuedDrainRetry(() => undefined) + for (let attempt = 1; attempt < 10; attempt += 1) { + retry.retryLater('chat', 'card') + } + expect(retry.retryLater('chat', 'other')).toBe(true) + expect(isTransientStorageFailure(locked())).toBe(true) + expect(isTransientStorageFailure(new Error('wrapped', { cause: locked() }))).toBe(true) + expect(isTransientStorageFailure(new Error('disk full'))).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts new file mode 100644 index 000000000000..a64dcdf11296 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts @@ -0,0 +1,81 @@ +// The queue's automatic send meeting storage another connection holds (SQLITE_BUSY or locked): not +// the person's failure, so nothing is shown and the card is not held. The same send is retried +// after a backoff, in memory only, until it lands, the card moves on (a Send now, Delete or Stop) +// or the process ends; after a bounded run it falls back to the held card, so a person can act and +// nothing is owed forever. Any other failure holds the card at once, as before. + +import { isTransientSqliteContention } from '../../sqlite/sqlite-read-failure' +import { + RECONCILIATION_MAX_FAILED_ATTEMPTS, + reconciliationBackoffDelay +} from './structured-agent-session-reconciliation-backoff' + +/** Whether a failure, or one it wraps, is SQLite contention (`isTransientSqliteContention`). */ +export function isTransientStorageFailure(error: unknown): boolean { + let current: unknown = error + for (let depth = 0; current !== undefined && depth < 5; depth += 1) { + if (isTransientSqliteContention(current)) { + return true + } + current = current instanceof Error ? current.cause : undefined + } + return false +} + +type PendingRetry = { + messageId: string + failures: number + timer: ReturnType | null +} + +export class StructuredAgentSessionQueuedDrainRetry { + private readonly pending = new Map() + + /** `wake`: schedules the chat's drain, whose step re-derives everything. */ + constructor(private readonly wake: (sessionId: string) => void) {} + + /** Whether this card's send waits out a backoff: a step woken meanwhile leaves it to the timer. */ + waiting(sessionId: string, messageId: string): boolean { + const entry = this.pending.get(sessionId) + return entry?.messageId === messageId && entry.timer !== null + } + + /** A transient failure of this card's send: true while it is retried later, false once the + * run is given up (the caller then holds the card). */ + retryLater(sessionId: string, messageId: string): boolean { + const previous = this.pending.get(sessionId) + const failures = previous?.messageId === messageId ? previous.failures + 1 : 1 + if (failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { + this.settled(sessionId) + return false + } + const delay = reconciliationBackoffDelay(failures) + const entry: PendingRetry = { messageId, failures, timer: null } + entry.timer = setTimeout(() => { + entry.timer = null + this.wake(sessionId) + }, delay) + // A backoff alone never keeps the process alive. + entry.timer.unref?.() + this.pending.set(sessionId, entry) + return true + } + + /** The send landed, or the card is no longer the queue's next: its run ends. */ + settled(sessionId: string): void { + const entry = this.pending.get(sessionId) + if (entry?.timer) { + clearTimeout(entry.timer) + } + this.pending.delete(sessionId) + } + + dispose(): void { + for (const entry of this.pending.values()) { + if (entry.timer) { + clearTimeout(entry.timer) + } + } + this.pending.clear() + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 48a380142842..daa8eedbfff6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -28,6 +28,10 @@ import type { AgentSessionTurnContext } from './structured-agent-session-turns' import { QueuedMessageNotConsumableError } from '../agent-session-journal/journal-queued-messages' import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + isTransientStorageFailure, + StructuredAgentSessionQueuedDrainRetry +} from './structured-agent-session-queued-drain-retry' import { structuredAgentSessionHostInstance, structuredQueuePauses @@ -233,6 +237,9 @@ export type QueuedMessageDrainDeps = { currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void + /** An exited child's lease still awaits its release (`structuredAgentSessionEndedChildHoldsLease`): + * the release's repair wakes the drain once it lands. */ + endedChildHoldsLease: (sessionId: string) => boolean logger: StructuredAgentSessionLogger } @@ -247,6 +254,7 @@ export type QueuedMessageDrainDeps = { export class StructuredAgentSessionQueuedMessageDrain { private readonly scheduled = new Set() private disposed = false + private readonly retry = new StructuredAgentSessionQueuedDrainRetry((id) => this.schedule(id)) constructor(private readonly deps: QueuedMessageDrainDeps) {} @@ -255,6 +263,7 @@ export class StructuredAgentSessionQueuedMessageDrain { * right before it appends, since quit can land while it awaits. */ dispose(): void { this.disposed = true + this.retry.dispose() } schedule(sessionId: string): void { @@ -318,6 +327,15 @@ export class StructuredAgentSessionQueuedMessageDrain { const work = this.deps.currentWork(sessionId) const next = work ? nextStructuredQueuedMessage({ journal, record, work }) : null if (this.disposed || !next) { + this.retry.settled(sessionId) + return + } + // An exited child's lease not yet released: its storage is failing now, and the release's + // repair wakes this step once it lands. A person's Send now never waits on it. + if ( + this.deps.endedChildHoldsLease(sessionId) || + this.retry.waiting(sessionId, next.messageId) + ) { return } // Always a fresh id: the submission names its draft by `queuedMessageId`, never by id equality. @@ -344,17 +362,22 @@ export class StructuredAgentSessionQueuedMessageDrain { } catch (error) { if (error instanceof QueuedMessageNotConsumableError) { // Lost a race with a Send-now, a Delete or a Stop; their transition stands. + this.retry.settled(sessionId) + return + } + // Storage another connection holds: retried after a backoff, with nothing shown. + if (isTransientStorageFailure(error) && this.retry.retryLater(sessionId, next.messageId)) { return } - // Pre-consume failure: the draft stays waiting, held with the marker on - // the card (a stored fact, so it survives eviction and restart). The - // hold's own commit notification publishes it. An explicit Send retries; - // no automatic retry loop. + // A refusal, or contention past its retries: the draft stays waiting, held with the marker + // on the card (a stored fact, so it survives eviction and restart). The hold's own commit + // notification publishes it. An explicit Send retries. await journal.queuedMessages .hold({ messageIds: [next.messageId], reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED }) .catch(() => {}) throw error } + this.retry.settled(sessionId) this.deps.wakeDelivery(sessionId) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index 1c2984ffa399..642bff62d995 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -16,6 +16,7 @@ import { sendQueuedStructuredAgentMessage } from './structured-agent-session-queued-mutations' import { deferredStructuredAgentSessionLogger } from './structured-agent-session-logger' +import { structuredAgentSessionEndedChildHoldsLease } from './structured-agent-session-child-close' /** `sessions` are the live conversations (their `touch` is the idle sweep's activity renewal, * which the drain's schedule rides); everything else comes from the host's mutation context, @@ -35,6 +36,8 @@ export function wireStructuredAgentSessionQueuedMessages( currentWork: (sessionId) => hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), + endedChildHoldsLease: (sessionId) => + structuredAgentSessionEndedChildHoldsLease({ deps: context().deps, sessions }, sessionId), // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts new file mode 100644 index 000000000000..c79b04ec0e57 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts @@ -0,0 +1,14 @@ +// The one backoff every in-memory retry of background bookkeeping uses: from 1 s to 30 s, given up +// after a bounded run (about three minutes), so nothing is owed forever. + +export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 30_000 } as const +/** Consecutive failed attempts after which a retry gives up. */ +export const RECONCILIATION_MAX_FAILED_ATTEMPTS = 10 + +/** The wait before the next try, after `failures` consecutive failed ones. */ +export function reconciliationBackoffDelay(failures: number): number { + return Math.min( + RECONCILIATION_BACKOFF_MS.max, + RECONCILIATION_BACKOFF_MS.first * 2 ** (failures - 1) + ) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts index 9fa1b9164d3b..aba47487c80a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts @@ -17,8 +17,9 @@ export type StructuredAgentSessionReconciliationLoad = | { kind: 'loaded'; session: StructuredAgentSessionHostSession } /** Opened by a reader meanwhile, or quitting: nothing read. */ | { kind: 'skipped' } - /** Nothing owed: no journal (nothing was ever written), or one damaged or written by a newer Orca, - * which no retry reads (reported once here). */ + /** Nothing owed: no journal (nothing was ever written), a named epoch holding no row (founded + * afresh in memory, so nothing durable is there to settle), or one damaged or written by a + * newer Orca, which no retry reads (reported once here). */ | { kind: 'nothing' } /** Replays a closed chat's journal, for a caller in a background slot. Throws only a failure that @@ -36,7 +37,17 @@ export async function loadStructuredAgentSessionForReconciliation( } try { const opened = await restoreStructuredAgentSessionRead(deps, sessionId) - return opened ? { kind: 'loaded', session: opened.session } : { kind: 'nothing' } + if (!opened) { + return { kind: 'nothing' } + } + const { journal } = opened.session + // The read founded a fresh epoch because the named one holds no row: nothing durable to settle, + // and a handle on an epoch the store never names would read as stale on every attempt. + if (readJournalSessionEpoch(deps.journalDatabase.db, sessionId) !== journal.cursor().epoch) { + await journal.close() + return { kind: 'nothing' } + } + return { kind: 'loaded', session: opened.session } } catch (error) { if (!isJournalOpenFailurePermanent(error)) { throw error diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts new file mode 100644 index 000000000000..dfaa8f7e6eb7 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts @@ -0,0 +1,45 @@ +// What the reconciliation keeps of a chat between its workers, in memory only: nothing here is +// owed past the process, and all of it goes with the chat's record. + +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionReconciliationDebts } from './structured-agent-session-reconciliation-pass' + +export class StructuredAgentSessionReconciliationMemory { + /** Proofs a retired worker still held (a lease in recovery, or a run given up), for the next + * one; dropped once taken. An exit's account is not kept: it judges only its own generation, + * which a later one may have replaced by then. */ + private readonly parked = new Map() + /** Where this host process first opened each chat's journal: every send at or before it was + * accepted by an earlier host process, every one after by this. Kept for the whole process (a + * close or a retire must not reset it, or this process's sends would read as an earlier one's). */ + private readonly firstOpened = new Map() + + park(sessionId: string, debts: StructuredAgentSessionReconciliationDebts): void { + if (debts.evidence) { + this.parked.set(sessionId, { evidence: debts.evidence }) + } + } + + take(sessionId: string): StructuredAgentSessionReconciliationDebts { + const debts = this.parked.get(sessionId) ?? {} + this.parked.delete(sessionId) + return debts + } + + noteOpened(sessionId: string, journal: Pick): void { + if (!this.firstOpened.has(sessionId)) { + this.firstOpened.set(sessionId, journal.openedAt()) + } + } + + processOpened(sessionId: string): AgentJournalCursor | undefined { + return this.firstOpened.get(sessionId) + } + + /** The chat's record is gone. */ + forget(sessionId: string): void { + this.parked.delete(sessionId) + this.firstOpened.delete(sessionId) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts new file mode 100644 index 000000000000..879d73860854 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts @@ -0,0 +1,150 @@ +// What a retired worker keeps for the chat's next one: the proofs it still held, never an exit's +// account (it judges only its own generation, which a later one may have replaced), and nothing at +// all once the chat's record is gone. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' +import { + openScanHost, + SCAN_NOW, + scanIdle as idle, + scanRecord, + scanTurnState as turnState, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-bbbbbbb1' +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-parked-debts-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +/** Generation 13 left a running turn; generation 14 outlived the earlier process, holding a send + * it was handed and never answered, so its lease is latched in recovery. */ +async function recoveringAt14() { + const base = scanRecord(CHAT, false) + await seedTestAgentSessionRecordStore(root, { + records: [ + { + ...base, + providerHandleChain: base.providerHandleChain.map((link) => ({ + ...link, + mintedAtFence: 14 + })), + lease: { ...base.lease, runtimeFence: 14 } + } + ] + }) + await seedScanJournal(root, CHAT, { handedOverAt: 14 }) + store = await openTestAgentSessionRecordStore(root) + let alive = true + const probeOwner = async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + host = openScanHost(root, store, { + probeOwner, + stopOwnerProcess: () => { + alive = false + } + }) + await host.reconcileRestartLeases() + await host.startupSettled() + await idle(host, [CHAT]) + expect(store.getRecord(CHAT)?.lease).toMatchObject({ + handoffStage: 'recovering', + runtimeFence: 14 + }) + return host +} + +const proof13 = { + kind: 'exit-observed' as const, + detail: 'generation 13 exited', + observedAt: SCAN_NOW - 1_000, + ownerFence: 13 +} + +/** Generation 13's exit account: it died while still starting. */ +const exit13: StructuredAgentSessionExitSettlement = { + ownerFence: 13, + settlementId: 'exit-13', + verdict: { state: 'interrupted', completedAt: SCAN_NOW - 1_000 }, + pendingSubmissionReason: 'provider_exited', + exitFailure: { kind: 'providerStartFailed' }, + exitedDuringStartup: { generation: 'generation-13' } +} + +function attach(current: StructuredAgentSessionHost) { + const params = attachParamsForRecord(store.getRecord(CHAT)!, { + clientOperationId: 'attach-1', + expectedRuntimeFence: 14 + }) + return current.attach({ callerKey: 'client-1' }, params).catch(() => undefined) +} + +describe("a worker that retires while the chat's lease recovers", () => { + it("parks generation 13's proof, never its exit account, which would judge generation 14's sends", async () => { + const current = await recoveringAt14() + const { reconciliation } = current.collaboratorsForTests() + + // Generation 13's exit could not be settled; the lease recovers, so the worker retires. + reconciliation.signal(CHAT, { evidence: proof13, exit: exit13 }) + await idle(current, [CHAT]) + expect(reconciliation['memory']['parked'].get(CHAT)).toEqual({ evidence: [proof13] }) + + // Generation 14 ends with no exit account of its own: an attach decides its recovery. + await attach(current) + await idle(current, [CHAT]) + + // The proof judged what generation 13 left, and was taken. + expect(await turnState(current, CHAT)).toBe('interrupted') + expect(reconciliation['memory']['parked'].has(CHAT)).toBe(false) + // Generation 14's send ran on a child that proved its start: in doubt, never "did not start". + const { submissions } = await current.journalSnapshot(CHAT) + expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-handed`)).toMatchObject({ + dispatchState: 'unknown' + }) + }) +}) + +describe('a chat whose record is gone', () => { + it('leaves nothing behind in the reconciliation: no parked proof, no first-open cursor', async () => { + const current = await recoveringAt14() + const { reconciliation } = current.collaboratorsForTests() + reconciliation.signal(CHAT, { evidence: proof13 }) + await idle(current, [CHAT]) + expect(await turnState(current, CHAT)).toBe('running') + expect(reconciliation['memory']['parked'].has(CHAT)).toBe(true) + expect(reconciliation['memory']['firstOpened'].has(CHAT)).toBe(true) + + const getRecord = store.getRecord.bind(store) + vi.spyOn(store, 'getRecord').mockImplementation((id) => (id === CHAT ? null : getRecord(id))) + reconciliation.signal(CHAT) + await idle(current, [CHAT]) + + expect(reconciliation['memory']['parked'].has(CHAT)).toBe(false) + expect(reconciliation['memory']['firstOpened'].has(CHAT)).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts index 7605c53f36a2..d7f847949724 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -24,7 +24,6 @@ // journal can never load, when the database is read-only, or at shutdown. Nothing a person does // (send, start, open, Stop, answer) ever waits on it. -import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -38,15 +37,16 @@ import { loadStructuredAgentSessionForReconciliation, structuredAgentSessionJournalIsCurrent } from './structured-agent-session-reconciliation-load' +import { StructuredAgentSessionReconciliationMemory } from './structured-agent-session-reconciliation-memory' +import { + RECONCILIATION_MAX_FAILED_ATTEMPTS, + reconciliationBackoffDelay +} from './structured-agent-session-reconciliation-backoff' import { StructuredAgentSessionReconciliationSlots, type StructuredAgentSessionReconciliationSlotWaiter } from './structured-agent-session-reconciliation-slots' -export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 30_000 } as const -/** Consecutive failed attempts after which the worker gives up (about three minutes of backoff). */ -export const RECONCILIATION_MAX_FAILED_ATTEMPTS = 10 - export type StructuredAgentSessionReconciliationSignal = { /** Proof the lease no longer holds (`AgentSessionGenerationEnd.evidence`). */ evidence?: AgentSessionGenerationEnd['evidence'] @@ -65,11 +65,15 @@ export type StructuredAgentSessionReconciliationContext = track: (operation: Promise) => Promise /** Publishes what is current now and wakes the queued-card drain. */ publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void + /** Startup's lease reconcile (`createReaderReconcile`): whether every lease is settled. */ + reconcile: (sessionId: string) => Promise } /** `done`: a pass found nothing owed, unless a signal came meanwhile; `retire`: nothing this - * worker can do, whatever came meanwhile (a later signal starts a new one). */ -type Outcome = 'done' | 'retire' | 'again' | 'failed' + * worker can do, whatever came meanwhile (a later signal starts a new one); `again`: a pass wrote + * rows, so the next one verifies at once; `stale`: no pass ran (its read went stale), which backs + * off like a failure, since nothing moved. */ +type Outcome = 'done' | 'retire' | 'again' | 'stale' | 'failed' type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { debts: StructuredAgentSessionReconciliationDebts @@ -86,11 +90,7 @@ type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { export class StructuredAgentSessionReconciliation { private readonly workers = new Map() - /** Debts a retired worker still held (a lease in recovery, or a run given up), for the next. */ - private readonly parked = new Map() - /** Process-scoped, never persisted: where this host process first opened each chat's journal. - * Every send at or before it was accepted by an earlier host process, every one after by this. */ - private readonly firstOpened = new Map() + private readonly memory = new StructuredAgentSessionReconciliationMemory() private disposed = false private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) private readonly unsubscribe: () => void @@ -134,11 +134,8 @@ export class StructuredAgentSessionReconciliation { } /** A handle this host opened on the chat's journal; only its first one marks the boundary. */ - noteOpened = (sessionId: string, journal: Pick): void => { - if (!this.firstOpened.has(sessionId)) { - this.firstOpened.set(sessionId, journal.openedAt()) - } - } + noteOpened = (sessionId: string, journal: Pick): void => + this.memory.noteOpened(sessionId, journal) /** A reader opened the chat: whatever its worker still waits a slot for goes ahead of the scan. */ prioritize = (sessionId: string): void => { @@ -183,7 +180,7 @@ export class StructuredAgentSessionReconciliation { let worker = this.workers.get(sessionId) if (!worker) { worker = { - debts: this.parked.get(sessionId) ?? {}, + debts: this.memory.take(sessionId), running: false, dirty: false, timer: null, @@ -191,7 +188,6 @@ export class StructuredAgentSessionReconciliation { attempted: [], urgent: false } - this.parked.delete(sessionId) this.workers.set(sessionId, worker) } return worker @@ -213,9 +209,22 @@ export class StructuredAgentSessionReconciliation { private async runAttempt(sessionId: string, worker: ChatWorker): Promise { try { const { store } = this.context.deps - if (this.disposed || store.readOnly || !store.getRecord(sessionId)) { + if (!this.disposed && !store.getRecord(sessionId)) { + // The chat is gone: nothing of it is owed again. + this.memory.forget(sessionId) return 'retire' } + if (this.disposed || store.readOnly) { + return 'retire' + } + // Startup's reconcile failed (storage was busy): retried here, the same deduped step, so a + // gone owner's lease does not read live until a person sends. It never stops a process. + if ( + store.getRecord(sessionId)?.lease.unreconciled && + !(await this.context.reconcile(sessionId)) + ) { + return 'failed' + } if (!this.context.sessions.has(sessionId) && !worker.loaded) { const stop = await this.load(sessionId, worker) if (stop) { @@ -237,14 +246,14 @@ export class StructuredAgentSessionReconciliation { const session = this.sessionFor(sessionId, worker) // Closed, or written by another handle, since this attempt began: the next one reads it again. if (!session) { - return 'again' + return 'stale' } const pass = await runStructuredAgentSessionReconciliationPass( this.context, sessionId, session, worker.debts, - this.firstOpened.get(sessionId) ?? session.journal.openedAt() + this.memory.processOpened(sessionId) ?? session.journal.openedAt() ) if (pass.failed.length > 0) { this.warn(sessionId, pass.failed[0]) @@ -292,7 +301,7 @@ export class StructuredAgentSessionReconciliation { } if (loaded.kind === 'loaded') { worker.loaded = loaded.session - this.noteOpened(sessionId, loaded.session.journal) + this.memory.noteOpened(sessionId, loaded.session.journal) } // Skipped: a reader opened it meanwhile, and the pass uses their conversation. return loaded.kind === 'nothing' ? 'retire' : null @@ -321,7 +330,7 @@ export class StructuredAgentSessionReconciliation { this.dropLoaded(worker) return } - if (outcome === 'failed') { + if (outcome === 'failed' || outcome === 'stale') { worker.failures += 1 if (worker.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { this.context.deps.logger.warn("gave up settling a gone agent's leftover work for now", { @@ -332,11 +341,7 @@ export class StructuredAgentSessionReconciliation { this.retire(sessionId, worker) return } - const delay = Math.min( - RECONCILIATION_BACKOFF_MS.max, - RECONCILIATION_BACKOFF_MS.first * 2 ** (worker.failures - 1) - ) - this.schedule(sessionId, worker, delay) + this.schedule(sessionId, worker, reconciliationBackoffDelay(worker.failures)) return } worker.failures = 0 @@ -365,9 +370,8 @@ export class StructuredAgentSessionReconciliation { } worker.timer = null this.workers.delete(sessionId) - const { evidence, exit } = worker.debts - if (!this.disposed && (evidence || exit) && this.context.deps.store.getRecord(sessionId)) { - this.parked.set(sessionId, worker.debts) + if (!this.disposed && this.context.deps.store.getRecord(sessionId)) { + this.memory.park(sessionId, worker.debts) } if (!worker.running) { this.dropLoaded(worker) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 29f0c9f1047d..f8bf3a29a16b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -54,8 +54,8 @@ export type ReaderBookkeepingFailures = { clear: () => void } -/** Startup and the read carry on past a failure: the next attach or send reconciles and resolves - * recovery again before it acts. */ +/** Startup and the read carry on past a failure: each chat's reconciliation worker retries it with + * a backoff, and the next attach or send reconciles and resolves recovery again before it acts. */ export function reportEachFailureOnce( logger: StructuredAgentSessionLogger ): ReaderBookkeepingFailures { @@ -77,8 +77,9 @@ export function reportEachFailureOnce( } } -/** The reconcile a reader runs, at startup and before each restored read: it never throws, since - * an unreconciled lease grants no writer and the next send reconciles again before it acts. +/** The reconcile a reader runs, at startup, before each restored read and in the reconciliation + * worker: it never throws, since an unreconciled lease grants no writer and the next send + * reconciles again before it acts. * Answers whether every lease is settled. */ export function createReaderReconcile( reconcile: (sessionId: string) => Promise, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index 6c6efc49d728..a91904964819 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -80,7 +80,8 @@ export async function restoreStructuredAgentSessionsOnRestart( } // One check for the pass. Each chat checks again while it holds, since another writer can mark // leases unreconciled mid-pass; after the first failure, retrying per chat only waits on the - // same store again, and the next attach or send settles those chats instead. + // same store again; each chat's reconciliation worker retries it with a backoff instead, and the + // next attach or send reconciles before it acts. let settled = await input.reconcile(first.sessionId) const settleLeases = async (sessionId: string): Promise => { // A session latched in recovery exits here at startup, without waiting for a client. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index e32d55a85997..ad83758a68e5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -93,7 +93,8 @@ export function createStructuredAgentSessionHostRestore( // Tracked like a start, so a quit waits for an attempt before it closes what it writes to. track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => - startup.clientDelivery.publishGenerationEnded(sessionId, options) + startup.clientDelivery.publishGenerationEnded(sessionId, options), + reconcile }) let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts index a04f69c93655..def0d0f52183 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts @@ -135,3 +135,31 @@ describe('a visible chat whose agent outlived the earlier process', () => { expect(await turnState(current, CHAT)).toBe('interrupted') }) }) + +describe('a chat whose startup lease reconcile failed (storage was busy)', () => { + it('is reconciled by its worker after a backoff, and its turn settles with no attach or send', async () => { + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) + await seedScanJournal(root, CHAT) + store = await openTestAgentSessionRecordStore(root) + const busy = () => new Error('database is locked') + // Startup's reconcile and the worker's first retry meet a locked store; the next one lands. + const reconcile = vi + .spyOn(store, 'reconcileOnRestart') + .mockRejectedValueOnce(busy()) + .mockRejectedValueOnce(busy()) + host = openScanHost(root, store) + const current = host + + await current.reconcileRestartLeases() + expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(true) + await vi.waitFor(() => expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(false), { + timeout: 5_000 + }) + await idle(current, [CHAT]) + + expect(reconcile.mock.calls.length).toBeGreaterThanOrEqual(3) + // The owner was proven gone: the turn ends, and nothing reads working. + expect(await turnState(current, CHAT)).toBe('interrupted') + expect(current.currentWork(CHAT)?.working()).toBe(false) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts index f199a855d9b1..8b1a3e95cea2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts @@ -16,9 +16,12 @@ import { closeTestJournalHostDatabases, openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { publishJournalSessionEpoch } from '../agent-session-journal/journal-row-table' import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { HOST_TEST_LOCATION as LOCATION } from './structured-agent-session-host-test-data' +import * as reconciliationLoad from './structured-agent-session-reconciliation-load' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { openScanHost, @@ -212,4 +215,37 @@ describe('a startup worker whose step cannot succeed', () => { expect(warnings).toEqual([]) expect(await turnState(current, CHAT)).toBe('unverifiable') }) + + it('retires at once on a journal whose named epoch holds no row, reading it once', async () => { + // An older build's repair crashed after naming the epoch: nothing durable is there to settle. + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, true)] }) + publishJournalSessionEpoch( + openTestJournalHostDatabase(root).db, + { sessionId: CHAT, workspaceId: LOCATION.workspaceId }, + 'empty-epoch' + ) + store = await openTestAgentSessionRecordStore(root) + const counter = countOpens() + const current = openHost() + await current.reconcileRestartLeases() + await current.startupSettled() + + expect(current.collaboratorsForTests().reconciliation.owes(CHAT)).toBe(false) + await new Promise((resolve) => setTimeout(resolve, 1_200)) + expect(counter.opens).toBe(1) + }) + + it('backs off a read that goes stale on every attempt instead of re-reading at once', async () => { + await seed([CHAT]) + vi.spyOn(reconciliationLoad, 'structuredAgentSessionJournalIsCurrent').mockReturnValue(false) + const counter = countOpens() + const current = openHost() + await current.reconcileRestartLeases() + await new Promise((resolve) => setTimeout(resolve, 2_500)) + + // Attempts at 0 s, 1 s and 3 s: a read each, never a loop. + expect(counter.opens).toBeGreaterThanOrEqual(2) + expect(counter.opens).toBeLessThanOrEqual(3) + expect(current.collaboratorsForTests().reconciliation.owes(CHAT)).toBe(true) + }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts index 7e146dbcf694..45cb244af8ef 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts @@ -84,11 +84,11 @@ export function scanRecord( /** The chat's journal as the crashed process left it: a running turn at fence 13, and, when * asked, a send it accepted and never handed over (at `queuedFence`, the released lease's 14 by - * default). */ + * default), or one it handed to the generation at `handedOverAt` that never answered it. */ export async function seedScanJournal( root: string, sessionId: string, - options: { queued?: true; queuedFence?: number } = {} + options: { queued?: true; queuedFence?: number; handedOverAt?: number } = {} ): Promise { const journal = await openAgentSessionJournal({ identity: { @@ -117,6 +117,25 @@ export async function seedScanJournal( source: { kind: 'user' } }) } + const fence = options.handedOverAt + if (fence !== undefined) { + const clientMessageId = `${sessionId}-handed` + await journal.appendSubmission({ + clientMessageId, + payloadFingerprint: '1'.repeat(64), + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'handed over' }] }, + fence, + handoverRecorded: true, + origin: 'client', + source: { kind: 'user' } + }) + await journal.resolveDispatch({ + clientMessageId, + state: 'pending', + fence, + turnScope: AGENT_JOURNAL_THREAD_SCOPE + }) + } await journal.close() } From d074bdff1c52d1dea21837e12d3a27b77e5e0b18 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:08:38 -0700 Subject: [PATCH 09/22] fix(native-chat): one restart reconcile retry for the store; the queue names a card only when the drain would send it - the drain's wait on an ended child's release dies with the worker, so a give-up never strands a card - a failed restart reconcile is retried once for the whole store, with one backoff - an exit account judges only the generation right after it - the published next card honours the drain's own waits, so no frame reads Working over a settled turn --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + ...d-agent-session-background-task-channel.ts | 12 +- ...tructured-agent-session-client-delivery.ts | 19 ++- .../structured-agent-session-host.ts | 7 +- ...ructured-agent-session-mutation-context.ts | 2 + ...d-agent-session-queued-drain-retry.test.ts | 137 ++++++++++++++++-- ...ctured-agent-session-queued-drain-retry.ts | 27 ++-- ...tructured-agent-session-queued-messages.ts | 40 +++-- ...ctured-agent-session-queued-publication.ts | 13 +- .../structured-agent-session-queued-wiring.ts | 17 ++- ...red-agent-session-reconciliation-memory.ts | 32 ++++ ...ession-reconciliation-parked-debts.test.ts | 24 +++ ...tured-agent-session-reconciliation-pass.ts | 7 +- ...red-agent-session-reconciliation-worker.ts | 29 ++-- ...nt-session-restart-reconcile-retry.test.ts | 106 ++++++++++++++ .../structured-agent-session-subscribers.ts | 5 + 16 files changed, 403 insertions(+), 75 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index b560ffc59288..e719f73ef9bd 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -117,6 +117,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts index 4af996896938..0f87c4164497 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-task-channel.ts @@ -8,10 +8,6 @@ import { structuredChildWorkLegacyTasks } from '../../../shared/structured-agent import { structuredRunningChildWork } from '../../../shared/agent-child-work-listing' import { readStructuredAgentSessionHistoryResult } from './structured-agent-session-history-result' import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' -import { - structuredQueueSendGate, - tryReadQueuePublication -} from './structured-agent-session-queued-publication' import type { AgentSessionHistoryScope } from './agent-session-history-page' import type { AgentSessionSubscribers, @@ -58,13 +54,7 @@ export class StructuredAgentSessionBackgroundTaskChannel { ...(work ? { work } : {}) }) const backgroundTasks = this.read(request.sessionId) - const queue = tryReadQueuePublication( - journal, - structuredQueueSendGate( - { store: this.deps.store, sessions: this.sessions }, - request.sessionId - ) - ) + const queue = this.subscribers.readQueuePublication(request.sessionId) const hostNow = this.deps.now?.() ?? Date.now() return { ...result, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index 92d468ca6b94..b6a3edb4d4b0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -7,7 +7,8 @@ import type { SubscriberFieldHooks } from './agent-session-subscriber-frame-fiel import { AgentSessionSubscribers } from './structured-agent-session-subscribers' import { structuredQueueSendGate, - tryReadQueuePublication + tryReadQueuePublication, + type QueuedDrainWaits } from './structured-agent-session-queued-publication' import type { StructuredAgentSessionHostDeps, @@ -37,8 +38,12 @@ export class StructuredAgentSessionClientDelivery { private readonly sessions: Map, now: () => number, private readonly deps: () => StructuredAgentSessionHostDeps, - /** `activity`: a journal write, which renews the idle clock; a generation's end is not one. */ - private readonly onJournalActivity: (sessionId: string, activity?: boolean) => void, + /** The queued-card drain: every publish wakes it (`activity`: a journal write, which renews the + * idle clock; a generation's end is not one), and a card it holds back is never named next. */ + private readonly queue: { + onJournalActivity: (sessionId: string, activity?: boolean) => void + drain: { waits: QueuedDrainWaits } + }, onAgentStarted: (sessionId: string) => void, /** A session's child records changed; the chat strip republishes from them. */ onChildWorkChanged: (sessionId: string) => void, @@ -67,7 +72,11 @@ export class StructuredAgentSessionClientDelivery { readQueuePublication: (sessionId) => tryReadQueuePublication( sessions.get(sessionId)?.journal, - structuredQueueSendGate({ store: this.deps().store, sessions }, sessionId) + structuredQueueSendGate( + { store: this.deps().store, sessions }, + sessionId, + this.queue.drain.waits + ) ), readBackgroundTasks, onJournalPublished: (sessionId, journal) => this.publishJournal(sessionId, journal), @@ -184,7 +193,7 @@ export class StructuredAgentSessionClientDelivery { // subscribed, which is the whole reason a backgrounded chat can complete at all. After the // status publish, so it reads the projection that publish cached. this.turnCompletionFeed.observe(sessionId, journal) - this.onJournalActivity(sessionId, activity) + this.queue.onJournalActivity(sessionId, activity) } /** The current-work projection over this journal (`structuredAgentSessionCurrentWork`). */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index d20dee5218ff..105dbc8eedff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -67,9 +67,7 @@ export class StructuredAgentSessionHost { logger: sessionLogger.deferredStructuredAgentSessionLogger(() => this.deps.logger), onOpened: (sessionId, journal) => { this.queued.drain.schedule(sessionId) - this.restore.reconciliation.noteOpened(sessionId, journal) - // A reader holds it now: its settlement goes ahead of the rest of the startup scan. - this.restore.reconciliation.prioritize(sessionId) + this.restore.reconciliation.opened(sessionId, journal) }, now: () => this.now() }) @@ -79,7 +77,7 @@ export class StructuredAgentSessionHost { this.sessions, () => this.now(), () => this.deps, - (sessionId, activity) => this.queued.onJournalActivity(sessionId, activity), + this.queued, (sessionId) => this.restartResume.onAgentStarted(sessionId), (sessionId) => this.backgroundTasks.publish(sessionId), (sessionId) => this.backgroundTasks.read(sessionId) @@ -294,6 +292,7 @@ export class StructuredAgentSessionHost { wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), stopAgent: (sessionId, ending) => this.lifetime.stopAgent(sessionId, ending), wakeQueuedDrain: (sessionId) => this.queued.drain.schedule(sessionId), + reconciliationOwes: (sessionId) => this.restore.reconciliation.owes(sessionId), acquireAborts: this.runtimeState.acquireAborts, optionRevisions: this.runtimeState.optionRevisions, now: () => this.now() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index e1f6f40f0b00..06ce9ef2d15b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -39,6 +39,8 @@ export type StructuredAgentSessionMutationContext = { joinChildClose: (sessionId: string) => Promise /** A message was accepted: the session's delivery loop hands it over. */ wakeDelivery: (sessionId: string) => void + /** The chat's reconciliation worker still owes something (`StructuredAgentSessionReconciliation`). */ + reconciliationOwes: (sessionId: string) => boolean /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. * Each caller names why (`ending`). */ stopAgent: (sessionId: string, ending: StructuredAgentSessionStopEnding) => Promise diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts index e57706b3cc27..78622226fa0d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts @@ -1,11 +1,21 @@ // The queue's automatic send when storage fails: another connection holding the database is // retried with nothing shown, and only a refusal, or contention past its retries, holds the card -// with "Couldn't send". An exited child's lease not yet released keeps the drain from trying. +// with "Couldn't send". A worker still owing an ended generation's release keeps the drain from +// trying, and while the drain waits no client is told the card sends next (it would read Working). import { afterEach, describe, expect, it, vi } from 'vitest' import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession +} from '../../../shared/structured-agent-session-reducer' import { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { exitChild, reconciled } from './structured-agent-session-leftover-settlement.test-fixture' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { + exitChild, + reconciled, + REJECT_RECOVERED_ROWS +} from './structured-agent-session-leftover-settlement.test-fixture' import { createQueuedMessageTestRig, eventually, @@ -49,6 +59,27 @@ async function pausedReason(current: QueuedMessageTestRig): Promise Object.assign(new Error('database is locked'), { errcode: 5 }) +/** What a client reads after each frame: Working is the host's, or a card it is told sends next. */ +async function watchChat(current: QueuedMessageTestRig) { + let state = EMPTY_STRUCTURED_AGENT_SESSION + const reads: { working: boolean; turnRunning: boolean }[] = [] + await current.host.subscribe({ + id: 'chat', + sessionId: SESSION, + emit: (event) => { + state = reduceStructuredAgentSession(state, { type: 'event', event }) + reads.push({ + working: state.working === true || (state.nextQueuedMessageId ?? null) !== null, + turnRunning: state.latestTurn?.turn.state === 'running' + }) + } + }) + return reads +} + +const statusWorking = (current: QueuedMessageTestRig) => + current.host.readStatusSummary(SESSION)?.status === 'working' + describe("the queue's automatic send", () => { it("waits for an exited child's release, then sends on its own, never marked", async () => { const { current, draftId } = await queuedBehindWork() @@ -56,12 +87,18 @@ describe("the queue's automatic send", () => { const release = vi .spyOn(current.store, 'transitionHandoff') .mockRejectedValue(new Error('database is locked')) + const reads = await watchChat(current) + const before = reads.length await exitChild(current) await new Promise((resolve) => setTimeout(resolve, 250)) // The host reads nothing working, yet the drain does not try while the release is owed. expect(await current.handoff(draftId)).toBeUndefined() expect(await pausedReason(current)).toBeUndefined() + // The chat reads idle, as the status does: no card is named as sent next while it waits. + expect(reads.length).toBeGreaterThan(before) + expect(reads.at(-1)).toEqual({ working: false, turnRunning: false }) + expect(statusWorking(current)).toBe(false) release.mockRestore() await reconciled(current) @@ -74,10 +111,14 @@ describe("the queue's automatic send", () => { .spyOn(AgentSessionJournal.prototype, 'appendSubmission') .mockRejectedValueOnce(locked()) .mockRejectedValueOnce(locked()) + const reads = await watchChat(current) await current.settleAccepted(working, 'a') await vi.waitFor(() => expect(append).toHaveBeenCalledTimes(1)) expect(await pausedReason(current)).toBeUndefined() + // Waiting out the backoff, the card is not named as sent next: the chat reads idle meanwhile. + await vi.waitFor(() => expect(reads.at(-1)?.working).toBe(false)) + expect(statusWorking(current)).toBe(false) await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) expect(append.mock.calls.length).toBeGreaterThanOrEqual(3) }) @@ -95,31 +136,107 @@ describe("the queue's automatic send", () => { }) }) +describe('a dead turn the worker settles while the release is still owed', () => { + it('publishes one view: no frame reads Working over it, and the card sends once released', async () => { + const { current, draftId } = await queuedBehindWork() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const release = vi.spyOn(current.store, 'transitionHandoff').mockRejectedValue(locked()) + const { db } = openTestJournalHostDatabase(current.root) + db.exec(REJECT_RECOVERED_ROWS) + const reads = await watchChat(current) + const before = reads.length + + // The exit's own settlement and release both fail; the worker settles the turn a pass later. + await exitChild(current) + db.exec('DROP TRIGGER reject_recovered') + const settled = (await current.host.journalSnapshot(SESSION)).items.length + await eventually(async () => + expect((await current.host.journalSnapshot(SESSION)).items.length).toBeGreaterThan(settled) + ) + expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(true) + expect(reads.slice(before).filter((read) => read.working)).toEqual([]) + expect(statusWorking(current)).toBe(false) + + release.mockRestore() + await reconciled(current) + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + }) +}) + describe('a send that keeps meeting a locked database', () => { + const journal = {} + it('is given up after a bounded run, and only then held', () => { vi.useFakeTimers() const wake = vi.fn() const retry = new StructuredAgentSessionQueuedDrainRetry(wake) for (let attempt = 1; attempt < 10; attempt += 1) { - expect(retry.retryLater('chat', 'card')).toBe(true) - expect(retry.waiting('chat', 'card')).toBe(true) + expect(retry.retryLater('chat', 'card', journal)).toBe(true) + expect(retry.waiting('chat', 'card', journal)).toBe(true) vi.advanceTimersByTime(30_000) - expect(retry.waiting('chat', 'card')).toBe(false) + expect(retry.waiting('chat', 'card', journal)).toBe(false) } expect(wake).toHaveBeenCalledTimes(9) // The tenth failure falls back to the held card. - expect(retry.retryLater('chat', 'card')).toBe(false) + expect(retry.retryLater('chat', 'card', journal)).toBe(false) }) - it('starts over for a different card, and tells contention from other failures', () => { + it('starts over for another card, a reopened chat, or a card a person moved on', () => { vi.useFakeTimers() const retry = new StructuredAgentSessionQueuedDrainRetry(() => undefined) - for (let attempt = 1; attempt < 10; attempt += 1) { - retry.retryLater('chat', 'card') + const exhaust = (messageId: string, on: object) => { + for (let attempt = 1; attempt < 10; attempt += 1) { + retry.retryLater('chat', messageId, on) + } } - expect(retry.retryLater('chat', 'other')).toBe(true) + exhaust('card', journal) + expect(retry.retryLater('chat', 'other', journal)).toBe(true) + exhaust('card', journal) + expect(retry.retryLater('chat', 'card', {})).toBe(true) + exhaust('card', journal) + retry.settled('chat') + expect(retry.retryLater('chat', 'card', journal)).toBe(true) + }) + + it('keeps one timer per chat: a new failure replaces the wait before it', () => { + vi.useFakeTimers() + const wake = vi.fn() + const retry = new StructuredAgentSessionQueuedDrainRetry(wake) + retry.retryLater('chat', 'card', journal) + retry.retryLater('chat', 'card', journal) + vi.advanceTimersByTime(60_000) + expect(wake).toHaveBeenCalledTimes(1) + }) + + it('tells contention from other failures', () => { expect(isTransientStorageFailure(locked())).toBe(true) expect(isTransientStorageFailure(new Error('wrapped', { cause: locked() }))).toBe(true) expect(isTransientStorageFailure(new Error('disk full'))).toBe(false) }) }) + +describe('an exited child whose release the worker gave up on', () => { + it('no longer holds the card: once the worker retires, the queue sends it on its own', async () => { + const { current, draftId } = await queuedBehindWork() + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const release = vi + .spyOn(current.store, 'transitionHandoff') + .mockRejectedValue(new Error('database is locked')) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + await exitChild(current) + const { reconciliation } = current.host.collaboratorsForTests() + for (let step = 0; step < 12 && reconciliation.owes(SESSION); step += 1) { + await vi.advanceTimersByTimeAsync(31_000) + } + expect(reconciliation.owes(SESSION)).toBe(false) + release.mockRestore() + vi.useRealTimers() + + // Sent, or held with main's marker: never left waiting with neither. + await eventually(async () => + expect( + Boolean(await current.handoff(draftId)) || (await pausedReason(current)) !== undefined + ).toBe(true) + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts index a64dcdf11296..314ee039f897 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts @@ -24,6 +24,8 @@ export function isTransientStorageFailure(error: unknown): boolean { type PendingRetry = { messageId: string + /** The conversation the failures were met on: a reopened chat's card starts fresh. */ + journal: object failures: number timer: ReturnType | null } @@ -35,33 +37,33 @@ export class StructuredAgentSessionQueuedDrainRetry { constructor(private readonly wake: (sessionId: string) => void) {} /** Whether this card's send waits out a backoff: a step woken meanwhile leaves it to the timer. */ - waiting(sessionId: string, messageId: string): boolean { + waiting(sessionId: string, messageId: string, journal: object): boolean { const entry = this.pending.get(sessionId) - return entry?.messageId === messageId && entry.timer !== null + return entry?.messageId === messageId && entry.journal === journal && entry.timer !== null } /** A transient failure of this card's send: true while it is retried later, false once the * run is given up (the caller then holds the card). */ - retryLater(sessionId: string, messageId: string): boolean { + retryLater(sessionId: string, messageId: string, journal: object): boolean { const previous = this.pending.get(sessionId) - const failures = previous?.messageId === messageId ? previous.failures + 1 : 1 + const failures = + previous?.messageId === messageId && previous.journal === journal ? previous.failures + 1 : 1 + this.settled(sessionId) if (failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { - this.settled(sessionId) return false } - const delay = reconciliationBackoffDelay(failures) - const entry: PendingRetry = { messageId, failures, timer: null } + const entry: PendingRetry = { messageId, journal, failures, timer: null } entry.timer = setTimeout(() => { entry.timer = null this.wake(sessionId) - }, delay) + }, reconciliationBackoffDelay(failures)) // A backoff alone never keeps the process alive. entry.timer.unref?.() this.pending.set(sessionId, entry) return true } - /** The send landed, or the card is no longer the queue's next: its run ends. */ + /** The send landed, or the card moved on (sent, a person's Send now or Delete): its run ends. */ settled(sessionId: string): void { const entry = this.pending.get(sessionId) if (entry?.timer) { @@ -71,11 +73,8 @@ export class StructuredAgentSessionQueuedDrainRetry { } dispose(): void { - for (const entry of this.pending.values()) { - if (entry.timer) { - clearTimeout(entry.timer) - } + for (const sessionId of this.pending.keys()) { + this.settled(sessionId) } - this.pending.clear() } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index daa8eedbfff6..548445c8f426 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -85,6 +85,9 @@ export type StructuredQueueHold = 'blocked' | 'working' | 'prompt' export type StructuredQueueGateInput = { record: AgentSessionRecord | null work: StructuredAgentSessionCurrentWork + /** The drain's own waits past the gate (`StructuredAgentSessionQueuedMessageDrain.waits`): a + * card it holds back is not one a client is told it sends next. */ + drainWaits?: (messageId: string) => boolean } export function structuredQueueHold(input: StructuredQueueGateInput): StructuredQueueHold | null { @@ -113,7 +116,7 @@ export function nextStructuredQueuedMessage( if (next === null || input.work.working()) { return null } - return structuredQueueHold(input) === null ? next : null + return structuredQueueHold(input) === null && !input.drainWaits?.(next.messageId) ? next : null } /** @@ -237,9 +240,12 @@ export type QueuedMessageDrainDeps = { currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void - /** An exited child's lease still awaits its release (`structuredAgentSessionEndedChildHoldsLease`): - * the release's repair wakes the drain once it lands. */ - endedChildHoldsLease: (sessionId: string) => boolean + /** An ended child still holds the lease and the chat's worker still owes its release: its retire + * wakes the drain, whether the release landed or was given up. A settlement still owed with the + * lease released holds nothing: bookkeeping never gates the queue. */ + releaseOwed: (sessionId: string) => boolean + /** Re-sends what clients read (`AgentSessionSubscribers.publish`): a send put off writes nothing. */ + publish: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger } @@ -266,9 +272,21 @@ export class StructuredAgentSessionQueuedMessageDrain { this.retry.dispose() } + /** Whether the send of this card waits: an ended generation's release is still owed + * (`releaseOwed`), or a transient failure's backoff runs. A person's Send now waits on neither. */ + waits = (sessionId: string, messageId: string, journal: object): boolean => + this.deps.releaseOwed(sessionId) || this.retry.waiting(sessionId, messageId, journal) + + /** A person moved the card on (Send now, Delete): a send being retried starts fresh. */ + cardMoved(sessionId: string): void { + this.retry.settled(sessionId) + } + schedule(sessionId: string): void { const journal = this.disposed ? undefined : this.deps.sessions.get(sessionId)?.journal if (!journal) { + // Closed: a backoff's wake finds no chat, and its entry goes (a reopen starts fresh). + this.retry.settled(sessionId) return } // Cheap pre-check so token streams do not pay a serialized step per delta. @@ -330,12 +348,7 @@ export class StructuredAgentSessionQueuedMessageDrain { this.retry.settled(sessionId) return } - // An exited child's lease not yet released: its storage is failing now, and the release's - // repair wakes this step once it lands. A person's Send now never waits on it. - if ( - this.deps.endedChildHoldsLease(sessionId) || - this.retry.waiting(sessionId, next.messageId) - ) { + if (this.waits(sessionId, next.messageId, journal)) { return } // Always a fresh id: the submission names its draft by `queuedMessageId`, never by id equality. @@ -366,7 +379,12 @@ export class StructuredAgentSessionQueuedMessageDrain { return } // Storage another connection holds: retried after a backoff, with nothing shown. - if (isTransientStorageFailure(error) && this.retry.retryLater(sessionId, next.messageId)) { + if ( + isTransientStorageFailure(error) && + this.retry.retryLater(sessionId, next.messageId, journal) + ) { + // Clients were told this card sends next; it now waits, so the chat reads idle meanwhile. + this.deps.publish(sessionId, journal) return } // A refusal, or contention past its retries: the draft stays waiting, held with the marker diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index ac9da94d19ee..e0b1365d7a30 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -33,16 +33,25 @@ export type QueuePublication = { /** What the drain's gate reads beyond the journal; resolved per read. */ export type QueueSendGate = (journal: AgentSessionJournal) => StructuredQueueGateInput +/** `StructuredAgentSessionQueuedMessageDrain.waits`. */ +export type QueuedDrainWaits = (sessionId: string, messageId: string, journal: object) => boolean + +/** `drainWaits`: the drain's own waits, which a publication reads so a client is never told a card + * sends next while the drain holds it back (the chat would read Working with nothing running). */ export function structuredQueueSendGate( host: StructuredAgentSessionCurrentWorkHost, - sessionId: string + sessionId: string, + drainWaits?: QueuedDrainWaits ): QueueSendGate { return (journal) => { const record = host.store.getRecord(sessionId) const ended = host.sessions.get(sessionId)?.lastEndedChild return { record, - work: structuredAgentSessionCurrentWork(journal, { record, ...(ended ? { ended } : {}) }) + work: structuredAgentSessionCurrentWork(journal, { record, ...(ended ? { ended } : {}) }), + ...(drainWaits + ? { drainWaits: (messageId: string) => drainWaits(sessionId, messageId, journal) } + : {}) } } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index 642bff62d995..5622527ca4b9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -36,8 +36,10 @@ export function wireStructuredAgentSessionQueuedMessages( currentWork: (sessionId) => hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), - endedChildHoldsLease: (sessionId) => - structuredAgentSessionEndedChildHoldsLease({ deps: context().deps, sessions }, sessionId), + releaseOwed: (sessionId) => + structuredAgentSessionEndedChildHoldsLease({ deps: context().deps, sessions }, sessionId) && + context().reconciliationOwes(sessionId), + publish: (sessionId, journal) => context().publish(sessionId, journal), // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) }) @@ -55,11 +57,18 @@ export function wireStructuredAgentSessionQueuedMessages( queuedMessageSend: ( caller: StructuredAgentSessionCaller, params: Parameters[2] - ) => sendQueuedStructuredAgentMessage(context(), caller, params), + ) => { + // A person's Send now moves the card on: a send the drain was retrying starts fresh. + drain.cardMoved(params.envelope.sessionId) + return sendQueuedStructuredAgentMessage(context(), caller, params) + }, queuedMessageDelete: ( caller: StructuredAgentSessionCaller, params: Parameters[2] - ) => deleteQueuedStructuredAgentMessage(context(), caller, params), + ) => { + drain.cardMoved(params.envelope.sessionId) + return deleteQueuedStructuredAgentMessage(context(), caller, params) + }, queuedMessagesResume: ( caller: StructuredAgentSessionCaller, params: Parameters[2] diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts index dfaa8f7e6eb7..9501aac5f1cb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts @@ -4,6 +4,10 @@ import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionReconciliationDebts } from './structured-agent-session-reconciliation-pass' +import { + RECONCILIATION_MAX_FAILED_ATTEMPTS, + reconciliationBackoffDelay +} from './structured-agent-session-reconciliation-backoff' export class StructuredAgentSessionReconciliationMemory { /** Proofs a retired worker still held (a lease in recovery, or a run given up), for the next @@ -14,6 +18,14 @@ export class StructuredAgentSessionReconciliationMemory { * accepted by an earlier host process, every one after by this. Kept for the whole process (a * close or a retire must not reset it, or this process's sends would read as an earlier one's). */ private readonly firstOpened = new Map() + /** Startup's lease reconcile is store-wide, so every chat's worker retries it as one: one call + * at a time, and after a failure none before the backoff's next step, so N workers never stack + * N waits on a busy store. */ + private readonly restartReconcile: { + running: Promise | null + failures: number + notBefore: number + } = { running: null, failures: 0, notBefore: 0 } park(sessionId: string, debts: StructuredAgentSessionReconciliationDebts): void { if (debts.evidence) { @@ -37,6 +49,26 @@ export class StructuredAgentSessionReconciliationMemory { return this.firstOpened.get(sessionId) } + /** Whether the store-wide reconcile settled every lease: joined when one runs, false while its + * backoff waits or once it gave up (an attach or a send then reconciles before it acts). */ + reconcileOnce(reconcile: () => Promise): Promise { + const shared = this.restartReconcile + if (shared.running) { + return shared.running + } + // The wall clock, as the workers' own backoff timers run on it. + if (Date.now() < shared.notBefore || shared.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { + return Promise.resolve(false) + } + shared.running = reconcile().then((settled) => { + shared.running = null + shared.failures = settled ? 0 : shared.failures + 1 + shared.notBefore = settled ? 0 : Date.now() + reconciliationBackoffDelay(shared.failures) + return settled + }) + return shared.running + } + /** The chat's record is gone. */ forget(sessionId: string): void { this.parked.delete(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts index 879d73860854..afdbae3531a3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts @@ -13,6 +13,7 @@ import { seedTestAgentSessionRecordStore } from '../../runtime/agent-session-record-store-test-harness' import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' import { attachParamsForRecord } from './structured-agent-session-conversation-open' import type { StructuredAgentSessionHost } from './structured-agent-session-host' import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' @@ -129,6 +130,29 @@ describe("a worker that retires while the chat's lease recovers", () => { }) }) +describe('a live worker waiting out a backoff with an exit account', () => { + it("drops it once a later generation acquired, so generation 14's send stays in doubt", async () => { + const current = await recoveringAt14() + const { reconciliation } = current.collaboratorsForTests() + // The pass fails once (storage busy): the worker backs off still holding generation 13's. + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune').mockRejectedValueOnce( + Object.assign(new Error('database is locked'), { errcode: 5 }) + ) + reconciliation.signal(CHAT, { evidence: proof13, exit: exit13 }) + await reconciliation.attempted(CHAT) + + // Generation 14's recovery is decided meanwhile, with no exit account of its own. + await attach(current) + await idle(current, [CHAT]) + + const { submissions } = await current.journalSnapshot(CHAT) + expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-handed`)).toMatchObject({ + dispatchState: 'unknown' + }) + }) +}) + describe('a chat whose record is gone', () => { it('leaves nothing behind in the reconciliation: no parked proof, no first-open cursor', async () => { const current = await recoveringAt14() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index 79abde80c42a..2ab872d70006 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -69,7 +69,12 @@ export async function runStructuredAgentSessionReconciliationPass( processOpened: AgentJournalCursor ): Promise { const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false } - const recovering = context.deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering' + const lease = context.deps.store.getRecord(sessionId)?.lease + const recovering = lease?.handoffStage === 'recovering' + // An exit's account judges only its own generation: once a later one acquired, it is spent. + if (debts.exit && (!lease || lease.runtimeFence > debts.exit.ownerFence + 1)) { + delete debts.exit + } if ( !recovering && structuredAgentSessionEndedChildHoldsLease(context, sessionId) && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts index d7f847949724..6064ccf46bd4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts @@ -71,8 +71,8 @@ export type StructuredAgentSessionReconciliationContext = /** `done`: a pass found nothing owed, unless a signal came meanwhile; `retire`: nothing this * worker can do, whatever came meanwhile (a later signal starts a new one); `again`: a pass wrote - * rows, so the next one verifies at once; `stale`: no pass ran (its read went stale), which backs - * off like a failure, since nothing moved. */ + * rows, so the next one verifies at once; `stale`: no pass ran (its read went stale, or the + * store-wide reconcile has not settled), which backs off quietly like a failure. */ type Outcome = 'done' | 'retire' | 'again' | 'stale' | 'failed' type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { @@ -106,11 +106,13 @@ export class StructuredAgentSessionReconciliation { if (session) { session.operationalRevision = (session.operationalRevision ?? 0) + 1 } + // Published once the worker owes: the queue's next card waits on an owed release, so no client + // is told that card sends next meanwhile. + const worker = this.disposed ? null : this.workerFor(sessionId) this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) - if (this.disposed) { + if (!worker) { return } - const worker = this.workerFor(sessionId) if (signal.evidence) { worker.debts.evidence = [...(worker.debts.evidence ?? []), signal.evidence] } @@ -133,12 +135,10 @@ export class StructuredAgentSessionReconciliation { } } - /** A handle this host opened on the chat's journal; only its first one marks the boundary. */ - noteOpened = (sessionId: string, journal: Pick): void => + /** A reader opened the chat: its handle marks the process boundary if it is the first, and what + * its worker still waits a slot for goes ahead of the scan. */ + opened = (sessionId: string, journal: Pick): void => { this.memory.noteOpened(sessionId, journal) - - /** A reader opened the chat: whatever its worker still waits a slot for goes ahead of the scan. */ - prioritize = (sessionId: string): void => { const worker = this.workers.get(sessionId) if (worker) { this.slots.prioritize(worker) @@ -217,13 +217,14 @@ export class StructuredAgentSessionReconciliation { if (this.disposed || store.readOnly) { return 'retire' } - // Startup's reconcile failed (storage was busy): retried here, the same deduped step, so a - // gone owner's lease does not read live until a person sends. It never stops a process. + // Startup's reconcile failed (storage was busy): retried as one store-wide step, so a gone + // owner's lease does not read live until a person sends. It never stops a process. + const reconcile = () => this.context.reconcile(sessionId) if ( store.getRecord(sessionId)?.lease.unreconciled && - !(await this.context.reconcile(sessionId)) + !(await this.memory.reconcileOnce(reconcile)) ) { - return 'failed' + return 'stale' } if (!this.context.sessions.has(sessionId) && !worker.loaded) { const stop = await this.load(sessionId, worker) @@ -372,6 +373,8 @@ export class StructuredAgentSessionReconciliation { this.workers.delete(sessionId) if (!this.disposed && this.context.deps.store.getRecord(sessionId)) { this.memory.park(sessionId, worker.debts) + // The queue's automatic send waits on a release a worker owes: it re-derives now. + this.context.publishGenerationEnded(sessionId) } if (!worker.running) { this.dropLoaded(worker) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts new file mode 100644 index 000000000000..3c5b0cb1d011 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts @@ -0,0 +1,106 @@ +// Startup's lease reconcile, failed because another connection held the database, is retried for +// every chat as ONE store-wide step: one call per backoff step however many chats wait on it, so N +// workers never stack N waits on a busy store, and it never stops a process. + +import { DatabaseSync } from 'node:sqlite' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + scanRecord, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined +let locker: DatabaseSync | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-restart-reconcile-retry-')) +}) + +afterEach(async () => { + try { + locker?.exec('ROLLBACK') + } catch { + // Never begun. + } + locker?.close() + locker = undefined + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +async function seedChats(count: number, released: boolean): Promise { + const chats = Array.from({ length: count }, (_, i) => `chat-r${String(i).padStart(6, '0')}`) + await seedTestAgentSessionRecordStore(root, { + records: chats.map((id) => scanRecord(id, released)) + }) + for (const id of chats) { + await seedScanJournal(root, id) + } + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + return chats +} + +describe('a failed startup reconcile, retried by every chat that waits on it', () => { + it('runs once per backoff step for 40 chats, and stops no process', async () => { + await seedChats(40, false) + const reconcile = vi + .spyOn(store, 'reconcileOnRestart') + .mockRejectedValue(Object.assign(new Error('database is locked'), { errcode: 5 })) + const stops: unknown[] = [] + host = openScanHost(root, store, { + probeOwner: async () => ({ outcome: 'pid-absent' }), + stopOwnerProcess: (...args) => stops.push(args) + }) + + await host.reconcileRestartLeases() + await new Promise((resolve) => setTimeout(resolve, 3_500)) + + // Startup, then the shared retry at 0 s, 1 s and 3 s: never one per chat. + expect(reconcile.mock.calls.length).toBeGreaterThanOrEqual(3) + expect(reconcile.mock.calls.length).toBeLessThanOrEqual(4) + expect(stops).toEqual([]) + }) + + it('waits on a real write lock once per backoff step, not once per chat', async () => { + await seedChats(8, true) + expect(store.listRecords().every((record) => record.lease.unreconciled)).toBe(true) + openTestJournalHostDatabase(root).db.pragma('busy_timeout = 300') + const reconcile = vi.spyOn(store, 'reconcileOnRestart') + locker = new DatabaseSync(join(root, 'agent-session-journal.db')) + locker.exec('BEGIN IMMEDIATE') + host = openScanHost(root, store) + + await host.reconcileRestartLeases() + await new Promise((resolve) => setTimeout(resolve, 4_000)) + + expect(reconcile.mock.calls.length).toBeLessThanOrEqual(5) + expect(store.listRecords().every((record) => record.lease.unreconciled)).toBe(true) + + // The lock lifts: the next step reconciles every lease. + locker.exec('ROLLBACK') + await vi.waitFor( + () => expect(store.listRecords().some((record) => record.lease.unreconciled)).toBe(false), + { timeout: 10_000 } + ) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts index b3b961a63446..5e0470430e6e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-subscribers.ts @@ -163,6 +163,11 @@ export class AgentSessionSubscribers { } } + /** The queue as frames publish it, for a history page to read the same. */ + readQueuePublication(sessionId: string): QueuePublication | undefined { + return this.hooks.readQueuePublication?.(sessionId) + } + /** Each subscriber's frame for what changed, with none of a journal write's other edges. */ deliverFrames( sessionId: string, From 9a0db04fda5b9e1b31375493d93a77869a0cfeb7 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:40:51 -0700 Subject: [PATCH 10/22] refactor(native-chat): one retry coordinator for a gone agent's leftover work - an exit records the agent's death before anything it flushes or settles, so no frame reads its work as running - one connection-scoped coordinator replaces the per-chat worker backoff, the store-wide reconcile retry and the queue's own retry timer; give-up ends an episode, and any new trigger or a successful commit starts the next - background writes fail fast on a locked database instead of holding the main thread for the busy timeout; a person's writes keep it - a queued card is delivered over a child this host saw end, through acquisition's own ownership check; one the coordinator gives up on shows the failed-send state even when that hold cannot be saved --- .../scripts/vitest-sqlite-runtime-files.mjs | 5 +- .../agent-session-journal/journal-database.ts | 29 +- .../journal-host-database.ts | 15 +- .../journal-lifecycle-batch-appender.ts | 27 +- .../journal-queued-messages.ts | 22 +- .../journal-row-writer.ts | 21 +- .../journal-store-collaborators.ts | 2 +- .../journal-store-contracts.ts | 6 + .../journal-submission-writer.ts | 11 +- .../journal-unsent-send-hold.ts | 17 +- .../queued-message-holds.ts | 8 +- .../structured-agent-session-agent-start.ts | 3 +- ...ed-agent-session-background-writes.test.ts | 238 +++++++++++ .../structured-agent-session-child-close.ts | 16 +- .../structured-agent-session-child-exit.ts | 34 +- ...tructured-agent-session-client-delivery.ts | 6 +- ...ctured-agent-session-exit-delivery.test.ts | 178 ++++++++ .../structured-agent-session-host.ts | 9 +- .../structured-agent-session-lease-release.ts | 20 +- ...ssion-leftover-settlement-triggers.test.ts | 27 +- ...tured-agent-session-leftover-settlement.ts | 3 + ...ructured-agent-session-mutation-context.ts | 5 +- ...structured-agent-session-provider-child.ts | 16 +- ...ctured-agent-session-queued-abandonment.ts | 36 ++ ...d-agent-session-queued-drain-retry.test.ts | 242 ----------- ...ctured-agent-session-queued-drain-retry.ts | 80 ---- ...tructured-agent-session-queued-messages.ts | 116 +++--- ...ctured-agent-session-queued-publication.ts | 51 ++- .../structured-agent-session-queued-wiring.ts | 5 +- ...red-agent-session-reconciliation-memory.ts | 77 ---- ...ession-reconciliation-parked-debts.test.ts | 22 +- ...tured-agent-session-reconciliation-pass.ts | 30 +- ...gent-session-reconciliation-retry.test.ts} | 4 +- ...ured-agent-session-reconciliation-retry.ts | 311 ++++++++++++++ ...ured-agent-session-reconciliation-visit.ts | 186 +++++++++ ...red-agent-session-reconciliation-worker.ts | 384 ------------------ ...nt-session-restart-reconcile-retry.test.ts | 9 +- ...uctured-agent-session-restart-reconcile.ts | 54 ++- ...ctured-agent-session-resume-eligibility.ts | 23 +- .../structured-agent-session-reveal.ts | 22 +- ...ctured-agent-session-startup-settlement.ts | 4 +- ...ured-agent-session-unexpected-exit.test.ts | 8 +- ...red-agent-session-unsent-send-hold.test.ts | 10 +- .../runtime/agent-session-record-store.ts | 26 +- .../agent-session-store-transactions.ts | 18 +- ...gent-session-surface-release-transition.ts | 10 +- src/main/sqlite/sqlite-read-failure.ts | 13 + 47 files changed, 1432 insertions(+), 1027 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts rename src/main/native-chat/agent-session-wire/{structured-agent-session-reconciliation-worker.test.ts => structured-agent-session-reconciliation-retry.test.ts} (98%) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts delete mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 9b091c6d7018..22fac58d8b9c 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -116,12 +116,13 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-lease-renewer.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts', - 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts', - 'src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', diff --git a/src/main/native-chat/agent-session-journal/journal-database.ts b/src/main/native-chat/agent-session-journal/journal-database.ts index 0dee9537b3f1..ebd0e6ab00dd 100644 --- a/src/main/native-chat/agent-session-journal/journal-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-database.ts @@ -18,6 +18,14 @@ import { ensureCommandReceiptsTable } from './command-receipt-schema' import { ensureAgentSessionAttachmentClaimTables } from '../agent-session-attachments/agent-session-attachment-claims' export const JOURNAL_BUSY_TIMEOUT_MS = 5000 + +/** `background`: bookkeeping no person waits on. It never waits for another connection's lock: + * SQLITE_BUSY at once, and its caller retries on its own schedule. */ +export type JournalWriteOptions = { background?: true } + +export function backgroundWrite(background: true | undefined): JournalWriteOptions | undefined { + return background ? { background } : undefined +} /** Bounds the WAL a checkpoint leaves behind; SQLite truncates it back to this after a reset. */ export const JOURNAL_SIZE_LIMIT_BYTES = 32 * 1024 * 1024 /** Every commit is fsynced before its caller continues. */ @@ -139,7 +147,26 @@ function migrateJournalSchema(db: Database.Database, stored: number): void { export function runJournalTransaction( db: Database.Database, run: (db: Database.Database) => T, - onStranded: () => void = () => undefined + onStranded: () => void = () => undefined, + options: JournalWriteOptions = {} +): T { + if (!options.background) { + return runImmediateTransaction(db, run, onStranded) + } + // Set and restored around this one synchronous transaction: no other statement runs between. + const busyTimeout = journalPragmaNumber(db, 'busy_timeout') + db.pragma('busy_timeout = 0') + try { + return runImmediateTransaction(db, run, onStranded) + } finally { + db.pragma(`busy_timeout = ${busyTimeout}`) + } +} + +function runImmediateTransaction( + db: Database.Database, + run: (db: Database.Database) => T, + onStranded: () => void ): T { db.exec('BEGIN IMMEDIATE') try { diff --git a/src/main/native-chat/agent-session-journal/journal-host-database.ts b/src/main/native-chat/agent-session-journal/journal-host-database.ts index aaea0228b6e9..264fcfae04f5 100644 --- a/src/main/native-chat/agent-session-journal/journal-host-database.ts +++ b/src/main/native-chat/agent-session-journal/journal-host-database.ts @@ -10,8 +10,10 @@ import type Database from '../../sqlite/sync-database' import { openJournalDatabase, runJournalTransaction, + type JournalWriteOptions, type OpenJournalDatabase } from './journal-database' +export type { JournalWriteOptions } from './journal-database' import { journalOpenRefusalError } from './journal-open-failure' import { AgentSessionJournalError } from './journal-write-guards' @@ -61,10 +63,15 @@ export class JournalHostDatabase { } /** One IMMEDIATE transaction; see `runJournalTransaction`. */ - transaction(run: (db: Database.Database) => T): T { - return runJournalTransaction(this.db, run, () => { - this.stranded = true - }) + transaction(run: (db: Database.Database) => T, options?: JournalWriteOptions): T { + return runJournalTransaction( + this.db, + run, + () => { + this.stranded = true + }, + options + ) } /** Last, after every store has drained. A close that fails keeps the handle, so the retried diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index 99548fa5a81f..a1a79d92cd28 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -20,6 +20,7 @@ import type { JournalResolvedLifecycleBatchInput } from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' +import type { JournalWriteOptions } from './journal-database' import { journalQueuedRejectionRowBuilders } from './journal-pending-submission-recovery' export class JournalLifecycleBatchAppender { @@ -28,7 +29,8 @@ export class JournalLifecycleBatchAppender { state: () => JournalReducerState cursor: () => AgentJournalCursor enqueueRows: ( - plan: () => readonly ((seq: number, ts: number) => JournalRow)[] + plan: () => readonly ((seq: number, ts: number) => JournalRow)[], + options?: JournalWriteOptions ) => Promise } ) {} @@ -66,16 +68,19 @@ export class JournalLifecycleBatchAppender { /** `append`, with every row chosen at the batch's own turn in the queue. */ appendPlanned(input: JournalPlannedLifecycleBatchInput): Promise { return this.deps - .enqueueRows(() => { - if (this.wasApplied(input.settlementId)) { - return [] - } - const { mutations, dispatches } = input.plan() - return [ - ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), - ...this.planMutations(input, mutations) - ] - }) + .enqueueRows( + () => { + if (this.wasApplied(input.settlementId)) { + return [] + } + const { mutations, dispatches } = input.plan() + return [ + ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), + ...this.planMutations(input, mutations) + ] + }, + input.background ? { background: true } : undefined + ) .then((rows) => this.cursorAfter(rows)) } diff --git a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts index 3337727689cd..4041e47aede3 100644 --- a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts +++ b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts @@ -13,12 +13,12 @@ import { AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS, AGENT_SESSION_OPERATION_FUTURE_SKEW_MS } from '../../../shared/agent-session-host-authority' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import type { JournalReducerState } from './journal-reducer' import type { JournalRow } from './journal-row-schema' import type { JournalOperationReceipt } from './journal-row-writer' import type { JournalSubmissionConsume } from './journal-store-contracts' -import { holdQueuedMessages } from './queued-message-holds' +import { holdQueuedMessages, type QueuedMessageHold } from './queued-message-holds' import { deriveQueuePauses, journalUserStopInForce, @@ -149,17 +149,18 @@ export class JournalQueuedMessages { return row }, () => inserted, - receipt?.committed + { adopted: receipt?.committed } ) } /** Hold one waiting draft whose conversion failed. Stored on the row, so it * survives handle eviction and restart; withdraw and consume clear it in their * own UPDATE. */ - hold(input: { messageIds: readonly string[]; reason: QueuedMessageHoldReason }): Promise { + hold(input: QueuedMessageHold & JournalWriteOptions): Promise { return this.transact( (db) => holdQueuedMessages(db, { ...input, sessionId: this.deps.sessionId }), - (held) => held > 0 + (held) => held > 0, + input ).then(() => undefined) } @@ -259,13 +260,13 @@ export class JournalQueuedMessages { private transact( run: (db: Database.Database) => JournalWriteResult, changed: (result: T) => boolean, - adopted?: () => void + options: { adopted?: (() => void) | undefined } & JournalWriteOptions = {} ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) - const result = this.deps.database().transaction(run) + const result = this.deps.database().transaction(run, options) if (changed(result)) { - adopted?.() + options.adopted?.() this.changeRevision++ this.deps.committed() } @@ -358,7 +359,7 @@ export class JournalQueuedMessages { * crash → downgrade → upgrade, where the old build rejected the leftover with * no hook), then retention runs. */ - repairAndPrune(): Promise { + repairAndPrune(options?: JournalWriteOptions): Promise { // No draft, no work, and no write. if (this.deps.readOnly() || this.list().length === 0) { return Promise.resolve() @@ -377,7 +378,8 @@ export class JournalQueuedMessages { }) ) }, - (changed) => changed > 0 + (changed) => changed > 0, + options ).then(() => undefined) } } diff --git a/src/main/native-chat/agent-session-journal/journal-row-writer.ts b/src/main/native-chat/agent-session-journal/journal-row-writer.ts index 4eb2ff6bdd8e..3cf4e5e51891 100644 --- a/src/main/native-chat/agent-session-journal/journal-row-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-row-writer.ts @@ -1,6 +1,7 @@ import type Database from '../../sqlite/sync-database' import { insertJournalRow } from './journal-row-table' import type { JournalHostDatabase } from './journal-host-database' +import type { JournalWriteOptions } from './journal-database' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { JournalRow } from './journal-row-schema' import { assertJournalFence, assertJournalWritable } from './journal-write-guards' @@ -47,7 +48,8 @@ export class JournalRowWriter { enqueue( build: (seq: number, ts: number) => JournalRow, hook?: JournalRowTransactionHook, - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) @@ -61,7 +63,7 @@ export class JournalRowWriter { hook?.(db, row) receipt?.write(db) this.runBookkeeping(db, row) - }) + }, options) } catch (error) { this.deps.rolledBack?.() throw error @@ -80,15 +82,17 @@ export class JournalRowWriter { * durable unless all are, so no reader ever meets some without the rest. */ enqueueRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[], - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): Promise { - return this.deps.serialize(() => this.writeRows(plan, receipt)) + return this.deps.serialize(() => this.writeRows(plan, receipt, options)) } /** `enqueueRows`' write, for a caller already running at its own turn in the queue. */ writeRows( plan: () => readonly ((seq: number, ts: number) => JournalRow)[], - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): JournalRow[] { assertJournalWritable(this.deps.readOnly(), this.deps.sessionId) const first = this.deps.nextSequence() @@ -108,7 +112,7 @@ export class JournalRowWriter { this.runBookkeeping(db, row) } receipt?.write(db) - }) + }, options) } catch (error) { this.deps.rolledBack?.() throw error @@ -137,9 +141,10 @@ export class JournalRowWriter { append( build: (seq: number, ts: number) => JournalRow, hook?: JournalRowTransactionHook, - receipt?: JournalOperationReceipt + receipt?: JournalOperationReceipt, + options?: JournalWriteOptions ): Promise { - return this.enqueue(build, hook, receipt).then((row) => ({ + return this.enqueue(build, hook, receipt, options).then((row) => ({ epoch: row.epoch, sequence: row.seq })) diff --git a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts index 1d1ced079f77..7033bf952ae0 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-collaborators.ts @@ -118,7 +118,7 @@ export function createJournalStoreCollaborators(host: JournalStoreHost): Journal const lifecycleBatchAppender = new JournalLifecycleBatchAppender({ state: host.state, cursor: host.cursor, - enqueueRows: (plan) => rowWriter.enqueueRows(plan) + enqueueRows: (plan, options) => rowWriter.enqueueRows(plan, undefined, options) }) return { epochController, diff --git a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts index 5f16fcfc4f06..4e6f948c6629 100644 --- a/src/main/native-chat/agent-session-journal/journal-store-contracts.ts +++ b/src/main/native-chat/agent-session-journal/journal-store-contracts.ts @@ -32,6 +32,8 @@ export type ResolveDispatchInput = { clientMessageId: string fence: number recovered?: true + /** Bookkeeping no person waits on (`JournalWriteOptions`); not written to the row. */ + background?: true } & /** A null identity: the provider took the message without echoing an item of its own, as a * conversation command it carries out in place. */ @@ -97,6 +99,8 @@ export type JournalPlannedLifecycleBatchInput = Pick< JournalLifecycleBatchInput, 'settlementId' | 'fence' | 'recovered' > & { + /** Bookkeeping no person waits on (`JournalWriteOptions`). */ + background?: true plan: () => { mutations: readonly JournalLifecycleMutationInput[] dispatches: readonly ResolveDispatchInput[] @@ -138,6 +142,8 @@ export type JournalSubmissionConsume = { /** The queue's own send: refused in the consume's transaction while the queue's pause holds * the card. Send-now omits it. */ yieldsToPause?: true + /** The queue's automatic send is bookkeeping no person waits on (`JournalWriteOptions`). */ + background?: true } export type JournalItemAppendInput = { diff --git a/src/main/native-chat/agent-session-journal/journal-submission-writer.ts b/src/main/native-chat/agent-session-journal/journal-submission-writer.ts index 1fc0ac21d0da..d397a74f6dc3 100644 --- a/src/main/native-chat/agent-session-journal/journal-submission-writer.ts +++ b/src/main/native-chat/agent-session-journal/journal-submission-writer.ts @@ -6,6 +6,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import type { JournalQueuedMessages } from './journal-queued-messages' +import { backgroundWrite } from './journal-database' import { journalSubmissionHook, type JournalAttachmentClaim } from './journal-submission-hook' import type { JournalReducerState } from './journal-reducer' import { journalDispatchRowBuilder, journalSubmissionRowBuilder } from './journal-row-builders' @@ -48,7 +49,8 @@ export class JournalSubmissionWriter { return rowWriter.append( journalSubmissionRowBuilder(state, identity, input, consume), journalSubmissionHook(queuedMessages, claimAttachments, input, consume), - receipt + receipt, + backgroundWrite(consume?.background) ) } @@ -64,6 +66,11 @@ export class JournalSubmissionWriter { input: ResolveDispatchInput, hook?: JournalRowTransactionHook ): Promise { - return this.deps.rowWriter.append(journalDispatchRowBuilder(this.deps.state, input), hook) + return this.deps.rowWriter.append( + journalDispatchRowBuilder(this.deps.state, input), + hook, + undefined, + backgroundWrite(input.background) + ) } } diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts index 037f167b9474..6a4abb6e2306 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts @@ -18,6 +18,7 @@ import { agentSessionFailureFact } from '../../../shared/agent-session-failure' import { agentSessionFailureWords } from '../../../shared/agent-session-failure-words' import { agentSessionSendBodyFingerprint } from '../../../shared/structured-agent-session-send-mutation' import type { AgentSessionJournal } from './journal-store' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import type { JournalRowTransactionHook } from './journal-row-writer' import type { QueuedMessagePositionMove } from './queued-message-positions' @@ -88,8 +89,11 @@ export async function holdUnsentSends( /** In place of the queued sends: those a child that ended before it answered its start was * handed and never echoed. It ran none, so each is unsent as surely as a queued one. */ unrun?: true + /** Bookkeeping no person waits on (`JournalWriteOptions`): contention stops it, untouched. */ + background?: true } ): Promise { + const background = input.background ? { background: input.background } : {} const { hold } = input const unsent = journal .submissions() @@ -165,10 +169,17 @@ export async function holdUnsentSends( try { // The send names its card in the same row, so no surface draws it once the card is gone. await journal.resolveDispatch( - card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject, + { + ...(card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject), + ...background + }, keep ) } catch (error) { + // Another connection holds the database: nothing was written, so nothing is lost by waiting. + if (input.background && isSqliteContentionFailure(error)) { + throw error + } if (!keep) { failures.push(error) continue @@ -181,7 +192,9 @@ export async function holdUnsentSends( cause: hold.cause, error: error instanceof Error ? error.message : String(error) }) - await journal.resolveDispatch(reject).catch((fallback: unknown) => failures.push(fallback)) + await journal + .resolveDispatch({ ...reject, ...background }) + .catch((fallback: unknown) => failures.push(fallback)) } } if (failures.length > 0) { diff --git a/src/main/native-chat/agent-session-journal/queued-message-holds.ts b/src/main/native-chat/agent-session-journal/queued-message-holds.ts index c3f90590c117..6134594778da 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-holds.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-holds.ts @@ -6,15 +6,13 @@ import type Database from '../../sqlite/sync-database' import type { QueuedMessageHoldReason } from './queued-message-table' +export type QueuedMessageHold = { messageIds: readonly string[]; reason: QueuedMessageHoldReason } + /** Hold waiting drafts from auto-sending. The hold retires with the row: consume * and withdraw clear it in their own UPDATE. Returns how many rows it newly reached. */ export function holdQueuedMessages( db: Database.Database, - input: { - sessionId: string - messageIds: readonly string[] - reason: QueuedMessageHoldReason - } + input: QueuedMessageHold & { sessionId: string } ): number { const update = db.prepare( `UPDATE queued_messages SET hold_reason = ? diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts index 2c180c121b33..f61746cb05ee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-agent-start.ts @@ -146,7 +146,8 @@ async function startStructuredAgentSessionAgent( } const params = structuredAgentSessionResumeParams( record, - structuredAgentSessionResumeOperationId(context.now()) + structuredAgentSessionResumeOperationId(context.now()), + context.sessions.get(sessionId)?.lastEndedChild ) if (!params) { return record.lease.unreconciled diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts new file mode 100644 index 000000000000..ca4f034f0053 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -0,0 +1,238 @@ +// Background bookkeeping meeting another connection's write lock: it fails at once and ends the +// retry's round (no 5 s wait on the main thread), while a person's write in the same tick still +// waits the lock out. One episode's give-up is never a latch: a commit or a signal re-arms it, and +// a queued send given up on reads as not sent even when its stored hold cannot be written. + +import { Worker } from 'node:worker_threads' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import { journalDatabasePath } from '../agent-session-journal/journal-host-database' +import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + currentJournal, + exitWhileSettlementFails, + leaveUnfinishedWork, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' + +let rig: QueuedMessageTestRig | undefined +let locker: Worker | undefined + +afterEach(async () => { + vi.useRealTimers() + await locker?.terminate() + locker = undefined + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Another connection, on its own thread, holds the write lock for `ms`: this thread's waits + * cannot release it, as another process's could not. Resolves once the lock is held. */ +function holdWriteLock(stateDirectory: string, ms: number): Promise<{ released: Promise }> { + const worker = new Worker( + `const { workerData, parentPort } = require('node:worker_threads') + const { DatabaseSync } = require('node:sqlite') + const db = new DatabaseSync(workerData.file) + db.exec('BEGIN IMMEDIATE') + parentPort.postMessage('locked') + setTimeout(() => { db.exec('ROLLBACK'); db.close(); parentPort.postMessage('released') }, workerData.ms)`, + { eval: true, workerData: { file: journalDatabasePath(stateDirectory), ms } } + ) + locker = worker + let release: () => void = () => undefined + const released = new Promise((resolve) => (release = resolve)) + return new Promise((resolve) => + worker.on('message', (message) => (message === 'locked' ? resolve({ released }) : release())) + ) +} + +/** The longest the main thread went without running a timer, while `run` ran. */ +async function longestStall(run: () => Promise): Promise { + let last = performance.now() + let longest = 0 + const probe = setInterval(() => { + const now = performance.now() + longest = Math.max(longest, now - last) + last = now + }, 2) + try { + await run() + return Math.max(longest, performance.now() - last) + } finally { + clearInterval(probe) + } +} + +const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) + +/** A chat whose exit left its settlement owed to the retry, with nothing refusing it now. */ +async function settlementOwed(): Promise { + rig = await createQueuedMessageTestRig({ restartable: true }) + await rig.workingSend() + await leaveUnfinishedWork(rig, { prompt: true }) + const { database } = await exitWhileSettlementFails(rig) + database.db.exec('DROP TRIGGER reject_recovered') + return rig +} + +describe("another connection's write lock", () => { + it('ends a background round at once, never stalling the main thread, and a send then waits only for the lock', async () => { + const current = await settlementOwed() + const { reconciliation } = current.host.collaboratorsForTests() + const { released } = await holdWriteLock(current.root, 1_500) + + const started = performance.now() + const stall = await longestStall(async () => { + reconciliation.signal(SESSION) + await reconciliation.attempted(SESSION) + }) + expect(performance.now() - started).toBeLessThan(150) + expect(stall).toBeLessThan(60) + expect(reconciliation.owes(SESSION)).toBe(true) + expect(turnState(current)).toBe('running') + + // A person's send waits the lock out (its own 5 s budget), never behind background work. + const sent = performance.now() + const { result } = current.send('go on') + expect(await result).toMatchObject({ ok: true }) + await released + expect(performance.now() - sent).toBeLessThan(2_000) + // Its commit proves the connection free: a fresh episode lands the settlement at once. + await eventually(() => expect(turnState(current)).toBe('interrupted')) + }) + + it('refuses a background transaction at once while a write in the same tick waits it out', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const { store } = rig + const database = openTestJournalHostDatabase(rig.root) + const busyTimeout = database.db.pragma('busy_timeout', { simple: true }) + const touch = + (by: number) => (record: Parameters[1]>[0]) => ({ + ...record, + lease: { ...record.lease, lastRenewedAt: record.lease.lastRenewedAt + by } + }) + const { released } = await holdWriteLock(rig.root, 400) + + const started = performance.now() + const background = store.transitionHandoff(SESSION, touch(1), { background: true }) + const user = store.transitionHandoff(SESSION, touch(2)) + + // Refused, though the lock lifts long inside its 5 s: it never waited. + await expect(background).rejects.toSatisfy(isSqliteContentionFailure) + await expect(user).resolves.toMatchObject({ sessionId: SESSION }) + expect(performance.now() - started).toBeGreaterThan(300) + await released + expect(database.db.pragma('busy_timeout', { simple: true })).toBe(busyTimeout) + }) +}) + +describe('an episode the retry gives up', () => { + /** Lets every visit a fired timer began run to its end, with no fake time passing. */ + async function settleTurns(): Promise { + for (let turn = 0; turn < 50; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } + } + + async function exhaust(current: QueuedMessageTestRig): Promise> { + const attempts = vi + .spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') + .mockRejectedValue(locked()) + for (let round = 0; round < 12; round += 1) { + await vi.advanceTimersByTimeAsync(31_000) + await settleTurns() + } + const given = attempts.mock.calls.length + await vi.advanceTimersByTimeAsync(120_000) + await settleTurns() + // Ten rounds, then no timer: the set is kept, and nothing retries on its own. + expect(attempts.mock.calls.length).toBe(given) + expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(true) + attempts.mockRestore() + return attempts + } + + it.each(['a commit on the connection', 'a new signal'])( + 'is re-armed by %s, and the work lands', + async (trigger) => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + const current = await settlementOwed() + await exhaust(current) + expect(turnState(current)).toBe('running') + + const { reconciliation } = current.host.collaboratorsForTests() + if (trigger === 'a new signal') { + reconciliation.signal(SESSION) + } else { + await currentJournal(current).appendQueueResume( + current.store.getRecord(SESSION)!.lease.runtimeFence + ) + } + await settleTurns() + + expect(turnState(current)).toBe('interrupted') + expect(reconciliation.owes(SESSION)).toBe(false) + } + ) +}) + +describe('a queued send the retry gives up on', () => { + it('reads as not sent from memory when its stored hold cannot be written either', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockRejectedValue(locked()) + const hold = vi.spyOn(JournalQueuedMessages.prototype, 'hold').mockRejectedValue(locked()) + + await current.settleAccepted(working, 'a') + for (let round = 0; round < 12; round += 1) { + await vi.advanceTimersByTimeAsync(31_000) + } + + expect(hold).toHaveBeenCalledTimes(1) + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages).toEqual([ + expect.objectContaining({ + messageId: queued.value.queued.messageId, + paused: true, + pausedReason: QUEUED_MESSAGE_PAUSED_SEND_FAILED + }) + ]) + expect(page.ok && page.page.nextQueuedMessageId).toBeNull() + }) + + it('holds the card with "Couldn\'t send" at once for a refusal', async () => { + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + await current.send('after this', 'queue-if-active').result + vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockRejectedValueOnce( + new Error('disk full') + ) + + await current.settleAccepted(working, 'a') + await eventually(async () => { + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages?.[0]?.pausedReason).toBe( + QUEUED_MESSAGE_PAUSED_SEND_FAILED + ) + }) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index be3ccd7645e7..34ef73dea479 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -15,6 +15,8 @@ import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' +import type { JournalWriteOptions } from '../agent-session-journal/journal-database' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { releaseStoredStructuredAgentSessionOwnerAfterExit } from './structured-agent-session-lease-release' import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' @@ -126,7 +128,8 @@ export function structuredAgentSessionEndedChildHoldsLease( * whether the lease still names that child. */ export async function releaseLeaseOfEndedStructuredAgentSessionChild( context: Pick, - sessionId: string + sessionId: string, + options?: JournalWriteOptions ): Promise { const ended = context.sessions.get(sessionId)?.lastEndedChild if (!ended || !structuredAgentSessionEndedChildHoldsLease(context, sessionId)) { @@ -137,13 +140,18 @@ export async function releaseLeaseOfEndedStructuredAgentSessionChild( sessionId, expectedFence: ended.fence, now: context.now(), - ...(ended.reason ? { exitReason: ended.reason } : {}) - }).catch((error: unknown) => + ...(ended.reason ? { exitReason: ended.reason } : {}), + ...options + }).catch((error: unknown) => { + // Background work stops on contention, for its retry; anything else is only reported. + if (options?.background && isSqliteContentionFailure(error)) { + throw error + } context.deps.logger.warn("releasing an exited agent's lease failed", { scope: 'ended-child-lease-release', sessionId, error }) - ) + }) return structuredAgentSessionEndedChildHoldsLease(context, sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 3dd260ddf140..808703a120bc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -14,7 +14,7 @@ import type { StructuredAgentSessionHostSession, StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' -import { endProviderChild } from './structured-agent-session-provider-child' +import { recordProviderChildEnd } from './structured-agent-session-provider-child' import { releaseStoredStructuredAgentSessionOwnerAfterExit, type StructuredAgentSessionLeaseStore @@ -160,27 +160,33 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< ? ('hostRestarted' as const) : ('chatClosed' as const) : undefined + // Death first: before the tail, the settlement or the release writes anything, every reader + // projects this generation as ended, so no frame they publish reads it Working and no prompt it + // raised is answerable. The child stays on record (its sink drains the tail) until `endChild`. + recordProviderChildEnd(session, { + generation: child.generation, + fence: child.fence, + // A close keeps the cause of the stop that asked for it, and ends where it was asked. + cause: expected ? (close?.cause ?? 'evict') : 'exit', + reason: expected ? (close?.reason ?? null) : exit.reason, + ...(!expected && exit.failure ? { failure: exit.failure } : {}), + duringStartup: exitedDuringStartup, + // The adapter publishes an exit only once it saw the root go, first-hand or proven. + rootGone: true, + ...(expected && close ? { endedAt: close.requestedAt } : {}) + }) + context.generationEnded(sessionId) const endChild = (): void => { context.startupAttempts?.childEnded(sessionId, child) - endProviderChild(session, { - generation: child.generation, - fence: child.fence, - // A close keeps the cause of the stop that asked for it, and ends where it was asked. - cause: expected ? (close?.cause ?? 'evict') : 'exit', - reason: expected ? (close?.reason ?? null) : exit.reason, - ...(!expected && exit.failure ? { failure: exit.failure } : {}), - duringStartup: exitedDuringStartup, - // The adapter publishes an exit only once it saw the root go, first-hand or proven. - rootGone: true, - ...(expected && close ? { endedAt: close.requestedAt } : {}) - }) + if (session.child === child) { + session.child = null + } context.publishStatus?.(sessionId) } const record = context.store.getRecord(sessionId) if (!record || record.lease.handoffStage !== null) { // An acquisition or recovery already owns this lease's transition. endChild() - context.generationEnded(sessionId) context.wakeDelivery?.(sessionId) return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index b6a3edb4d4b0..2f51f49d3741 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -8,7 +8,7 @@ import { AgentSessionSubscribers } from './structured-agent-session-subscribers' import { structuredQueueSendGate, tryReadQueuePublication, - type QueuedDrainWaits + type QueuedDrainHolds } from './structured-agent-session-queued-publication' import type { StructuredAgentSessionHostDeps, @@ -42,7 +42,7 @@ export class StructuredAgentSessionClientDelivery { * idle clock; a generation's end is not one), and a card it holds back is never named next. */ private readonly queue: { onJournalActivity: (sessionId: string, activity?: boolean) => void - drain: { waits: QueuedDrainWaits } + drain: QueuedDrainHolds }, onAgentStarted: (sessionId: string) => void, /** A session's child records changed; the chat strip republishes from them. */ @@ -75,7 +75,7 @@ export class StructuredAgentSessionClientDelivery { structuredQueueSendGate( { store: this.deps().store, sessions }, sessionId, - this.queue.drain.waits + this.queue.drain ) ), readBackgroundTasks, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts new file mode 100644 index 000000000000..c4a9ad9d3484 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts @@ -0,0 +1,178 @@ +// A queued card when the agent ahead of it exits: every frame from the exit on reads the dead +// generation as ended (not Working, none of its prompts answerable), and the card is sent at once, +// whether the exit's release write lands, waits or keeps failing: an acquisition then replaces the +// owner once it is proven dead, and never one that is alive or unproven. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import { + agentSessionLeaseFixture, + agentSessionRecordFixture +} from '../../../shared/agent-session-record.test-fixture' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession +} from '../../../shared/structured-agent-session-reducer' +import type { StructuredAgentSessionHostDeps } from './structured-agent-session-host-types' +import { + exitChild, + leaveUnfinishedWork +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + eventually, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** A card queued behind a running turn that raised a prompt. */ +async function queuedBehindPrompt( + probeOwner?: StructuredAgentSessionHostDeps['probeOwner'] +): Promise<{ current: QueuedMessageTestRig; draftId: string }> { + rig = await createQueuedMessageTestRig({ + restartable: true, + ...(probeOwner ? { probeOwner } : {}) + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + return { current, draftId: queued.value.queued.messageId } +} + +/** Every release write refused, as storage that keeps failing would. */ +function refuseReleases(current: QueuedMessageTestRig): () => number { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const transition = current.store.transitionHandoff.bind(current.store) + let refused = 0 + vi.spyOn(current.store, 'transitionHandoff').mockImplementation((sessionId, change, options) => + transition( + sessionId, + (record) => { + const next = change(record) + if (next.lease.claimStatus === 'released') { + refused += 1 + throw new Error('disk full') + } + return next + }, + options + ) + ) + return () => refused +} + +/** What a client reads after each frame, from the host's own fields. */ +async function watchChat(current: QueuedMessageTestRig) { + let state = EMPTY_STRUCTURED_AGENT_SESSION + const reads: { working: boolean; settled: boolean; prompts: number; cardSent: boolean }[] = [] + await current.host.subscribe({ + id: 'chat', + sessionId: SESSION, + emit: (event) => { + state = reduceStructuredAgentSession(state, { type: 'event', event }) + reads.push({ + working: state.working === true, + settled: state.latestTurn?.turn.state !== 'running', + prompts: state.actionablePromptIds?.length ?? 0, + cardSent: state.submissions.some((entry) => entry.queuedMessageId !== undefined) + }) + } + }) + return reads +} + +const provenDead: StructuredAgentSessionHostDeps['probeOwner'] = async () => ({ + outcome: 'pid-absent' +}) + +describe('an exit whose settlement lands while its release keeps failing', () => { + it('reads the dead generation as ended in every frame, and the card is sent over the proven-dead owner', async () => { + const { current, draftId } = await queuedBehindPrompt(provenDead) + const refused = refuseReleases(current) + const reads = await watchChat(current) + const before = reads.length + + await exitChild(current) + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + await eventually(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) + + expect(refused()).toBeGreaterThan(0) + const untilSent = reads.slice(before).filter((read) => !read.cardSent) + expect(untilSent.length).toBeGreaterThan(0) + // Not Working, and the dead generation's prompt answerable, in no frame before the card's send. + expect(untilSent.filter((read) => read.working)).toEqual([]) + expect(untilSent.filter((read) => read.prompts > 0)).toEqual([]) + // Its turn was settled before the send, and the lease belongs to the generation that took it. + expect(untilSent.at(-1)?.settled).toBe(true) + expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') + }) +}) + +describe('an exit that settles and releases cleanly', () => { + it('sends the card with no added delay', async () => { + const { current, draftId } = await queuedBehindPrompt() + const started = Date.now() + + await exitChild(current) + await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) + await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) + + // Inside one turn of the chat's lane: no retry round, no backoff. + expect(Date.now() - started).toBeLessThan(250) + expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(false) + }) +}) + +describe('an owner this host cannot prove dead', () => { + it.each([ + ['alive', { outcome: 'identity-matched', matchedOn: ['process-start-time'] }], + ['unproven', { outcome: 'indeterminate', reason: 'no start time' }] + ] satisfies [string, AgentSessionOwnerProbe][])( + 'is never replaced when %s: no agent starts over it, and the card comes back', + async (_name, probe) => { + const { current, draftId } = await queuedBehindPrompt(async () => probe) + refuseReleases(current) + const owner = current.store.getRecord(SESSION)?.lease + + await exitChild(current) + await eventually(async () => + expect((await current.handoff(draftId))?.dispatchState).toBe('rejected') + ) + + expect(current.dispatch).toHaveBeenCalledTimes(1) + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + claimStatus: 'live', + runtimeFence: owner?.runtimeFence, + ownerProcess: owner?.ownerProcess + }) + expect(await current.drafts()).toEqual([{ messageId: draftId, state: 'returned' }]) + } + ) + + it('widens nothing for a lease this host never saw end: another fence, or no root proof', () => { + const record = agentSessionRecordFixture( + agentSessionLeaseFixture({ claimStatus: 'live', handoffStage: null, runtimeFence: 7 }) + ) + expect(isResumableStructuredAgentSessionRecord(record)).toBe(false) + expect(isResumableStructuredAgentSessionRecord(record, { fence: 6, rootGone: true })).toBe( + false + ) + expect(isResumableStructuredAgentSessionRecord(record, { fence: 7, rootGone: false })).toBe( + false + ) + expect(isResumableStructuredAgentSessionRecord(record, { fence: 7, rootGone: true })).toBe(true) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index 6cafd28e7ff3..c9a57d137529 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -137,7 +137,12 @@ export class StructuredAgentSessionHost { // `hasSession` inside the same serialized step as this `set`. onReadable: this.conversationDelivery.adoptOpened, onUnopened: (sessionId) => this.tabs.markUnopened(sessionId), - startup: { sessions: this.sessions, tasks: this.tasks, clientDelivery: this.clientDelivery } + startup: { + sessions: this.sessions, + tasks: this.tasks, + clientDelivery: this.clientDelivery, + queue: this.queued.drain + } }) this.eventRecovery = new StructuredAgentSessionEventRecovery({ deps, @@ -287,7 +292,7 @@ export class StructuredAgentSessionHost { wakeDelivery: (sessionId) => this.conversationDelivery.loop.wake(sessionId), stopAgent: (sessionId, ending) => this.lifetime.stopAgent(sessionId, ending), wakeQueuedDrain: (sessionId) => this.queued.drain.schedule(sessionId), - reconciliationOwes: (sessionId) => this.restore.reconciliation.owes(sessionId), + retry: this.restore.reconciliation, acquireAborts: this.runtimeState.acquireAborts, optionRevisions: this.runtimeState.optionRevisions, now: () => this.now() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts index 63e3add288e4..e752f02ed840 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts @@ -26,6 +26,8 @@ export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { now: number exitObservedAt?: number exitReason?: string + /** Bookkeeping no person waits on (`JournalWriteOptions`). */ + background?: true }): Promise { const record = input.store.getRecord(input.sessionId) if ( @@ -35,11 +37,15 @@ export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { ) { throw new Error('agent_session_checkpoint_stale') } - return releaseStoredAgentSessionOwnerAfterSurfaceClose(input.store, { - sessionId: input.sessionId, - expectedFence: input.expectedFence, - now: input.now, - ...(input.exitObservedAt === undefined ? {} : { exitObservedAt: input.exitObservedAt }), - ...(input.exitReason ? { exitReason: input.exitReason } : {}) - }) + return releaseStoredAgentSessionOwnerAfterSurfaceClose( + input.store, + { + sessionId: input.sessionId, + expectedFence: input.expectedFence, + now: input.now, + ...(input.exitObservedAt === undefined ? {} : { exitObservedAt: input.exitObservedAt }), + ...(input.exitReason ? { exitReason: input.exitReason } : {}) + }, + input.background ? { background: true } : undefined + ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts index f8f2befc1843..138fb3cb1377 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -1,4 +1,4 @@ -// The chat's reconciliation worker, driven by every way a generation ends: what it settles, that +// The retry of background bookkeeping, driven by every way a generation ends: what it settles, that // it settles in one commit, and that nothing a person does ever waits on it. import { afterEach, describe, expect, it, vi } from 'vitest' @@ -201,7 +201,7 @@ describe('a settlement that cannot commit never gates the user', () => { }) }) -describe('the worker’s retry', () => { +describe('the retry', () => { /** Lets every attempt a fired timer began run to its end, with no fake time passing. */ async function settleTurns(): Promise { for (let turn = 0; turn < 50; turn++) { @@ -209,7 +209,7 @@ describe('the worker’s retry', () => { } } - it('backs off from 1 s, coalesces the signals meanwhile, and settles once the fault clears', async () => { + it('backs off from 1 s, doubling; a new signal starts a fresh episode at once; settles once the fault clears', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) try { rig = await createQueuedMessageTestRig({ restartable: true }) @@ -221,26 +221,31 @@ describe('the worker’s retry', () => { const attempts = vi.spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') expect(reconciliation.owes(SESSION)).toBe(true) - // Signals during the backoff join the one worker; none runs an attempt of its own. - reconciliation.signal(SESSION) - reconciliation.signal(SESSION, { restate: true }) - await settleTurns() await vi.advanceTimersByTimeAsync(990) await settleTurns() expect(attempts).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(10) await settleTurns() expect(attempts).toHaveBeenCalledTimes(1) - expect(reconciliation.owes(SESSION)).toBe(true) - expect(turnState(current)).toBe('running') - - database.db.exec('DROP TRIGGER reject_recovered') // The second failure doubled the wait. await vi.advanceTimersByTimeAsync(1_990) await settleTurns() expect(attempts).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(10) await settleTurns() + expect(attempts).toHaveBeenCalledTimes(2) + + // A new signal is a fresh episode: a round at once, and the budget back at its first step. + reconciliation.signal(SESSION) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(3) + expect(turnState(current)).toBe('running') + database.db.exec('DROP TRIGGER reject_recovered') + await vi.advanceTimersByTimeAsync(990) + await settleTurns() + expect(attempts).toHaveBeenCalledTimes(3) + await vi.advanceTimersByTimeAsync(10) + await settleTurns() expect(reconciliation.owes(SESSION)).toBe(false) expect(turnState(current)).toBe('interrupted') } finally { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts index eb0eb957e981..ed84e30119c2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts @@ -49,6 +49,8 @@ export type StructuredAgentSessionLeftoverSettlementInput = { * proven-dead owner clears it, and a later release writes its own): only what that generation * and earlier ones left is settled, judged by it. Absent: the lease's own evidence judges. */ proof?: AgentSessionDeathEvidence & { ownerFence: number } + /** Bookkeeping no person waits on (`JournalWriteOptions`). */ + background?: true } export type StructuredAgentSessionLeftoverSettlement = @@ -87,6 +89,7 @@ export async function settleStructuredAgentSessionLeftovers( : `${STALE_SESSION_ROW_PREFIX}${sessionId}:${fence}:${proof ? `proof-${proof.ownerFence}:` : ''}seq-${journal.cursor().sequence}`, fence, recovered: true, + ...(input.background ? { background: true } : {}), plan: () => { const exited = exit ? planStructuredAgentSessionDeadGeneration( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 06ce9ef2d15b..1e3bed77f926 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -1,6 +1,7 @@ // The context every client mutation of a session runs with, and the one path each takes: admit the // envelope against the lease, then run its plan inside the session's serialize. +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' import type { AgentSessionMutationEnvelope, AgentSessionMutationResult @@ -39,8 +40,8 @@ export type StructuredAgentSessionMutationContext = { joinChildClose: (sessionId: string) => Promise /** A message was accepted: the session's delivery loop hands it over. */ wakeDelivery: (sessionId: string) => void - /** The chat's reconciliation worker still owes something (`StructuredAgentSessionReconciliation`). */ - reconciliationOwes: (sessionId: string) => boolean + /** The host's retry of background bookkeeping (`StructuredAgentSessionRetry`). */ + retry: Pick /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. * Each caller names why (`ending`). */ stopAgent: (sessionId: string, ending: StructuredAgentSessionStopEnding) => Promise diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts index 978d65d055bf..080a70dfb7ff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -85,12 +85,26 @@ export function endProviderChild( ended: Omit & { endedAt?: AgentJournalCursor } +): boolean { + if (!recordProviderChildEnd(session, ended)) { + return false + } + session.child = null + return true +} + +/** Records how the child on record ended, leaving it on record for its own teardown: every reader + * projects its generation as ended from now on (`StructuredAgentSessionCurrentWork`). */ +export function recordProviderChildEnd( + session: ChildBearer, + ended: Omit & { + endedAt?: AgentJournalCursor + } ): boolean { const child = matchingChild(session, ended) if (!child) { return false } - session.child = null session.lastEndedChild = { ...ended, ...(child.startedFor === undefined ? {} : { startedFor: child.startedFor }), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts new file mode 100644 index 000000000000..8aecff8b6b01 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts @@ -0,0 +1,36 @@ +// A queued card whose automatic send the retry gave up on (`StructuredAgentSessionRetry`): it reads +// as not sent at once, from memory, while its stored hold is tried once. Memory only: a restart's +// earlier-process settlement converts the card. + +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import type { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' + +export class QueuedSendAbandonment { + /** By chat, until a person moves the card, its hold lands or a later attempt sends it. */ + private readonly cards = new Map() + + card(sessionId: string): string | undefined { + return this.cards.get(sessionId) + } + + clear(sessionId: string): void { + this.cards.delete(sessionId) + } + + /** Marks the card, then tries its stored hold once; should that fail too, the mark still shows. */ + mark(sessionId: string, messageId: string, queued: Pick): void { + this.cards.set(sessionId, messageId) + void queued + .hold({ + messageIds: [messageId], + reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED, + background: true + }) + .then(() => { + if (this.cards.get(sessionId) === messageId) { + this.cards.delete(sessionId) + } + }) + .catch(() => undefined) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts deleted file mode 100644 index 78622226fa0d..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.test.ts +++ /dev/null @@ -1,242 +0,0 @@ -// The queue's automatic send when storage fails: another connection holding the database is -// retried with nothing shown, and only a refusal, or contention past its retries, holds the card -// with "Couldn't send". A worker still owing an ended generation's release keeps the drain from -// trying, and while the drain waits no client is told the card sends next (it would read Working). - -import { afterEach, describe, expect, it, vi } from 'vitest' -import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' -import { - EMPTY_STRUCTURED_AGENT_SESSION, - reduceStructuredAgentSession -} from '../../../shared/structured-agent-session-reducer' -import { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { - exitChild, - reconciled, - REJECT_RECOVERED_ROWS -} from './structured-agent-session-leftover-settlement.test-fixture' -import { - createQueuedMessageTestRig, - eventually, - type QueuedMessageTestRig -} from './structured-agent-session-queued-message-rig.test-fixture' -import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' -import { - isTransientStorageFailure, - StructuredAgentSessionQueuedDrainRetry -} from './structured-agent-session-queued-drain-retry' - -let rig: QueuedMessageTestRig | undefined - -afterEach(async () => { - vi.useRealTimers() - await rig?.dispose() - rig = undefined - vi.restoreAllMocks() -}) - -/** A card queued behind a working turn. */ -async function queuedBehindWork(): Promise<{ - current: QueuedMessageTestRig - draftId: string - working: string -}> { - rig = await createQueuedMessageTestRig({ restartable: true }) - const current = rig - const working = await current.workingSend() - const queued = await current.send('after this', 'queue-if-active').result - if (!queued.ok || !('queued' in queued.value)) { - throw new Error('expected a queued receipt') - } - return { current, draftId: queued.value.queued.messageId, working } -} - -async function pausedReason(current: QueuedMessageTestRig): Promise { - const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) - return page.ok ? page.page.queuedMessages?.[0]?.pausedReason : undefined -} - -const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) - -/** What a client reads after each frame: Working is the host's, or a card it is told sends next. */ -async function watchChat(current: QueuedMessageTestRig) { - let state = EMPTY_STRUCTURED_AGENT_SESSION - const reads: { working: boolean; turnRunning: boolean }[] = [] - await current.host.subscribe({ - id: 'chat', - sessionId: SESSION, - emit: (event) => { - state = reduceStructuredAgentSession(state, { type: 'event', event }) - reads.push({ - working: state.working === true || (state.nextQueuedMessageId ?? null) !== null, - turnRunning: state.latestTurn?.turn.state === 'running' - }) - } - }) - return reads -} - -const statusWorking = (current: QueuedMessageTestRig) => - current.host.readStatusSummary(SESSION)?.status === 'working' - -describe("the queue's automatic send", () => { - it("waits for an exited child's release, then sends on its own, never marked", async () => { - const { current, draftId } = await queuedBehindWork() - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const release = vi - .spyOn(current.store, 'transitionHandoff') - .mockRejectedValue(new Error('database is locked')) - const reads = await watchChat(current) - const before = reads.length - - await exitChild(current) - await new Promise((resolve) => setTimeout(resolve, 250)) - // The host reads nothing working, yet the drain does not try while the release is owed. - expect(await current.handoff(draftId)).toBeUndefined() - expect(await pausedReason(current)).toBeUndefined() - // The chat reads idle, as the status does: no card is named as sent next while it waits. - expect(reads.length).toBeGreaterThan(before) - expect(reads.at(-1)).toEqual({ working: false, turnRunning: false }) - expect(statusWorking(current)).toBe(false) - - release.mockRestore() - await reconciled(current) - await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) - }) - - it('retries a send that meets a locked database, and sends it with nothing shown', async () => { - const { current, draftId, working } = await queuedBehindWork() - const append = vi - .spyOn(AgentSessionJournal.prototype, 'appendSubmission') - .mockRejectedValueOnce(locked()) - .mockRejectedValueOnce(locked()) - const reads = await watchChat(current) - - await current.settleAccepted(working, 'a') - await vi.waitFor(() => expect(append).toHaveBeenCalledTimes(1)) - expect(await pausedReason(current)).toBeUndefined() - // Waiting out the backoff, the card is not named as sent next: the chat reads idle meanwhile. - await vi.waitFor(() => expect(reads.at(-1)?.working).toBe(false)) - expect(statusWorking(current)).toBe(false) - await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) - expect(append.mock.calls.length).toBeGreaterThanOrEqual(3) - }) - - it('holds the card with "Couldn\'t send" at once for a refusal', async () => { - const { current, working } = await queuedBehindWork() - vi.spyOn(AgentSessionJournal.prototype, 'appendSubmission').mockRejectedValueOnce( - new Error('disk full') - ) - - await current.settleAccepted(working, 'a') - await eventually(async () => - expect(await pausedReason(current)).toBe(QUEUED_MESSAGE_PAUSED_SEND_FAILED) - ) - }) -}) - -describe('a dead turn the worker settles while the release is still owed', () => { - it('publishes one view: no frame reads Working over it, and the card sends once released', async () => { - const { current, draftId } = await queuedBehindWork() - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const release = vi.spyOn(current.store, 'transitionHandoff').mockRejectedValue(locked()) - const { db } = openTestJournalHostDatabase(current.root) - db.exec(REJECT_RECOVERED_ROWS) - const reads = await watchChat(current) - const before = reads.length - - // The exit's own settlement and release both fail; the worker settles the turn a pass later. - await exitChild(current) - db.exec('DROP TRIGGER reject_recovered') - const settled = (await current.host.journalSnapshot(SESSION)).items.length - await eventually(async () => - expect((await current.host.journalSnapshot(SESSION)).items.length).toBeGreaterThan(settled) - ) - expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(true) - expect(reads.slice(before).filter((read) => read.working)).toEqual([]) - expect(statusWorking(current)).toBe(false) - - release.mockRestore() - await reconciled(current) - await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) - }) -}) - -describe('a send that keeps meeting a locked database', () => { - const journal = {} - - it('is given up after a bounded run, and only then held', () => { - vi.useFakeTimers() - const wake = vi.fn() - const retry = new StructuredAgentSessionQueuedDrainRetry(wake) - for (let attempt = 1; attempt < 10; attempt += 1) { - expect(retry.retryLater('chat', 'card', journal)).toBe(true) - expect(retry.waiting('chat', 'card', journal)).toBe(true) - vi.advanceTimersByTime(30_000) - expect(retry.waiting('chat', 'card', journal)).toBe(false) - } - expect(wake).toHaveBeenCalledTimes(9) - // The tenth failure falls back to the held card. - expect(retry.retryLater('chat', 'card', journal)).toBe(false) - }) - - it('starts over for another card, a reopened chat, or a card a person moved on', () => { - vi.useFakeTimers() - const retry = new StructuredAgentSessionQueuedDrainRetry(() => undefined) - const exhaust = (messageId: string, on: object) => { - for (let attempt = 1; attempt < 10; attempt += 1) { - retry.retryLater('chat', messageId, on) - } - } - exhaust('card', journal) - expect(retry.retryLater('chat', 'other', journal)).toBe(true) - exhaust('card', journal) - expect(retry.retryLater('chat', 'card', {})).toBe(true) - exhaust('card', journal) - retry.settled('chat') - expect(retry.retryLater('chat', 'card', journal)).toBe(true) - }) - - it('keeps one timer per chat: a new failure replaces the wait before it', () => { - vi.useFakeTimers() - const wake = vi.fn() - const retry = new StructuredAgentSessionQueuedDrainRetry(wake) - retry.retryLater('chat', 'card', journal) - retry.retryLater('chat', 'card', journal) - vi.advanceTimersByTime(60_000) - expect(wake).toHaveBeenCalledTimes(1) - }) - - it('tells contention from other failures', () => { - expect(isTransientStorageFailure(locked())).toBe(true) - expect(isTransientStorageFailure(new Error('wrapped', { cause: locked() }))).toBe(true) - expect(isTransientStorageFailure(new Error('disk full'))).toBe(false) - }) -}) - -describe('an exited child whose release the worker gave up on', () => { - it('no longer holds the card: once the worker retires, the queue sends it on its own', async () => { - const { current, draftId } = await queuedBehindWork() - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - const release = vi - .spyOn(current.store, 'transitionHandoff') - .mockRejectedValue(new Error('database is locked')) - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) - await exitChild(current) - const { reconciliation } = current.host.collaboratorsForTests() - for (let step = 0; step < 12 && reconciliation.owes(SESSION); step += 1) { - await vi.advanceTimersByTimeAsync(31_000) - } - expect(reconciliation.owes(SESSION)).toBe(false) - release.mockRestore() - vi.useRealTimers() - - // Sent, or held with main's marker: never left waiting with neither. - await eventually(async () => - expect( - Boolean(await current.handoff(draftId)) || (await pausedReason(current)) !== undefined - ).toBe(true) - ) - }) -}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts deleted file mode 100644 index 314ee039f897..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-drain-retry.ts +++ /dev/null @@ -1,80 +0,0 @@ -// The queue's automatic send meeting storage another connection holds (SQLITE_BUSY or locked): not -// the person's failure, so nothing is shown and the card is not held. The same send is retried -// after a backoff, in memory only, until it lands, the card moves on (a Send now, Delete or Stop) -// or the process ends; after a bounded run it falls back to the held card, so a person can act and -// nothing is owed forever. Any other failure holds the card at once, as before. - -import { isTransientSqliteContention } from '../../sqlite/sqlite-read-failure' -import { - RECONCILIATION_MAX_FAILED_ATTEMPTS, - reconciliationBackoffDelay -} from './structured-agent-session-reconciliation-backoff' - -/** Whether a failure, or one it wraps, is SQLite contention (`isTransientSqliteContention`). */ -export function isTransientStorageFailure(error: unknown): boolean { - let current: unknown = error - for (let depth = 0; current !== undefined && depth < 5; depth += 1) { - if (isTransientSqliteContention(current)) { - return true - } - current = current instanceof Error ? current.cause : undefined - } - return false -} - -type PendingRetry = { - messageId: string - /** The conversation the failures were met on: a reopened chat's card starts fresh. */ - journal: object - failures: number - timer: ReturnType | null -} - -export class StructuredAgentSessionQueuedDrainRetry { - private readonly pending = new Map() - - /** `wake`: schedules the chat's drain, whose step re-derives everything. */ - constructor(private readonly wake: (sessionId: string) => void) {} - - /** Whether this card's send waits out a backoff: a step woken meanwhile leaves it to the timer. */ - waiting(sessionId: string, messageId: string, journal: object): boolean { - const entry = this.pending.get(sessionId) - return entry?.messageId === messageId && entry.journal === journal && entry.timer !== null - } - - /** A transient failure of this card's send: true while it is retried later, false once the - * run is given up (the caller then holds the card). */ - retryLater(sessionId: string, messageId: string, journal: object): boolean { - const previous = this.pending.get(sessionId) - const failures = - previous?.messageId === messageId && previous.journal === journal ? previous.failures + 1 : 1 - this.settled(sessionId) - if (failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { - return false - } - const entry: PendingRetry = { messageId, journal, failures, timer: null } - entry.timer = setTimeout(() => { - entry.timer = null - this.wake(sessionId) - }, reconciliationBackoffDelay(failures)) - // A backoff alone never keeps the process alive. - entry.timer.unref?.() - this.pending.set(sessionId, entry) - return true - } - - /** The send landed, or the card moved on (sent, a person's Send now or Delete): its run ends. */ - settled(sessionId: string): void { - const entry = this.pending.get(sessionId) - if (entry?.timer) { - clearTimeout(entry.timer) - } - this.pending.delete(sessionId) - } - - dispose(): void { - for (const sessionId of this.pending.keys()) { - this.settled(sessionId) - } - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 548445c8f426..8a0719479e85 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -28,10 +28,9 @@ import type { AgentSessionTurnContext } from './structured-agent-session-turns' import { QueuedMessageNotConsumableError } from '../agent-session-journal/journal-queued-messages' import type { QueuedMessageRow } from '../agent-session-journal/queued-message-table' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import { - isTransientStorageFailure, - StructuredAgentSessionQueuedDrainRetry -} from './structured-agent-session-queued-drain-retry' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' +import { QueuedSendAbandonment } from './structured-agent-session-queued-abandonment' import { structuredAgentSessionHostInstance, structuredQueuePauses @@ -240,11 +239,9 @@ export type QueuedMessageDrainDeps = { currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void - /** An ended child still holds the lease and the chat's worker still owes its release: its retire - * wakes the drain, whether the release landed or was given up. A settlement still owed with the - * lease released holds nothing: bookkeeping never gates the queue. */ - releaseOwed: (sessionId: string) => boolean - /** Re-sends what clients read (`AgentSessionSubscribers.publish`): a send put off writes nothing. */ + /** The host's retry (`StructuredAgentSessionRetry`), read lazily: it is built after the drain. */ + retry: () => Pick + /** Re-sends what clients read (`AgentSessionSubscribers.publish`): an abandon writes nothing. */ publish: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger } @@ -260,7 +257,7 @@ export type QueuedMessageDrainDeps = { export class StructuredAgentSessionQueuedMessageDrain { private readonly scheduled = new Set() private disposed = false - private readonly retry = new StructuredAgentSessionQueuedDrainRetry((id) => this.schedule(id)) + private readonly abandoned = new QueuedSendAbandonment() constructor(private readonly deps: QueuedMessageDrainDeps) {} @@ -269,24 +266,36 @@ export class StructuredAgentSessionQueuedMessageDrain { * right before it appends, since quit can land while it awaits. */ dispose(): void { this.disposed = true - this.retry.dispose() } - /** Whether the send of this card waits: an ended generation's release is still owed - * (`releaseOwed`), or a transient failure's backoff runs. A person's Send now waits on neither. */ - waits = (sessionId: string, messageId: string, journal: object): boolean => - this.deps.releaseOwed(sessionId) || this.retry.waiting(sessionId, messageId, journal) + /** Whether this card's automatic send waits for the retry's next round, or was given up on. A + * person's Send now waits on neither. */ + waits = (sessionId: string, messageId: string): boolean => + this.deps.retry().sendWaits(sessionId) || this.abandoned.card(sessionId) === messageId + + /** The card a given-up send left, which the queue shows as not sent. */ + abandonedCard = (sessionId: string): string | undefined => this.abandoned.card(sessionId) - /** A person moved the card on (Send now, Delete): a send being retried starts fresh. */ + /** A person moved the card on (Send now, Delete). */ cardMoved(sessionId: string): void { - this.retry.settled(sessionId) + this.abandoned.clear(sessionId) + } + + /** The retry gave up on the chat's automatic send: its card reads as not sent at once, and the + * stored hold is tried once; should that fail too, this mark still shows it. */ + abandon(sessionId: string): void { + const journal = this.deps.sessions.get(sessionId)?.journal + const next = journal ? oldestActionableQueuedMessage(journal) : null + if (!journal || !next) { + return + } + this.abandoned.mark(sessionId, next.messageId, journal.queuedMessages) + this.deps.publish(sessionId, journal) } schedule(sessionId: string): void { const journal = this.disposed ? undefined : this.deps.sessions.get(sessionId)?.journal if (!journal) { - // Closed: a backoff's wake finds no chat, and its entry goes (a reopen starts fresh). - this.retry.settled(sessionId) return } // Cheap pre-check so token streams do not pay a serialized step per delta. @@ -311,22 +320,38 @@ export class StructuredAgentSessionQueuedMessageDrain { void this.deps .serialize(sessionId, () => { this.scheduled.delete(sessionId) - return this.step(sessionId) + return this.step(sessionId, false) }) .catch((error: unknown) => { this.scheduled.delete(sessionId) - this.deps.logger.warn('draining queued messages failed', { - scope: 'queued-drain', - sessionId, - error - }) + this.warn(sessionId, error) + }) + } + + /** The retry's own send of the chat's next card (`StructuredAgentSessionRetry`): whether another + * connection's lock refused it, which ends the retry's round. */ + sendForRetry(sessionId: string): Promise<'contended' | 'done'> { + return this.deps + .serialize(sessionId, () => this.step(sessionId, true)) + .catch((error: unknown) => { + this.warn(sessionId, error) + return 'done' as const }) } - private async step(sessionId: string): Promise { + private warn(sessionId: string, error: unknown): void { + this.deps.logger.warn('draining queued messages failed', { + scope: 'queued-drain', + sessionId, + error + }) + } + + /** `forRetry`: the retry's own attempt, which a send it owes does not hold back. */ + private async step(sessionId: string, forRetry: boolean): Promise<'contended' | 'done'> { const session = this.deps.sessions.get(sessionId) if (this.disposed || !session) { - return + return 'done' } const journal = session.journal if (journal.queuedMessages.settlementOwed() || journal.queuedMessages.deliveredByEchoOwed()) { @@ -344,12 +369,8 @@ export class StructuredAgentSessionQueuedMessageDrain { const record = this.deps.getRecord(sessionId) const work = this.deps.currentWork(sessionId) const next = work ? nextStructuredQueuedMessage({ journal, record, work }) : null - if (this.disposed || !next) { - this.retry.settled(sessionId) - return - } - if (this.waits(sessionId, next.messageId, journal)) { - return + if (this.disposed || !next || (!forRetry && this.deps.retry().sendWaits(sessionId))) { + return 'done' } // Always a fresh id: the submission names its draft by `queuedMessageId`, never by id equality. const submissionId = createStructuredAgentSessionOperationId(randomUUID) @@ -369,33 +390,32 @@ export class StructuredAgentSessionQueuedMessageDrain { expect: 'waiting', settledByOp: null, hostInstance: structuredAgentSessionHostInstance(), - yieldsToPause: true + yieldsToPause: true, + background: true } ) } catch (error) { if (error instanceof QueuedMessageNotConsumableError) { // Lost a race with a Send-now, a Delete or a Stop; their transition stands. - this.retry.settled(sessionId) - return + return 'done' } - // Storage another connection holds: retried after a backoff, with nothing shown. - if ( - isTransientStorageFailure(error) && - this.retry.retryLater(sessionId, next.messageId, journal) - ) { - // Clients were told this card sends next; it now waits, so the chat reads idle meanwhile. - this.deps.publish(sessionId, journal) - return + // Another connection holds the database: the retry sends it in its next round, nothing shown. + if (isSqliteContentionFailure(error)) { + if (!forRetry) { + this.deps.retry().sendContended(sessionId) + } + return 'contended' } - // A refusal, or contention past its retries: the draft stays waiting, held with the marker - // on the card (a stored fact, so it survives eviction and restart). The hold's own commit - // notification publishes it. An explicit Send retries. + // A refusal: the draft stays waiting, held with the marker on the card (a stored fact, so it + // survives eviction and restart). The hold's own commit notification publishes it. An + // explicit Send retries. await journal.queuedMessages .hold({ messageIds: [next.messageId], reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED }) .catch(() => {}) throw error } - this.retry.settled(sessionId) + this.abandoned.clear(sessionId) this.deps.wakeDelivery(sessionId) + return 'done' } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index e0b1365d7a30..be47cfb2a798 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -30,41 +30,53 @@ export type QueuePublication = { nextQueuedMessageId: string | null } -/** What the drain's gate reads beyond the journal; resolved per read. */ -export type QueueSendGate = (journal: AgentSessionJournal) => StructuredQueueGateInput +/** What the drain's gate reads beyond the journal; resolved per read. `abandoned`: the card whose + * automatic send was given up on, shown as not sent. */ +export type QueueSendGate = ( + journal: AgentSessionJournal +) => StructuredQueueGateInput & { abandoned?: string } -/** `StructuredAgentSessionQueuedMessageDrain.waits`. */ -export type QueuedDrainWaits = (sessionId: string, messageId: string, journal: object) => boolean +/** What the drain itself holds back (`StructuredAgentSessionQueuedMessageDrain`). */ +export type QueuedDrainHolds = { + waits: (sessionId: string, messageId: string) => boolean + abandonedCard: (sessionId: string) => string | undefined +} -/** `drainWaits`: the drain's own waits, which a publication reads so a client is never told a card +/** `drain`: the drain's own holds, which a publication reads so a client is never told a card * sends next while the drain holds it back (the chat would read Working with nothing running). */ export function structuredQueueSendGate( host: StructuredAgentSessionCurrentWorkHost, sessionId: string, - drainWaits?: QueuedDrainWaits + drain?: QueuedDrainHolds ): QueueSendGate { return (journal) => { const record = host.store.getRecord(sessionId) const ended = host.sessions.get(sessionId)?.lastEndedChild + const abandoned = drain?.abandonedCard(sessionId) return { record, work: structuredAgentSessionCurrentWork(journal, { record, ...(ended ? { ended } : {}) }), - ...(drainWaits - ? { drainWaits: (messageId: string) => drainWaits(sessionId, messageId, journal) } - : {}) + ...(drain ? { drainWaits: (messageId: string) => drain.waits(sessionId, messageId) } : {}), + ...(abandoned ? { abandoned } : {}) } } } /** Waiting and returned rows only. `paused` is a per-card hold (a failed conversion); a person's * Stop pauses the queue, published once beside it. */ -function computePublishedQueuedMessages(journal: AgentSessionJournal): AgentSessionQueuedMessage[] { +function computePublishedQueuedMessages( + journal: AgentSessionJournal, + abandoned: string | undefined +): AgentSessionQueuedMessage[] { const published: AgentSessionQueuedMessage[] = [] for (const row of journal.queuedMessages.list()) { if (row.state !== 'waiting' && row.state !== 'returned') { continue } - const held = row.state === 'waiting' && row.holdReason !== null + // A send given up on reads as main's failed send even when its stored hold did not land. + const holdReason = + row.messageId === abandoned ? QUEUED_MESSAGE_PAUSED_SEND_FAILED : row.holdReason + const held = row.state === 'waiting' && holdReason !== null published.push({ messageId: row.messageId, position: row.position, @@ -72,7 +84,7 @@ function computePublishedQueuedMessages(journal: AgentSessionJournal): AgentSess state: row.state, ...(held ? { paused: true as const } : {}), // The stored reason is a typed marker; an unknown one reads as a plain hold. - ...(held && isPublishedPausedReason(row.holdReason) ? { pausedReason: row.holdReason } : {}), + ...(held && isPublishedPausedReason(holdReason) ? { pausedReason: holdReason } : {}), ...(row.state === 'returned' ? { returnedReason: row.returnedReason } : {}), ...(row.state === 'returned' && row.returnedRejection ? { returnedRejection: row.returnedRejection } @@ -96,13 +108,16 @@ type ListMemo = { key: string; serialized: string; list: AgentSessionQueuedMessa const listMemos = new WeakMap() const publications = new WeakMap() -function readPublishedQueuedMessages(journal: AgentSessionJournal): AgentSessionQueuedMessage[] { - const key = String(journal.queuedMessages.revision()) +function readPublishedQueuedMessages( + journal: AgentSessionJournal, + abandoned: string | undefined +): AgentSessionQueuedMessage[] { + const key = `${journal.queuedMessages.revision()}:${abandoned ?? ''}` const memo = listMemos.get(journal) if (memo && memo.key === key) { return memo.list } - const list = computePublishedQueuedMessages(journal) + const list = computePublishedQueuedMessages(journal, abandoned) // Belt for the identity dedup: equal recomputed content keeps the previous reference. const serialized = JSON.stringify(list) if (memo && memo.serialized === serialized) { @@ -126,7 +141,8 @@ export function readQueuePublication( journal: AgentSessionJournal, gate: QueueSendGate ): QueuePublication { - const queuedMessages = readPublishedQueuedMessages(journal) + const input = gate(journal) + const queuedMessages = readPublishedQueuedMessages(journal, input.abandoned) // Read per emit: the pause also turns on submissions (a turn starting). Only a person's Stop is // shown, and only over a card Resume would send, so its header never offers to send nothing; // deleting a blocking returned card shows it again. After a /clear, a restart or a close nothing @@ -150,8 +166,7 @@ export function readQueuePublication( resumable && !queuePauseLiftOnItsWay(resumable, journal.submissions()) ? resumable : null // The test narrows the type. const queuePause = pause?.reason === 'stopped' ? { reason: pause.reason } : null - const nextQueuedMessageId = - nextStructuredQueuedMessage({ journal, ...gate(journal) })?.messageId ?? null + const nextQueuedMessageId = nextStructuredQueuedMessage({ journal, ...input })?.messageId ?? null const previous = publications.get(journal) if ( previous && diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts index 5622527ca4b9..dc58fb3e783d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-wiring.ts @@ -16,7 +16,6 @@ import { sendQueuedStructuredAgentMessage } from './structured-agent-session-queued-mutations' import { deferredStructuredAgentSessionLogger } from './structured-agent-session-logger' -import { structuredAgentSessionEndedChildHoldsLease } from './structured-agent-session-child-close' /** `sessions` are the live conversations (their `touch` is the idle sweep's activity renewal, * which the drain's schedule rides); everything else comes from the host's mutation context, @@ -36,9 +35,7 @@ export function wireStructuredAgentSessionQueuedMessages( currentWork: (sessionId) => hostStructuredAgentSessionCurrentWork({ store: context().deps.store, sessions }, sessionId), wakeDelivery: (sessionId) => context().wakeDelivery(sessionId), - releaseOwed: (sessionId) => - structuredAgentSessionEndedChildHoldsLease({ deps: context().deps, sessions }, sessionId) && - context().reconciliationOwes(sessionId), + retry: () => context().retry, publish: (sessionId, journal) => context().publish(sessionId, journal), // Read lazily, like the rest of this wiring: the host's deps are not assigned yet. logger: deferredStructuredAgentSessionLogger(() => context().deps.logger) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts deleted file mode 100644 index 9501aac5f1cb..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-memory.ts +++ /dev/null @@ -1,77 +0,0 @@ -// What the reconciliation keeps of a chat between its workers, in memory only: nothing here is -// owed past the process, and all of it goes with the chat's record. - -import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionReconciliationDebts } from './structured-agent-session-reconciliation-pass' -import { - RECONCILIATION_MAX_FAILED_ATTEMPTS, - reconciliationBackoffDelay -} from './structured-agent-session-reconciliation-backoff' - -export class StructuredAgentSessionReconciliationMemory { - /** Proofs a retired worker still held (a lease in recovery, or a run given up), for the next - * one; dropped once taken. An exit's account is not kept: it judges only its own generation, - * which a later one may have replaced by then. */ - private readonly parked = new Map() - /** Where this host process first opened each chat's journal: every send at or before it was - * accepted by an earlier host process, every one after by this. Kept for the whole process (a - * close or a retire must not reset it, or this process's sends would read as an earlier one's). */ - private readonly firstOpened = new Map() - /** Startup's lease reconcile is store-wide, so every chat's worker retries it as one: one call - * at a time, and after a failure none before the backoff's next step, so N workers never stack - * N waits on a busy store. */ - private readonly restartReconcile: { - running: Promise | null - failures: number - notBefore: number - } = { running: null, failures: 0, notBefore: 0 } - - park(sessionId: string, debts: StructuredAgentSessionReconciliationDebts): void { - if (debts.evidence) { - this.parked.set(sessionId, { evidence: debts.evidence }) - } - } - - take(sessionId: string): StructuredAgentSessionReconciliationDebts { - const debts = this.parked.get(sessionId) ?? {} - this.parked.delete(sessionId) - return debts - } - - noteOpened(sessionId: string, journal: Pick): void { - if (!this.firstOpened.has(sessionId)) { - this.firstOpened.set(sessionId, journal.openedAt()) - } - } - - processOpened(sessionId: string): AgentJournalCursor | undefined { - return this.firstOpened.get(sessionId) - } - - /** Whether the store-wide reconcile settled every lease: joined when one runs, false while its - * backoff waits or once it gave up (an attach or a send then reconciles before it acts). */ - reconcileOnce(reconcile: () => Promise): Promise { - const shared = this.restartReconcile - if (shared.running) { - return shared.running - } - // The wall clock, as the workers' own backoff timers run on it. - if (Date.now() < shared.notBefore || shared.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { - return Promise.resolve(false) - } - shared.running = reconcile().then((settled) => { - shared.running = null - shared.failures = settled ? 0 : shared.failures + 1 - shared.notBefore = settled ? 0 : Date.now() + reconciliationBackoffDelay(shared.failures) - return settled - }) - return shared.running - } - - /** The chat's record is gone. */ - forget(sessionId: string): void { - this.parked.delete(sessionId) - this.firstOpened.delete(sessionId) - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts index afdbae3531a3..e208e2658dcf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts @@ -1,4 +1,4 @@ -// What a retired worker keeps for the chat's next one: the proofs it still held, never an exit's +// What the retry keeps for a chat whose lease recovers: the proofs it still holds, never an exit's // account (it judges only its own generation, which a later one may have replaced), and nothing at // all once the chat's record is gone. @@ -105,7 +105,7 @@ function attach(current: StructuredAgentSessionHost) { return current.attach({ callerKey: 'client-1' }, params).catch(() => undefined) } -describe("a worker that retires while the chat's lease recovers", () => { +describe('a chat whose lease recovers', () => { it("parks generation 13's proof, never its exit account, which would judge generation 14's sends", async () => { const current = await recoveringAt14() const { reconciliation } = current.collaboratorsForTests() @@ -113,7 +113,11 @@ describe("a worker that retires while the chat's lease recovers", () => { // Generation 13's exit could not be settled; the lease recovers, so the worker retires. reconciliation.signal(CHAT, { evidence: proof13, exit: exit13 }) await idle(current, [CHAT]) - expect(reconciliation['memory']['parked'].get(CHAT)).toEqual({ evidence: [proof13] }) + expect(reconciliation['owed'].get(CHAT)).toMatchObject({ + due: false, + debts: { evidence: [proof13] } + }) + expect(reconciliation['owed'].get(CHAT)?.debts.exit).toBeUndefined() // Generation 14 ends with no exit account of its own: an attach decides its recovery. await attach(current) @@ -121,7 +125,7 @@ describe("a worker that retires while the chat's lease recovers", () => { // The proof judged what generation 13 left, and was taken. expect(await turnState(current, CHAT)).toBe('interrupted') - expect(reconciliation['memory']['parked'].has(CHAT)).toBe(false) + expect(reconciliation['owed'].has(CHAT)).toBe(false) // Generation 14's send ran on a child that proved its start: in doubt, never "did not start". const { submissions } = await current.journalSnapshot(CHAT) expect(submissions.find((entry) => entry.clientMessageId === `${CHAT}-handed`)).toMatchObject({ @@ -130,7 +134,7 @@ describe("a worker that retires while the chat's lease recovers", () => { }) }) -describe('a live worker waiting out a backoff with an exit account', () => { +describe('a chat waiting out a backoff with an exit account', () => { it("drops it once a later generation acquired, so generation 14's send stays in doubt", async () => { const current = await recoveringAt14() const { reconciliation } = current.collaboratorsForTests() @@ -160,15 +164,15 @@ describe('a chat whose record is gone', () => { reconciliation.signal(CHAT, { evidence: proof13 }) await idle(current, [CHAT]) expect(await turnState(current, CHAT)).toBe('running') - expect(reconciliation['memory']['parked'].has(CHAT)).toBe(true) - expect(reconciliation['memory']['firstOpened'].has(CHAT)).toBe(true) + expect(reconciliation['owed'].get(CHAT)?.debts.evidence).toEqual([proof13]) + expect(reconciliation['firstOpened'].has(CHAT)).toBe(true) const getRecord = store.getRecord.bind(store) vi.spyOn(store, 'getRecord').mockImplementation((id) => (id === CHAT ? null : getRecord(id))) reconciliation.signal(CHAT) await idle(current, [CHAT]) - expect(reconciliation['memory']['parked'].has(CHAT)).toBe(false) - expect(reconciliation['memory']['firstOpened'].has(CHAT)).toBe(false) + expect(reconciliation['owed'].has(CHAT)).toBe(false) + expect(reconciliation['firstOpened'].has(CHAT)).toBe(false) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index 2ab872d70006..79639783edf7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -15,6 +15,7 @@ // An empty item plan says nothing of (a) or (b). import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { queuedMessageReopenMarkStart } from '../agent-session-journal/queued-message-reopen-floor' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-record' @@ -51,12 +52,17 @@ export type StructuredAgentSessionReconciliationDebts = { } export type StructuredAgentSessionReconciliationPass = { - /** A step could not finish: the worker retries after its backoff. */ + /** A step could not finish: the retry visits again after its backoff. */ failed: unknown[] /** Rows were written, so the next pass verifies nothing is left. */ wrote: boolean + /** The lease is latched in recovery: what ended waits for its decision's signal. */ + recovering: boolean } +/** Every write a pass makes is bookkeeping no person waits on (`JournalWriteOptions`). */ +const BACKGROUND = { background: true } as const + /** `session`: the chat's open conversation, or the worker's own read of a closed one. * `processOpened`: where this host first opened the chat's journal; a send at or before it was an * earlier process's. A later handle's own open cannot tell: this process may have accepted sends @@ -68,9 +74,9 @@ export async function runStructuredAgentSessionReconciliationPass( debts: StructuredAgentSessionReconciliationDebts, processOpened: AgentJournalCursor ): Promise { - const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false } const lease = context.deps.store.getRecord(sessionId)?.lease const recovering = lease?.handoffStage === 'recovering' + const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false, recovering } // An exit's account judges only its own generation: once a later one acquired, it is spent. if (debts.exit && (!lease || lease.runtimeFence > debts.exit.ownerFence + 1)) { delete debts.exit @@ -78,12 +84,12 @@ export async function runStructuredAgentSessionReconciliationPass( if ( !recovering && structuredAgentSessionEndedChildHoldsLease(context, sessionId) && - (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId)) + (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId, BACKGROUND)) ) { pass.failed.push(new Error('agent_session_exit_release_owed')) } await settleEarlierProcess(context, sessionId, session.journal, processOpened, pass) - if (recovering) { + if (recovering || pass.failed.some(isSqliteContentionFailure)) { // What ended is not known until its recovery is decided; the debts wait for that signal. return pass } @@ -94,7 +100,8 @@ export async function runStructuredAgentSessionReconciliationPass( journal: session.journal, ...(debts.exit ? { exit: debts.exit } : {}), ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), - ...(proof ? { proof } : {}) + ...(proof ? { proof } : {}), + background: true }) // Each proof the lease no longer holds judges what its own generation left, oldest first. const leaseEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null @@ -161,17 +168,24 @@ async function settleEarlierProcess( return false } ) - await attempt(() => journal.queuedMessages.repairAndPrune()) + // Another connection's lock ends the pass: the round ends, and every step is tried again. + if (!(await attempt(() => journal.queuedMessages.repairAndPrune(BACKGROUND)))) { + return + } // Before a Stop can withdraw one: a Stop never withdraws a card. let converted = false - await attempt(async () => { + const held = await attempt(async () => { converted = (await holdUnsentSends(journal, { fence, hostInstance: structuredAgentSessionHostInstance(), - hold: { cause: 'hostRestarted', which: (entry) => earlier(entry.acceptedSequence) } + hold: { cause: 'hostRestarted', which: (entry) => earlier(entry.acceptedSequence) }, + background: true })) !== null }) + if (!held && pass.failed.some(isSqliteContentionFailure)) { + return + } // After every card the earlier process left, so a card this one queued is never held by it. // Reported, never owed: the pause starts there anyway, and a failed mark leaves the floor set, // so the next pass marks again. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts similarity index 98% rename from src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts rename to src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts index f843e38ee4fa..a2749db3a012 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts @@ -1,5 +1,5 @@ -// The chat's reconciliation worker: what retires it, what keeps it, and what readers see before it -// has written anything. +// The retry of a chat's background bookkeeping: what settles it, what keeps it owed, and what +// readers see before it has written anything. import { afterEach, describe, expect, it, vi } from 'vitest' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts new file mode 100644 index 000000000000..672a0debfb29 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -0,0 +1,311 @@ +// The host's one retry of background bookkeeping, scoped to its database connection: what ended +// generations left, what an earlier host process left, and the queue's automatic sends that met +// another connection's lock. One set of owed chats, one round at a time, one timer and one budget. +// +// Signals: every committed revocation or supersession of a generation (the record store's +// `onGenerationEnded`), an exit this host observed (whether or not its release landed), a row an +// ended generation committed late, a queue send that met contention, and startup. Each one +// re-derives what readers see at once (the operational revision, the published view, the queued- +// card drain) and marks the chat owed. A round visits every owed chat, in the few background slots +// with a reader's open chat first: (a) the store-wide restart reconcile, once, while any lease is +// unreconciled; then per chat, in its lane, (b) release repair, (c) settlement and what an earlier +// process left (`runStructuredAgentSessionReconciliationPass`), (d) the queue's next send. Every +// write is background (`JournalWriteOptions`): another connection's lock fails it at once and ends +// the round, since contention is connection-wide. +// +// A failed round backs off from 1 s to 30 s; ten in a row end the episode: the timer stops, the set +// is kept, and a queue send still owed shows as not sent (`abandonSend`). A new signal, a reader's +// open, or a commit on the connection after contention starts a fresh episode. A lease latched in +// recovery is never decided here: its proofs wait in the set for its decision's signal. + +import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' +import { + RECONCILIATION_MAX_FAILED_ATTEMPTS, + reconciliationBackoffDelay +} from './structured-agent-session-reconciliation-backoff' +import { StructuredAgentSessionReconciliationSlots } from './structured-agent-session-reconciliation-slots' +import { + dropStructuredAgentSessionLoaded, + noteStructuredAgentSessionOpened, + visitStructuredAgentSessionOwedChat, + type StructuredAgentSessionOwedChat as OwedChat, + type StructuredAgentSessionRetryContext, + type StructuredAgentSessionVisit as Visit, + type StructuredAgentSessionVisitHost +} from './structured-agent-session-reconciliation-visit' + +export type { StructuredAgentSessionRetryContext } from './structured-agent-session-reconciliation-visit' + +export type StructuredAgentSessionReconciliationSignal = { + /** Proof the lease no longer holds (`AgentSessionGenerationEnd.evidence`). */ + evidence?: AgentSessionGenerationEnd['evidence'] + /** An observed exit whose own settlement did not land. */ + exit?: StructuredAgentSessionExitSettlement + /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ + restate?: boolean +} + +const BACKGROUND = { background: true } as const + +export class StructuredAgentSessionRetry { + private readonly owed = new Map() + /** Where this process first opened each chat's journal: every send at or before it was accepted + * by an earlier process. Kept for the whole process; dropped only with the chat's record. */ + private readonly firstOpened = new Map() + private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) + private readonly unsubscribe: () => void + private timer: ReturnType | 'soon' | null = null + private running = false + /** A chat became due while a round ran: another follows at once. */ + private again = false + /** Consecutive failed rounds in this episode; at the budget the episode ends. */ + private failures = 0 + /** The last round failed on another connection's lock, which a commit here proves gone. */ + private contended = false + private disposed = false + private readonly visitHost: StructuredAgentSessionVisitHost + + constructor(private readonly context: StructuredAgentSessionRetryContext) { + this.visitHost = { + context, + disposed: () => this.disposed, + firstOpened: this.firstOpened, + settle: (sessionId, chat) => this.settle(sessionId, chat), + warn: (sessionId, error) => this.warn(sessionId, error) + } + this.unsubscribe = context.deps.store.onGenerationEnded((ended) => + this.signal(ended.sessionId, { evidence: ended.evidence }) + ) + } + + signal = (sessionId: string, signal: StructuredAgentSessionReconciliationSignal = {}): void => { + const session = this.context.sessions.get(sessionId) + if (session) { + session.operationalRevision = (session.operationalRevision ?? 0) + 1 + } + const chat = this.disposed ? null : this.owe(sessionId) + if (chat && signal.evidence) { + chat.debts.evidence = [...(chat.debts.evidence ?? []), signal.evidence] + } + if (chat && signal.exit) { + chat.debts.exit = signal.exit + } + this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) + this.freshEpisode() + } + + /** A committed row: one below the lease's fence is an ended generation's late write, and any + * commit proves the connection free again after contention. */ + observeCommit = (sessionId: string, lowestFence: number | null): void => { + const fence = this.context.deps.store.getRecord(sessionId)?.lease.runtimeFence + if (lowestFence !== null && fence !== undefined && lowestFence < fence) { + this.signal(sessionId) + } else if (this.contended) { + this.freshEpisode() + } + } + + /** A reader opened the chat: its handle marks the process boundary if it is the first, and what + * it owes goes ahead of the scan, in a fresh episode. */ + opened = (sessionId: string, journal: Pick): void => { + noteStructuredAgentSessionOpened(this.firstOpened, sessionId, journal) + const chat = this.owed.get(sessionId) + if (chat) { + this.slots.prioritize(chat) + this.freshEpisode() + } + } + + /** The queue's automatic send met another connection's lock: the next round sends it again. */ + sendContended = (sessionId: string): void => { + if (!this.disposed) { + this.owe(sessionId).send = true + this.context.publishGenerationEnded(sessionId) + this.roundEnded(true, true) + } + } + + /** Whether the chat's queued send waits for a round (or for a fresh episode, given up). */ + sendWaits = (sessionId: string): boolean => this.owed.get(sessionId)?.send === true + + /** Resolves once the chat's next visit finished, or at once when it owes nothing. */ + attempted = (sessionId: string): Promise => { + const chat = this.owed.get(sessionId) + return chat?.due ? new Promise((resolve) => chat.attempted.push(resolve)) : Promise.resolve() + } + + /** Whether a visit is owed: something was signalled, or failed, and has not settled. */ + owes = (sessionId: string): boolean => this.owed.get(sessionId)?.due === true + + /** Resolves once the chat owes nothing, or waits out a failed round's backoff. */ + idle = async (sessionId: string): Promise => { + while (this.owes(sessionId) && (this.running || this.timer === 'soon' || this.failures === 0)) { + await this.attempted(sessionId) + } + } + + /** Shutdown: the timer stops and every slot wait ends; a visit already writing finishes, + * tracked, and one that has not started work starts none. */ + dispose = (): void => { + this.disposed = true + this.unsubscribe() + this.stopTimer() + for (const [sessionId, chat] of this.owed) { + chat.slotWait?.abort() + this.settle(sessionId, chat) + } + } + + processOpened(sessionId: string): AgentJournalCursor | undefined { + return this.firstOpened.get(sessionId) + } + + private owe(sessionId: string): OwedChat { + let chat = this.owed.get(sessionId) + if (!chat) { + chat = { debts: {}, due: true, dirty: false, attempted: [], urgent: false } + this.owed.set(sessionId, chat) + } + chat.due = true + chat.dirty = true + this.again ||= this.running + return chat + } + + /** A new reason to try: the budget resets and a round runs now. */ + private freshEpisode(): void { + if (this.disposed) { + return + } + this.failures = 0 + this.contended = false + if (!this.running && this.timer !== 'soon') { + this.stopTimer() + this.timer = 'soon' + queueMicrotask(() => void this.round()) + } + } + + private async round(): Promise { + this.timer = null + if (this.disposed || this.running) { + return + } + this.running = true + this.again = false + let failed = false + let contended = false + const end = (visit: Visit) => { + failed ||= visit === 'failed' || visit === 'contended' + contended ||= visit === 'contended' + this.again ||= visit === 'again' + } + if (this.context.deps.store.listRecords().some((record) => record.lease.unreconciled)) { + // Store-wide: once a round, for every chat, never once per chat. + if (!(await this.context.reconcile('retry', BACKGROUND))) { + end('contended') + } + } + const due = [...this.owed].filter(([, chat]) => chat.due) + await Promise.all( + due.map(async ([sessionId, chat]) => { + const visit = contended ? null : await this.visitInSlot(sessionId, chat, () => contended) + if (visit) { + end(visit) + } + }) + ) + this.running = false + this.roundEnded(failed, contended) + // After the round's outcome is set, so a waiter reads its backoff, never a round still running. + for (const [, chat] of due) { + chat.attempted.splice(0).forEach((resolve) => resolve()) + } + } + + private roundEnded(failed: boolean, contended: boolean): void { + if (this.disposed || this.running) { + return + } + if (!failed) { + this.failures = 0 + this.contended = false + if (this.again && this.timer === null) { + this.freshEpisode() + } + return + } + this.failures += 1 + this.contended = contended + this.stopTimer() + if (this.failures < RECONCILIATION_MAX_FAILED_ATTEMPTS) { + const timer = setTimeout(() => void this.round(), reconciliationBackoffDelay(this.failures)) + // A backoff alone never keeps the process alive. + timer.unref?.() + this.timer = timer + return + } + // The episode ends; the set stays for the next signal, open or commit. + this.context.deps.logger.warn("gave up settling a gone agent's leftover work for now", { + scope: 'reconciliation', + error: new Error(`${this.failures} rounds failed`) + }) + for (const [sessionId, chat] of this.owed) { + if (chat.send) { + this.context.abandonSend(sessionId) + } + } + } + + /** An open chat goes straight to its lane; a closed one is replayed in a background slot. */ + private async visitInSlot( + sessionId: string, + chat: OwedChat, + stopped: () => boolean + ): Promise { + if (this.context.sessions.has(sessionId) || chat.loaded) { + return this.context.track(this.visit(sessionId, chat)) + } + // The replay yields a macrotask first (`loadStructuredAgentSessionForReconciliation`). + const result = await this.slots.run(chat, async () => { + if (stopped() || this.disposed) { + return null + } + return this.context.track(this.visit(sessionId, chat)) + }) + return result?.value ?? null + } + + private visit(sessionId: string, chat: OwedChat): Promise { + return visitStructuredAgentSessionOwedChat(this.visitHost, sessionId, chat) + } + + private settle(sessionId: string, chat: OwedChat): 'settled' { + if (this.owed.get(sessionId) === chat) { + this.owed.delete(sessionId) + } + dropStructuredAgentSessionLoaded(chat) + chat.attempted.splice(0).forEach((resolve) => resolve()) + return 'settled' + } + + private stopTimer(): void { + if (this.timer && this.timer !== 'soon') { + clearTimeout(this.timer) + } + if (this.timer !== 'soon') { + this.timer = null + } + } + + private warn(sessionId: string, error: unknown): void { + this.context.deps.logger.warn("settling a gone agent's leftover work did not finish", { + scope: 'reconciliation', + sessionId, + error + }) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts new file mode 100644 index 000000000000..baff908f8208 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts @@ -0,0 +1,186 @@ +// One visit of the retry (`StructuredAgentSessionRetry`) to one owed chat: release repair, +// settlement and what an earlier process left (`runStructuredAgentSessionReconciliationPass`) in +// the chat's lane, then the queue's next send. Its outcome tells the round how to go on. + +import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' +import { + runStructuredAgentSessionReconciliationPass, + type StructuredAgentSessionReconciliationDebts, + type StructuredAgentSessionReconciliationPassContext +} from './structured-agent-session-reconciliation-pass' +import { + loadStructuredAgentSessionForReconciliation, + structuredAgentSessionJournalIsCurrent +} from './structured-agent-session-reconciliation-load' +import type { StructuredAgentSessionReconciliationSlotWaiter } from './structured-agent-session-reconciliation-slots' + +export type StructuredAgentSessionRetryContext = StructuredAgentSessionReconciliationPassContext & { + deps: StructuredAgentSessionReconciliationPassContext['deps'] + /** The chat's action lane. */ + serialize: (sessionId: string, task: () => Promise) => Promise + /** Lets a quit wait for a visit already writing. */ + track: (operation: Promise) => Promise + /** Publishes what is current now and wakes the queued-card drain. */ + publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void + /** Startup's store-wide lease reconcile (`createReaderReconcile`): whether every lease settled. */ + reconcile: (sessionId: string, options: { background: true }) => Promise + /** The queue's automatic send of the chat's next card, and an episode giving up on it + * (`StructuredAgentSessionQueuedMessageDrain.sendForRetry`, `.abandon`). */ + sendQueued: (sessionId: string) => Promise<'contended' | 'done'> + abandonSend: (sessionId: string) => void +} + +/** `again`: it wrote, or was signalled meanwhile, so the next round verifies nothing is left. */ +export type StructuredAgentSessionVisit = 'settled' | 'again' | 'parked' | 'failed' | 'contended' + +/** A chat with something owed, until a visit finds nothing left. */ +export type StructuredAgentSessionOwedChat = StructuredAgentSessionReconciliationSlotWaiter & { + debts: StructuredAgentSessionReconciliationDebts + /** A visit is owed now; false while only a recovering lease's proofs wait for its decision. */ + due: boolean + /** The queue's automatic send met contention: it waits for the next round to wake it. */ + send?: true + /** Signalled while its visit ran: the next round visits it again. */ + dirty: boolean + attempted: (() => void)[] + /** The visit's own read of the chat while nobody has it open; closed when the chat settles. */ + loaded?: StructuredAgentSessionHostSession +} + +/** What a visit reads of the retry running it. */ +export type StructuredAgentSessionVisitHost = { + context: StructuredAgentSessionRetryContext + disposed: () => boolean + /** Where this process first opened each chat's journal (`StructuredAgentSessionRetry`). */ + firstOpened: Map + /** The chat leaves the owed set. */ + settle: (sessionId: string, chat: StructuredAgentSessionOwedChat) => 'settled' + warn: (sessionId: string, error: unknown) => void +} + +type Chat = StructuredAgentSessionOwedChat +type Visit = StructuredAgentSessionVisit + +export async function visitStructuredAgentSessionOwedChat( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Promise { + const { context } = host + try { + const { store } = context.deps + if (!store.getRecord(sessionId) || store.readOnly) { + // The chat is gone, or nothing here may write: nothing of it is owed again. + if (!store.getRecord(sessionId)) { + host.firstOpened.delete(sessionId) + } + return host.settle(sessionId, chat) + } + if (store.getRecord(sessionId)?.lease.unreconciled) { + return 'failed' + } + if (!context.sessions.has(sessionId) && !chat.loaded) { + const loaded = await loadStructuredAgentSessionForReconciliation( + context.deps, + sessionId, + () => host.disposed() || context.sessions.has(sessionId) + ) + if (loaded.kind === 'loaded') { + chat.loaded = loaded.session + noteStructuredAgentSessionOpened(host.firstOpened, sessionId, loaded.session.journal) + } else if (loaded.kind === 'nothing') { + return host.settle(sessionId, chat) + } + } + const visit = await context.serialize(sessionId, () => pass(host, sessionId, chat)) + if (visit !== 'settled' && visit !== 'parked') { + return visit + } + // (d) once nothing else of the chat is due: the send it owes, and the card named again. + if (chat.send) { + if ((await context.sendQueued(sessionId)) === 'contended') { + return 'contended' + } + delete chat.send + context.publishGenerationEnded(sessionId) + } + return visit === 'settled' ? host.settle(sessionId, chat) : visit + } catch (error) { + host.warn(sessionId, error) + return isSqliteContentionFailure(error) ? 'contended' : 'failed' + } +} + +/** Where this process first opened the chat, if this is the first. */ +export function noteStructuredAgentSessionOpened( + firstOpened: Map, + sessionId: string, + journal: { openedAt: () => AgentJournalCursor } +): void { + if (!firstOpened.has(sessionId)) { + firstOpened.set(sessionId, journal.openedAt()) + } +} + +/** Closes the visit's own read of the chat. */ +export function dropStructuredAgentSessionLoaded(chat: Chat): void { + const journal = chat.loaded?.journal + chat.loaded = undefined + void journal?.close() +} + +/** The pass, inside the chat's lane, on the conversation a reader holds or the visit's own. */ +async function pass( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Promise { + const session = sessionFor(host, sessionId, chat) + if (host.disposed() || !session) { + // Closed, or written by another handle, since this visit began: the next one reads it again. + return host.disposed() ? 'settled' : 'failed' + } + chat.dirty = false + const result = await runStructuredAgentSessionReconciliationPass( + host.context, + sessionId, + session, + chat.debts, + host.firstOpened.get(sessionId) ?? session.journal.openedAt() + ) + if (result.failed.length > 0) { + host.warn(sessionId, result.failed[0]) + return result.failed.some(isSqliteContentionFailure) ? 'contended' : 'failed' + } + if (result.wrote || chat.dirty) { + return 'again' + } + if (result.recovering && chat.debts.evidence) { + // Its proofs wait for the recovery's decision, whose release signals again. An exit's account + // judges only its own generation, which a later one may have replaced by then. + delete chat.debts.exit + chat.due = false + return 'parked' + } + return 'settled' +} + +function sessionFor( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Pick | undefined { + const open = host.context.sessions.get(sessionId) + const { loaded } = chat + if ( + open || + (loaded && + !structuredAgentSessionJournalIsCurrent(host.context.deps, sessionId, loaded.journal)) + ) { + dropStructuredAgentSessionLoaded(chat) + return open + } + return loaded +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts deleted file mode 100644 index 6064ccf46bd4..000000000000 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-worker.ts +++ /dev/null @@ -1,384 +0,0 @@ -// One in-memory reconciliation worker per chat: what ended generations left, and what an earlier -// host process left, settled in the background after every signal that a generation ended. -// -// Signals: every committed revocation or supersession of a generation (the record store's -// `onGenerationEnded`), an exit this host observed (whether or not its release landed), a row an -// ended generation committed late, and startup. Each one re-derives what readers see at once (the -// operational revision, the published view, the queued-card drain), then wakes the worker. Startup -// carries no debt: it only wakes every chat, since every pass derives what is owed. -// -// The worker never decides a lease latched in recovery: that signals a process that may still -// run, which only a person's view or action on the chat does (the visible-tab restore, a start, an -// attach). It still keeps what the earlier process left as cards; the decision's release is the -// signal that settles the rest, so a worker retiring meanwhile parks the debts it held for it. -// -// The worker coalesces: a chat has at most one attempt running and one timer waiting, and every -// attempt re-derives everything owed (`runStructuredAgentSessionReconciliationPass`). A chat nobody -// has open is replayed in one of a few background slots outside its lane, as the worker's own read -// (`structured-agent-session-reconciliation-load.ts`): never the chat's conversation, so nothing it -// writes is published and nothing it closes was anyone's. Only the pass takes the lane, so a send -// that arrived meanwhile goes first; a chat a reader opens moves ahead of the rest of the scan. -// A failure backs off from 1 s to 30 s, in memory only, and is given up after a bounded run of -// them: the next signal or the next startup derives whatever is still owed again. The worker -// retires once a pass finds nothing owed and wrote nothing, when the chat's record is gone or its -// journal can never load, when the database is read-only, or at shutdown. Nothing a person does -// (send, start, open, Stop, answer) ever waits on it. - -import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' -import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' -import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' -import { - runStructuredAgentSessionReconciliationPass, - type StructuredAgentSessionReconciliationDebts, - type StructuredAgentSessionReconciliationPassContext -} from './structured-agent-session-reconciliation-pass' -import { - loadStructuredAgentSessionForReconciliation, - structuredAgentSessionJournalIsCurrent -} from './structured-agent-session-reconciliation-load' -import { StructuredAgentSessionReconciliationMemory } from './structured-agent-session-reconciliation-memory' -import { - RECONCILIATION_MAX_FAILED_ATTEMPTS, - reconciliationBackoffDelay -} from './structured-agent-session-reconciliation-backoff' -import { - StructuredAgentSessionReconciliationSlots, - type StructuredAgentSessionReconciliationSlotWaiter -} from './structured-agent-session-reconciliation-slots' - -export type StructuredAgentSessionReconciliationSignal = { - /** Proof the lease no longer holds (`AgentSessionGenerationEnd.evidence`). */ - evidence?: AgentSessionGenerationEnd['evidence'] - /** An observed exit whose own settlement did not land. */ - exit?: StructuredAgentSessionExitSettlement - /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ - restate?: boolean -} - -export type StructuredAgentSessionReconciliationContext = - StructuredAgentSessionReconciliationPassContext & { - deps: StructuredAgentSessionReconciliationPassContext['deps'] - /** The chat's action lane. */ - serialize: (sessionId: string, task: () => Promise) => Promise - /** Lets a quit wait for an attempt already writing. */ - track: (operation: Promise) => Promise - /** Publishes what is current now and wakes the queued-card drain. */ - publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void - /** Startup's lease reconcile (`createReaderReconcile`): whether every lease is settled. */ - reconcile: (sessionId: string) => Promise - } - -/** `done`: a pass found nothing owed, unless a signal came meanwhile; `retire`: nothing this - * worker can do, whatever came meanwhile (a later signal starts a new one); `again`: a pass wrote - * rows, so the next one verifies at once; `stale`: no pass ran (its read went stale, or the - * store-wide reconcile has not settled), which backs off quietly like a failure. */ -type Outcome = 'done' | 'retire' | 'again' | 'stale' | 'failed' - -type ChatWorker = StructuredAgentSessionReconciliationSlotWaiter & { - debts: StructuredAgentSessionReconciliationDebts - running: boolean - /** A signal arrived while an attempt ran: the next attempt follows at once. */ - dirty: boolean - /** `soon`: an attempt queued for this turn of the event loop; else a backoff's timer. */ - timer: ReturnType | 'soon' | null - failures: number - attempted: (() => void)[] - /** The worker's own read of the chat while nobody has it open; closed when it retires. */ - loaded?: StructuredAgentSessionHostSession -} - -export class StructuredAgentSessionReconciliation { - private readonly workers = new Map() - private readonly memory = new StructuredAgentSessionReconciliationMemory() - private disposed = false - private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) - private readonly unsubscribe: () => void - - constructor(private readonly context: StructuredAgentSessionReconciliationContext) { - this.unsubscribe = context.deps.store.onGenerationEnded((ended) => - this.signal(ended.sessionId, { evidence: ended.evidence }) - ) - } - - signal = (sessionId: string, signal: StructuredAgentSessionReconciliationSignal = {}): void => { - const session = this.context.sessions.get(sessionId) - if (session) { - session.operationalRevision = (session.operationalRevision ?? 0) + 1 - } - // Published once the worker owes: the queue's next card waits on an owed release, so no client - // is told that card sends next meanwhile. - const worker = this.disposed ? null : this.workerFor(sessionId) - this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) - if (!worker) { - return - } - if (signal.evidence) { - worker.debts.evidence = [...(worker.debts.evidence ?? []), signal.evidence] - } - if (signal.exit) { - worker.debts.exit = signal.exit - } - if (worker.running) { - worker.dirty = true - } else if (worker.failures === 0 && !worker.timer) { - // Coalesced with whatever else this tick signals; during a backoff the timer runs it. - this.schedule(sessionId, worker, 0) - } - } - - /** A committed row below the lease's fence: a generation that ended wrote it late. */ - observeCommit = (sessionId: string, lowestFence: number | null): void => { - const fence = this.context.deps.store.getRecord(sessionId)?.lease.runtimeFence - if (lowestFence !== null && fence !== undefined && lowestFence < fence) { - this.signal(sessionId) - } - } - - /** A reader opened the chat: its handle marks the process boundary if it is the first, and what - * its worker still waits a slot for goes ahead of the scan. */ - opened = (sessionId: string, journal: Pick): void => { - this.memory.noteOpened(sessionId, journal) - const worker = this.workers.get(sessionId) - if (worker) { - this.slots.prioritize(worker) - } - } - - /** Resolves once the chat's next attempt finished, or at once when it has no worker. */ - attempted = (sessionId: string): Promise => { - const worker = this.workers.get(sessionId) - return worker ? new Promise((resolve) => worker.attempted.push(resolve)) : Promise.resolve() - } - - /** Whether the chat has a worker: something was owed at its last attempt. */ - owes = (sessionId: string): boolean => this.workers.has(sessionId) - - /** Resolves once the chat's worker retired, or is waiting out a backoff. */ - idle = async (sessionId: string): Promise => { - for (;;) { - const worker = this.workers.get(sessionId) - if (!worker || (!worker.running && worker.failures > 0)) { - return - } - await new Promise((resolve) => worker.attempted.push(resolve)) - } - } - - /** Shutdown: every timer stops and every slot wait ends; an attempt already writing finishes, - * tracked, and one that has not started work starts none. */ - dispose = (): void => { - this.disposed = true - this.unsubscribe() - for (const [sessionId, worker] of this.workers) { - worker.slotWait?.abort() - this.retire(sessionId, worker) - } - } - - private workerFor(sessionId: string): ChatWorker { - let worker = this.workers.get(sessionId) - if (!worker) { - worker = { - debts: this.memory.take(sessionId), - running: false, - dirty: false, - timer: null, - failures: 0, - attempted: [], - urgent: false - } - this.workers.set(sessionId, worker) - } - return worker - } - - private attempt(sessionId: string): void { - const worker = this.workers.get(sessionId) - if (!worker || worker.running) { - return - } - worker.timer = null - worker.running = true - worker.dirty = false - void this.context - .track(this.runAttempt(sessionId, worker)) - .then((outcome) => this.afterAttempt(sessionId, worker, outcome)) - } - - private async runAttempt(sessionId: string, worker: ChatWorker): Promise { - try { - const { store } = this.context.deps - if (!this.disposed && !store.getRecord(sessionId)) { - // The chat is gone: nothing of it is owed again. - this.memory.forget(sessionId) - return 'retire' - } - if (this.disposed || store.readOnly) { - return 'retire' - } - // Startup's reconcile failed (storage was busy): retried as one store-wide step, so a gone - // owner's lease does not read live until a person sends. It never stops a process. - const reconcile = () => this.context.reconcile(sessionId) - if ( - store.getRecord(sessionId)?.lease.unreconciled && - !(await this.memory.reconcileOnce(reconcile)) - ) { - return 'stale' - } - if (!this.context.sessions.has(sessionId) && !worker.loaded) { - const stop = await this.load(sessionId, worker) - if (stop) { - return stop - } - } - return await this.context.serialize(sessionId, () => this.runPass(sessionId, worker)) - } catch (error) { - this.warn(sessionId, error) - return 'failed' - } - } - - /** The pass, inside the chat's lane, on the conversation a reader holds or the worker's own. */ - private async runPass(sessionId: string, worker: ChatWorker): Promise { - if (this.disposed) { - return 'retire' - } - const session = this.sessionFor(sessionId, worker) - // Closed, or written by another handle, since this attempt began: the next one reads it again. - if (!session) { - return 'stale' - } - const pass = await runStructuredAgentSessionReconciliationPass( - this.context, - sessionId, - session, - worker.debts, - this.memory.processOpened(sessionId) ?? session.journal.openedAt() - ) - if (pass.failed.length > 0) { - this.warn(sessionId, pass.failed[0]) - return 'failed' - } - return pass.wrote ? 'again' : 'done' - } - - private sessionFor( - sessionId: string, - worker: ChatWorker - ): Pick | undefined { - const open = this.context.sessions.get(sessionId) - if (open) { - this.dropLoaded(worker) - return open - } - const { loaded } = worker - if ( - loaded && - !structuredAgentSessionJournalIsCurrent(this.context.deps, sessionId, loaded.journal) - ) { - this.dropLoaded(worker) - return undefined - } - return loaded - } - - /** Replays the journal in a background slot as the worker's own read; null to go on to the pass, - * else how the attempt ends. */ - private async load(sessionId: string, worker: ChatWorker): Promise { - const result = await this.slots.run(worker, () => - loadStructuredAgentSessionForReconciliation( - this.context.deps, - sessionId, - () => this.disposed || this.context.sessions.has(sessionId) - ) - ) - const loaded = result?.value - if (!loaded || this.disposed) { - if (loaded?.kind === 'loaded') { - void loaded.session.journal.close() - } - return 'retire' - } - if (loaded.kind === 'loaded') { - worker.loaded = loaded.session - this.memory.noteOpened(sessionId, loaded.session.journal) - } - // Skipped: a reader opened it meanwhile, and the pass uses their conversation. - return loaded.kind === 'nothing' ? 'retire' : null - } - - private dropLoaded(worker: ChatWorker): void { - const journal = worker.loaded?.journal - worker.loaded = undefined - void journal?.close() - } - - private warn(sessionId: string, error: unknown): void { - this.context.deps.logger.warn("settling a gone agent's leftover work did not finish", { - scope: 'reconciliation', - sessionId, - error - }) - } - - private afterAttempt(sessionId: string, worker: ChatWorker, outcome: Outcome): void { - worker.running = false - const attempted = worker.attempted.splice(0) - attempted.forEach((resolve) => resolve()) - if (this.workers.get(sessionId) !== worker || this.disposed) { - // Retired while this attempt ran: its own read closes now that nothing writes through it. - this.dropLoaded(worker) - return - } - if (outcome === 'failed' || outcome === 'stale') { - worker.failures += 1 - if (worker.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { - this.context.deps.logger.warn("gave up settling a gone agent's leftover work for now", { - scope: 'reconciliation', - sessionId, - error: new Error(`${worker.failures} attempts failed`) - }) - this.retire(sessionId, worker) - return - } - this.schedule(sessionId, worker, reconciliationBackoffDelay(worker.failures)) - return - } - worker.failures = 0 - if (outcome === 'again' || (outcome === 'done' && worker.dirty)) { - this.schedule(sessionId, worker, 0) - return - } - this.retire(sessionId, worker) - } - - private schedule(sessionId: string, worker: ChatWorker, delay: number): void { - if (delay === 0) { - worker.timer = 'soon' - queueMicrotask(() => this.attempt(sessionId)) - return - } - const timer = setTimeout(() => this.attempt(sessionId), delay) - // A backoff alone never keeps the process alive. - timer.unref?.() - worker.timer = timer - } - - private retire(sessionId: string, worker: ChatWorker): void { - if (worker.timer && worker.timer !== 'soon') { - clearTimeout(worker.timer) - } - worker.timer = null - this.workers.delete(sessionId) - if (!this.disposed && this.context.deps.store.getRecord(sessionId)) { - this.memory.park(sessionId, worker.debts) - // The queue's automatic send waits on a release a worker owes: it re-derives now. - this.context.publishGenerationEnded(sessionId) - } - if (!worker.running) { - this.dropLoaded(worker) - worker.attempted.splice(0).forEach((resolve) => resolve()) - } - } -} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts index 3c5b0cb1d011..7b69f67e383b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts @@ -1,6 +1,6 @@ // Startup's lease reconcile, failed because another connection held the database, is retried for -// every chat as ONE store-wide step: one call per backoff step however many chats wait on it, so N -// workers never stack N waits on a busy store, and it never stops a process. +// every chat as ONE store-wide step of the retry's round: one call per round however many chats +// wait on it, so N chats never stack N waits on a busy store, and it never stops a process. import { DatabaseSync } from 'node:sqlite' import { mkdtemp, rm } from 'node:fs/promises' @@ -81,8 +81,8 @@ describe('a failed startup reconcile, retried by every chat that waits on it', ( expect(stops).toEqual([]) }) - it('waits on a real write lock once per backoff step, not once per chat', async () => { - await seedChats(8, true) + it('meets a real write lock once per round for 40 chats, never once per chat', async () => { + await seedChats(40, true) expect(store.listRecords().every((record) => record.lease.unreconciled)).toBe(true) openTestJournalHostDatabase(root).db.pragma('busy_timeout = 300') const reconcile = vi.spyOn(store, 'reconcileOnRestart') @@ -93,6 +93,7 @@ describe('a failed startup reconcile, retried by every chat that waits on it', ( await host.reconcileRestartLeases() await new Promise((resolve) => setTimeout(resolve, 4_000)) + // Startup's own, then one per round at 0 s, 1 s and 3 s: background, so each fails at once. expect(reconcile.mock.calls.length).toBeLessThanOrEqual(5) expect(store.listRecords().every((record) => record.lease.unreconciled)).toBe(true) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index f8bf3a29a16b..a841a649ba82 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -9,6 +9,7 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { classifyStoreFailure } from './structured-agent-session-attach' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import type { JournalWriteOptions } from '../agent-session-journal/journal-database' const MAX_RECONCILIATION_PASSES = 8 @@ -21,12 +22,18 @@ export function createRestartReconciler(deps: { records: readonly AgentSessionRecord[] ) => Promise> now: () => number -}): (sessionId: string) => Promise { +}): (sessionId: string, options?: JournalWriteOptions) => Promise { let pending: Promise | null = null - return async (sessionId) => { + return async (sessionId, options) => { if (!deps.store.listRecords().some((record) => record.lease.unreconciled)) { return null } + // Background bookkeeping joins a reader's run, never the reverse: no reader inherits its + // refusal to wait for another connection's lock. + if (options?.background && !pending) { + await reconcileCurrentLeases(deps, options) + return null + } if (!pending) { const run = reconcileCurrentLeases(deps) pending = run.finally(() => { @@ -82,13 +89,16 @@ export function reportEachFailureOnce( * reconciles again before it acts. * Answers whether every lease is settled. */ export function createReaderReconcile( - reconcile: (sessionId: string) => Promise, + reconcile: ( + sessionId: string, + options?: JournalWriteOptions + ) => Promise, failures: ReaderBookkeepingFailures -): (sessionId: string) => Promise { - return async (sessionId) => { +): (sessionId: string, options?: JournalWriteOptions) => Promise { + return async (sessionId, options) => { let failure: unknown try { - const refusal = await reconcile(sessionId) + const refusal = await reconcile(sessionId, options) if (!refusal) { failures.clear() return true @@ -114,20 +124,26 @@ function failureKey(failure: unknown): string { return String(failure) } -async function reconcileCurrentLeases(deps: { - store: AgentSessionRecordStore - probe: (record: AgentSessionRecord) => Promise - probeMany?: ( - records: readonly AgentSessionRecord[] - ) => Promise> - now: () => number -}): Promise { +async function reconcileCurrentLeases( + deps: { + store: AgentSessionRecordStore + probe: (record: AgentSessionRecord) => Promise + probeMany?: ( + records: readonly AgentSessionRecord[] + ) => Promise> + now: () => number + }, + options?: JournalWriteOptions +): Promise { for (let pass = 0; pass < MAX_RECONCILIATION_PASSES; pass += 1) { - await deps.store.reconcileOnRestart({ - probe: deps.probe, - ...(deps.probeMany ? { probeMany: deps.probeMany } : {}), - now: deps.now() - }) + await deps.store.reconcileOnRestart( + { + probe: deps.probe, + ...(deps.probeMany ? { probeMany: deps.probeMany } : {}), + now: deps.now() + }, + options + ) if (!deps.store.listRecords().some((record) => record.lease.unreconciled)) { return } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts index 93a7962d4a20..2419651b7143 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resume-eligibility.ts @@ -12,17 +12,32 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { randomUUID } from 'node:crypto' import type { AgentSessionAttachParams } from './structured-agent-session-attach' import { attachParamsForRecord } from './structured-agent-session-conversation-open' +import type { StructuredAgentSessionEndedChild } from './structured-agent-session-host-types' +import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' -export function isResumableStructuredAgentSessionRecord(record: AgentSessionRecord): boolean { - return agentSessionLeaseIsReleased(record.lease) +/** Released, or still held by a child this host saw end at the lease's own fence (`ended`, its + * `lastEndedChild`), whose release write did not land: the acquisition probes that owner and + * replaces it only once proven dead. Nobody else's exit widens this, so an SSH or unverifiable + * owner is refused as before. */ +export function isResumableStructuredAgentSessionRecord( + record: AgentSessionRecord, + ended?: Pick +): boolean { + return ( + agentSessionLeaseIsReleased(record.lease) || + (ended?.rootGone === true && + ended.fence === record.lease.runtimeFence && + isSurfaceReleasableAgentSessionRecord(record)) + ) } /** Attach params for a resume, or null when this record's lease is somebody else's problem. */ export function structuredAgentSessionResumeParams( record: AgentSessionRecord, - clientOperationId: string + clientOperationId: string, + ended?: Pick ): AgentSessionAttachParams | null { - if (!isResumableStructuredAgentSessionRecord(record)) { + if (!isResumableStructuredAgentSessionRecord(record, ended)) { return null } return attachParamsForRecord(record, { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index ad83758a68e5..efa3239d9683 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -25,9 +25,9 @@ import type { } from './structured-agent-session-host-types' import { scanStructuredAgentSessionsAtStartup } from './structured-agent-session-startup-settlement' import { - StructuredAgentSessionReconciliation, - type StructuredAgentSessionReconciliationContext -} from './structured-agent-session-reconciliation-worker' + StructuredAgentSessionRetry, + type StructuredAgentSessionRetryContext +} from './structured-agent-session-reconciliation-retry' import type { StructuredAgentSessionTaskQueue } from './structured-agent-session-task-queue' /** Throws its refusal as the code itself. */ @@ -70,9 +70,13 @@ export function createStructuredAgentSessionHostRestore( > & { reconcileLeases: (sessionId: string) => Promise resolveRecovery: (sessionId: string) => Promise - startup: Pick & { + startup: Pick & { tasks: Pick - clientDelivery: Pick + clientDelivery: Pick + queue: { + sendForRetry: (sessionId: string) => Promise<'contended' | 'done'> + abandon: (sessionId: string) => void + } } } ): { @@ -80,12 +84,12 @@ export function createStructuredAgentSessionHostRestore( /** The startup scan the last reconcile began; resolved once every chat's first attempt ran. */ startupSettled: () => Promise restoreReadableSessions: (sessionIds?: readonly string[]) => Promise - reconciliation: StructuredAgentSessionReconciliation + reconciliation: StructuredAgentSessionRetry } { const { reconcileLeases, resolveRecovery, startup, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) const reconcile = createReaderReconcile(reconcileLeases, failures) - const reconciliation = new StructuredAgentSessionReconciliation({ + const reconciliation = new StructuredAgentSessionRetry({ deps, sessions: startup.sessions, now: () => deps.now?.() ?? Date.now(), @@ -94,7 +98,9 @@ export function createStructuredAgentSessionHostRestore( track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => startup.clientDelivery.publishGenerationEnded(sessionId, options), - reconcile + reconcile, + sendQueued: (sessionId) => startup.queue.sendForRetry(sessionId), + abandonSend: (sessionId) => startup.queue.abandon(sessionId) }) let startupSettled: Promise = Promise.resolve() const restorer = new StructuredAgentSessionReadableRestorer({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts index 7c4330118e2d..45539542f109 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -10,12 +10,12 @@ // start or read of a chat late in the scan never waits behind it, and an opened chat goes first. import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' -import type { StructuredAgentSessionReconciliation } from './structured-agent-session-reconciliation-worker' +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' /** Resolves once every chat's first attempt finished, whatever it found. */ export function scanStructuredAgentSessionsAtStartup( store: Pick, - reconciliation: Pick + reconciliation: Pick ): Promise { const sessionIds = store.listRecords().map((record) => record.sessionId) for (const sessionId of sessionIds) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts index df4d5fb28c31..371bb87a9f63 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unexpected-exit.test.ts @@ -431,9 +431,11 @@ describe('provider-exit settlement', () => { await settleStructuredAgentSessionChildExit(context, event) expect(session.child).toBeNull() - expect(generationEnded).toHaveBeenCalledTimes(1) - // The worker is handed the exit's own account, to write once the journal takes it. - expect(generationEnded).toHaveBeenCalledWith(SESSION, { + // The death first, before anything is written; then the exit's own account, for the retry to + // write once the journal takes it. + expect(generationEnded).toHaveBeenCalledTimes(2) + expect(generationEnded).toHaveBeenNthCalledWith(1, SESSION) + expect(generationEnded).toHaveBeenLastCalledWith(SESSION, { restate: true, exit: expect.objectContaining({ ownerFence: 7, showUnexpectedExitOutcome: true }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts index d174065cc133..d4b653c4326f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unsent-send-hold.test.ts @@ -599,7 +599,7 @@ describe('what is not kept', () => { ) }) - it('a failed keep rejects nothing: a fresh send is delivered, the earlier one is never handed over, and the next open keeps it', async () => { + it('a failed keep rejects nothing: a fresh send is delivered, the earlier one is never handed over, and the retry keeps it', async () => { const id = await acceptWhileStarting(sendRequest('open write failed')) const resolve = AgentSessionJournal.prototype.resolveDispatch // The open's keep and its plain-rejection fallback both fail. @@ -621,14 +621,14 @@ describe('what is not kept', () => { const next = rig.send('wakes the loop') expect(await next.result).toMatchObject({ ok: true }) await eventually(() => expect(dispatchedTexts()).toEqual(['wakes the loop'])) + // The retry's next round lands the bookkeeping in this process and keeps it as a card. + await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) const earlier = await rig.submission(id) - expect(earlier).toMatchObject({ dispatchState: 'pending', handoverRecorded: true }) + expect(earlier).toMatchObject({ dispatchState: 'rejected', ...HOST_RESTARTED }) expect(earlier).not.toHaveProperty('handedOverAt') - expect(await rig.drafts()).toEqual([]) - // The next open's bookkeeping lands, and keeps it as a card. + // The next open finds it kept, and nothing hands it over. await quitRestart() await eventually(async () => expect(await rig.drafts()).toEqual([{ messageId: id, ...KEPT }])) - expect(await rig.submission(id)).toMatchObject({ dispatchState: 'rejected', ...HOST_RESTARTED }) expect(dispatchedTexts()).toEqual(['wakes the loop']) }) }) diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 714828272ad3..80e0b949cc55 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -1,6 +1,7 @@ /** Durable single-writer session records and their operation ledger, as rows in the host's chat * journal database. */ +import type { JournalWriteOptions } from '../native-chat/agent-session-journal/journal-database' import { agentSessionRefusalError } from '../../shared/agent-session-wire-refusals' import { createAgentSessionConversationReceipts } from './agent-session-conversation-receipts' import { commitConversationCommandRecord } from './agent-session-conversation-command-record' @@ -261,12 +262,11 @@ export class AgentSessionRecordStore { return this.mutate(args.sessionId, (record) => evictAgentSessionOwner({ ...args, record })) } - async transitionHandoff( + transitionHandoff = ( sessionId: string, - transition: (record: AgentSessionRecord) => AgentSessionRecord - ): Promise { - return this.mutate(sessionId, transition) - } + transition: (record: AgentSessionRecord) => AgentSessionRecord, + options?: JournalWriteOptions + ): Promise => this.mutate(sessionId, transition, options) /** * Adjudicate every lease this host loaded. No lease grants a writer until it appears here. On a @@ -274,13 +274,13 @@ export class AgentSessionRecordStore { * start, and none grants a writer there, since every grant is a write. */ async reconcileOnRestart( - args: AgentSessionRestartProbeArgs + args: AgentSessionRestartProbeArgs, + options?: JournalWriteOptions ): Promise> { const pending = this.listRecords().filter((record) => record.lease.unreconciled) const probes = await collectAgentSessionRestartProbes(pending, args) - return this.transact((draft) => applyAgentSessionRestartProbes(draft, probes, args.now), { - inMemoryWhenReadOnly: true - }) + const write = { inMemoryWhenReadOnly: true, ...options } + return this.transact((draft) => applyAgentSessionRestartProbes(draft, probes, args.now), write) } /** Admits one non-reservation mutation through the durable ledger. */ @@ -332,7 +332,8 @@ export class AgentSessionRecordStore { private async mutate( sessionId: string, - apply: (record: AgentSessionRecord) => AgentSessionRecord + apply: (record: AgentSessionRecord) => AgentSessionRecord, + options?: JournalWriteOptions ): Promise { return this.transact((draft) => { const record = draft.records.get(sessionId) @@ -344,7 +345,7 @@ export class AgentSessionRecordStore { const next = apply(record) draft.records.set(sessionId, next) return next - }) + }, options) } /** Told once, after commit, of each generation a transaction revoked or superseded, whichever @@ -365,7 +366,8 @@ export class AgentSessionRecordStore { * once its rows have committed. */ private transact = async ( apply: (draft: AgentSessionStoreState) => T, - options?: { inMemoryWhenReadOnly?: boolean } & AgentSessionGenerationEndOptions + options?: { inMemoryWhenReadOnly?: boolean } & AgentSessionGenerationEndOptions & + JournalWriteOptions ): Promise => { let ended: AgentSessionGenerationEnd[] = [] let heldBefore = true diff --git a/src/main/runtime/agent-session-store-transactions.ts b/src/main/runtime/agent-session-store-transactions.ts index 3877da33af42..7737de41caf9 100644 --- a/src/main/runtime/agent-session-store-transactions.ts +++ b/src/main/runtime/agent-session-store-transactions.ts @@ -8,6 +8,7 @@ // the async boundary every awaiting caller was written against. import type { JournalHostDatabase } from '../native-chat/agent-session-journal/journal-host-database' +import type { JournalWriteOptions } from '../native-chat/agent-session-journal/journal-database' import type { JournalOperationReceipt } from '../native-chat/agent-session-journal/journal-row-writer' import { journalOpenRefusalError } from '../native-chat/agent-session-journal/journal-open-failure' import { AgentSessionJournalError } from '../native-chat/agent-session-journal/journal-write-guards' @@ -115,9 +116,11 @@ export class AgentSessionStoreTransactions { */ transact( apply: (draft: AgentSessionStoreState) => T, - options: { inMemoryWhenReadOnly?: boolean } = {} + options: { inMemoryWhenReadOnly?: boolean } & JournalWriteOptions = {} ): Promise { - const run = this.queue.then(() => this.commit(apply, options.inMemoryWhenReadOnly === true)) + const run = this.queue.then(() => + this.commit(apply, options.inMemoryWhenReadOnly === true, options.background) + ) this.queue = run.catch(() => {}) return run } @@ -147,7 +150,11 @@ export class AgentSessionStoreTransactions { } } - private commit(apply: (draft: AgentSessionStoreState) => T, inMemoryWhenReadOnly: boolean): T { + private commit( + apply: (draft: AgentSessionStoreState) => T, + inMemoryWhenReadOnly: boolean, + background?: true + ): T { const readOnly = this.journalDatabase.readOnly if (readOnly && !inMemoryWhenReadOnly) { throw readOnlyStoreRefusal() @@ -155,7 +162,10 @@ export class AgentSessionStoreTransactions { const staged = this.stage(apply) const writes = staged.writes if (writes && !readOnly) { - this.journalDatabase.transaction((db) => writeAgentSessionStoreRows(db, writes)) + this.journalDatabase.transaction( + (db) => writeAgentSessionStoreRows(db, writes), + background ? { background } : undefined + ) } staged.adopt() return staged.result diff --git a/src/main/runtime/agent-session-surface-release-transition.ts b/src/main/runtime/agent-session-surface-release-transition.ts index 1099e8d6c045..0c9906b624c6 100644 --- a/src/main/runtime/agent-session-surface-release-transition.ts +++ b/src/main/runtime/agent-session-surface-release-transition.ts @@ -8,6 +8,7 @@ // The fence still moves, so the next owner is a new generation: an attach or settlement still // holding the stopped owner's fence is refused as stale rather than acting on its successor. +import type { JournalWriteOptions } from '../native-chat/agent-session-journal/journal-database' import { agentSessionRefusalError } from '../../shared/agent-session-wire-refusals' import { MAX_AGENT_SESSION_DEATH_DETAIL_CHARS, @@ -72,9 +73,12 @@ export function releaseStoredAgentSessionOwnerAfterSurfaceClose( now: number exitObservedAt?: number exitReason?: string - } + }, + options?: JournalWriteOptions ): Promise { - return store.transitionHandoff(args.sessionId, (record) => - releaseAgentSessionOwnerAfterSurfaceClose({ ...args, record }) + return store.transitionHandoff( + args.sessionId, + (record) => releaseAgentSessionOwnerAfterSurfaceClose({ ...args, record }), + options ) } diff --git a/src/main/sqlite/sqlite-read-failure.ts b/src/main/sqlite/sqlite-read-failure.ts index 498bf5c1de12..a1a6767eec1c 100644 --- a/src/main/sqlite/sqlite-read-failure.ts +++ b/src/main/sqlite/sqlite-read-failure.ts @@ -53,6 +53,19 @@ export function isTransientSqliteContention(error: unknown): boolean { return CONTENTION_MESSAGE.test(errorText(error)) } +/** `isTransientSqliteContention` for a failure that wraps others (`cause`, `AggregateError`). */ +export function isSqliteContentionFailure(error: unknown, depth = 0): boolean { + if (depth > 4 || error === undefined || error === null) { + return false + } + if (isTransientSqliteContention(error)) { + return true + } + const wrapped = error instanceof AggregateError ? error.errors : [] + const cause = error instanceof Error ? error.cause : undefined + return [cause, ...wrapped].some((inner) => isSqliteContentionFailure(inner, depth + 1)) +} + export type SqliteReadFailureKind = /** Another connection holds the lock; the same read can succeed later. */ | 'contended' From 16d6a8e31fdb4824bac0421efe55ef01b86192b2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:51:03 -0700 Subject: [PATCH 11/22] fix(native-chat): background visits take turns; a held epoch's failed founding reads as an unreadable journal - each chat's background writes wait their turn and yield before the next chat, so a person's operation queued meanwhile runs first - the queue's reopen mark fails fast on a locked database like the other background writes - a generation's end publishes a frame only when it changes what a reader sees - a held epoch whose own row cannot be written is refused as an unreadable journal, as an eager founding was --- .../journal-epoch-founding.ts | 11 ++++- .../agent-session-journal/journal-store.ts | 32 ++++++------- .../queued-message-reopen-floor.ts | 23 ++++++++++ ...ed-agent-session-background-writes.test.ts | 38 ++++++++++++++++ .../structured-agent-session-create.test.ts | 3 +- .../structured-agent-session-current-work.ts | 7 +-- ...ctured-agent-session-exit-delivery.test.ts | 42 +++++++++++++++++ .../structured-agent-session-queued-pause.ts | 13 ++++-- ...tured-agent-session-reconciliation-pass.ts | 9 +++- ...ured-agent-session-reconciliation-retry.ts | 45 ++++++++++++------- ...ured-agent-session-reconciliation-visit.ts | 37 +++++++++------ ...nt-session-restart-reconcile-retry.test.ts | 43 ++++++++++++++++++ 12 files changed, 247 insertions(+), 56 deletions(-) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts b/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts index a46d512f726d..d59334037231 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-founding.ts @@ -7,6 +7,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session- import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import type Database from '../../sqlite/sync-database' import type { JournalLoad } from './journal-open' +import { journalOpenRefusalError } from './journal-open-failure' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { insertJournalRow, publishJournalSessionEpoch } from './journal-row-table' import type { JournalRow } from './journal-row-schema' @@ -40,11 +41,17 @@ export class JournalEpochFounding { return this.pending !== null } - /** First inside a transaction that writes the chat's rows: a rollback takes the epoch with it. */ + /** First inside a transaction that writes the chat's rows: a rollback takes the epoch with it. + * Its own failure is the journal's open failing, as an eager open's would have been. */ inTransaction(db: Database.Database): void { - if (this.pending) { + if (!this.pending) { + return + } + try { insertJournalRow(db, this.identity.sessionId, this.pending) publishJournalSessionEpoch(db, this.identity, this.pending.epoch) + } catch (error) { + throw journalOpenRefusalError(error) } } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index afc8960fd2e6..88d9ce836f4d 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -1,7 +1,7 @@ // Append-only journal store for one agent session. It owns the chat's fold and write queue, and no // connection: every statement goes through the host's one journal database. -import { queuedMessageReopenFloor } from './queued-message-reopen-floor' +import { QueuedMessageReopenFloor } from './queued-message-reopen-floor' import type { AgentJournalDispatchRejection } from '../../../shared/agent-session-failure-words' import { randomUUID } from 'node:crypto' import type { @@ -26,7 +26,7 @@ import { } from '../../../shared/structured-agent-session-live-turn' import type { JournalReplacementItem } from './journal-epoch-replacement' import { readJournalSince } from './journal-cursor' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import { journalRowsAfterReader, type JournalLoad } from './journal-open' import { markJournalPendingSubmissionsUnknown, @@ -86,7 +86,7 @@ export class AgentSessionJournal { private state: JournalReducerState private openedThrough: AgentJournalCursor = { epoch: '', sequence: 0 } - private reopenUnmarked: AgentJournalCursor | null = null + private readonly reopen: QueuedMessageReopenFloor private onCommitted: ((rows?: readonly JournalRow[]) => void) | null = null private readonly queue: JournalWriteQueue private readonly rowWriter: JournalRowWriter @@ -140,6 +140,7 @@ export class AgentSessionJournal { this.submissionWriter = collaborators.submissionWriter this.stepWriter = collaborators.stepWriter this.queuedMessages = collaborators.queuedMessages + this.reopen = new QueuedMessageReopenFloor(this.queuedMessages) this.stopMarks = collaborators.stopMarks this.restore = collaborators.restore this.context = new JournalContextController({ @@ -172,7 +173,7 @@ export class AgentSessionJournal { /** Where the reopen's pause begins while no mark holds it: where this handle's mark would go. * Null once a mark is written. Per handle, so the next open derives it again. */ - reopenFloor = (): AgentJournalCursor | null => this.reopenUnmarked + reopenFloor = (): AgentJournalCursor | null => this.reopen.get() async open(): Promise { await this.restore() @@ -183,8 +184,7 @@ export class AgentSessionJournal { * a card it finds waits for a turn from where this handle opened, derived with nothing written, * unless an earlier mark already holds it. The mark that follows records the same start. */ holdReopenFromOpen(): void { - this.reopenUnmarked = - queuedMessageReopenFloor(this.queuedMessages, this.openedThrough) ?? this.reopenUnmarked + this.reopen.holdFromOpen(this.openedThrough) } /** Refuses every later write and resolves once the admitted ones have landed. Holds no @@ -316,20 +316,16 @@ export class AgentSessionJournal { } /** This open found waiting cards an earlier handle wrote (`queued-message-pause.ts`). */ - appendQueueReopen(fence: number, since?: number): Promise { - return this.rowWriter.append(journalQueueReopenRowBuilder(() => this.state, fence, since)) + appendQueueReopen(fence: number, since?: number, options?: JournalWriteOptions) { + const build = journalQueueReopenRowBuilder(() => this.state, fence, since) + return this.rowWriter.append(build, undefined, undefined, options) } - /** Marks the reopen when a card waits or is mid-hand-off (it may come back to waiting), from - * `since` when the chat stopped before now; a failed write leaves where the mark would have gone - * as the pause's start (`reopenFloor`), and throws. */ - async markQueueReopen(fence: number, since?: number): Promise { - if (this.queuedMessages.awaitReopenMark()) { - const sequence = since ?? this.state.lastSequence + 1 - this.reopenUnmarked = { epoch: this.state.epoch, sequence } - await this.appendQueueReopen(fence, since) - this.reopenUnmarked = null - } + /** Marks the reopen (`QueuedMessageReopenFloor.mark`), from `since` when the chat stopped before + * now; a failed write leaves the pause's start where the mark would have gone, and throws. */ + markQueueReopen(fence: number, since?: number, options?: JournalWriteOptions): Promise { + const at = { epoch: this.state.epoch, sequence: since ?? this.state.lastSequence + 1 } + return this.reopen.mark(at, () => this.appendQueueReopen(fence, since, options)) } appendLifecycleBatch(input: JournalLifecycleBatchInput): Promise { diff --git a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts index bfce00c8efac..d8e819a7951c 100644 --- a/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts +++ b/src/main/native-chat/agent-session-journal/queued-message-reopen-floor.ts @@ -15,6 +15,29 @@ export function queuedMessageReopenFloor( : null } +/** This handle's floor while no mark holds the pause (`AgentSessionJournal.reopenFloor`). */ +export class QueuedMessageReopenFloor { + private unmarked: AgentJournalCursor | null = null + + constructor(private readonly queue: Parameters[0]) {} + + get = (): AgentJournalCursor | null => this.unmarked + + holdFromOpen(opened: AgentJournalCursor): void { + this.unmarked = queuedMessageReopenFloor(this.queue, opened) ?? this.unmarked + } + + /** Marks the reopen when a card waits or is mid-hand-off (it may come back to waiting); a failed + * write leaves `at`, where the mark would have gone, as the pause's start, and throws. */ + async mark(at: AgentJournalCursor, append: () => Promise): Promise { + if (this.queue.awaitReopenMark()) { + this.unmarked = at + await append() + this.unmarked = null + } + } +} + /** Where the reopen mark of what an earlier host process left starts: this handle's unmarked * floor, else just past where this process first opened the chat; null when neither is in the * journal's current epoch, since a mark from now would hold cards this process queued since. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts index ca4f034f0053..8ebcb5d0c5e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -136,6 +136,44 @@ describe("another connection's write lock", () => { }) }) +describe("the pass's reopen mark", () => { + it('is background: under a real lock it fails at once, ends the round, and lands once the lock lifts', async () => { + rig = await createQueuedMessageTestRig() + const current = rig + await current.workingSend() + await current.send('queued before the crash', 'queue-if-active').result + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const mark = AgentSessionJournal.prototype.markQueueReopen + const tried: { options: unknown; failed: boolean; ms: number }[] = [] + vi.spyOn(AgentSessionJournal.prototype, 'markQueueReopen').mockImplementation(async function ( + this: AgentSessionJournal, + ...args: Parameters + ) { + // The first mark meets another connection's lock, taken after the pass's earlier writes. + if (tried.length === 0) { + await holdWriteLock(current.root, 1_000) + } + const started = performance.now() + try { + await mark.apply(this, args) + tried.push({ options: args[2], failed: false, ms: performance.now() - started }) + } catch (error) { + const ms = performance.now() - started + tried.push({ options: args[2], failed: isSqliteContentionFailure(error), ms }) + throw error + } + }) + await current.crashRestartHostProcess() + + await vi.waitFor(() => expect(tried.some((attempt) => !attempt.failed)).toBe(true), { + timeout: 8_000 + }) + expect(tried[0]).toMatchObject({ options: { background: true }, failed: true }) + expect(tried[0]!.ms).toBeLessThan(150) + expect(tried.at(-1)).toMatchObject({ options: { background: true }, failed: false }) + }) +}) + describe('an episode the retry gives up', () => { /** Lets every visit a fired timer began run to its end, with no fake time passing. */ async function settleTurns(): Promise { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts index c3f2a4855986..978cec42ccca 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-create.test.ts @@ -381,7 +381,8 @@ it.each(['journal_rows', 'agent_session_operations'])( expect(rig.store.getSessionTabId(SESSION)).toBeNull() expect(rig.store.listOperationRows()).toEqual([]) expect((await readPersistedTestAgentSessionStore(rig.root)).records).toEqual({}) - expect(loadTestJournal(rig.root, SESSION)?.state.submissions.size).toBe(0) + // The chat's first epoch is written with its first row, so a refused create leaves no journal. + expect(loadTestJournal(rig.root, SESSION)).toBeNull() expect(rig.host.hasSession(SESSION)).toBe(false) expect(rig.adapter.acquire).not.toHaveBeenCalled() expect(publishTab).not.toHaveBeenCalled() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index 03d3e42168d4..bcb1fc0c6a96 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -85,10 +85,11 @@ export class StructuredAgentSessionCurrentWork { readonly revision = 0 ) {} - /** What a reader that published this answer compares to learn it may have changed with no row: - * the live generation and the revision of what this host saw end. */ + /** What a reader that published this answer compares to learn it changed with no row: the live + * generation, since every answer here is the journal read through it. Not the revision, which + * moves on every generation-end signal: an end that changed nothing is published once. */ viewKey(): string { - return `${this.liveFence}:${this.revision}` + return `${this.liveFence}` } /** Whether the live generation's execution produced this item. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts index c4a9ad9d3484..0d617782cdd1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' import { agentSessionLeaseFixture, agentSessionRecordFixture @@ -136,6 +137,47 @@ describe('an exit that settles and releases cleanly', () => { }) }) +/** What a client shows, without the host clock every frame carries. */ +function readView(state: typeof EMPTY_STRUCTURED_AGENT_SESSION): string { + return JSON.stringify(state, (key, value: unknown) => (key === 'hostNow' ? undefined : value)) +} + +describe('each exit', () => { + // Two generation-end signals reach readers per exit: its death at once, and its outcome once + // settled. The second may restate; it never repeats a frame the first published. + it.each([ + ['its release lands', false], + ['its release keeps failing', true] + ])('publishes each frame once when %s', async (_name, refuse) => { + // No card behind it: a card's own start is a later generation, with frames of its own. + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + if (refuse) { + refuseReleases(current) + } + const events: AgentSessionSubscribeEvent[] = [] + await current.host.subscribe({ id: 'chat', sessionId: SESSION, emit: (e) => events.push(e) }) + const before = events.length + + await exitChild(current) + await current.host.collaboratorsForTests().reconciliation.attempted(SESSION) + + // Every frame changes what a client reads: none repeats a view it already has. + let state = EMPTY_STRUCTURED_AGENT_SESSION + const repeats: string[] = [] + for (const [index, event] of events.entries()) { + const next = reduceStructuredAgentSession(state, { type: 'event', event }) + if (index >= before && readView(next) === readView(state)) { + repeats.push(event.type) + } + state = next + } + expect(repeats).toEqual([]) + }) +}) + describe('an owner this host cannot prove dead', () => { it.each([ ['alive', { outcome: 'identity-matched', matchedOn: ['process-start-time'] }], diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts index 39f79d13a2b3..a6df16b8efae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts @@ -6,6 +6,8 @@ import { randomUUID } from 'node:crypto' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { DerivedQueuePause } from '../agent-session-journal/queued-message-pause' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import type { JournalWriteOptions } from '../agent-session-journal/journal-host-database' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' /** A per-process id, minted once per host process like the runtime's own * `runtimeId` (`orca-runtime-runtime-id.ts`), stamped on the cards this process writes or hands @@ -34,17 +36,22 @@ export function structuredQueuePauses(journal: PauseJournal): DerivedQueuePause[ * chat with cards the earlier process left, and so does a person's close: the idle sweep never closes a chat with cards waiting, so its * eviction never reopens one. `since`: where the chat stopped, for a mark written after a send * that came later, which must still lift it. Bookkeeping, so a failure is reported and never - * thrown; the pause then starts where the mark would have gone, holding no less. */ + * thrown; the pause then starts where the mark would have gone, holding no less. A background + * mark (`JournalWriteOptions`) that met another connection's lock throws, ending the retry's round. */ export async function markStructuredQueueReopen( sessionId: string, journal: Pick, fence: number, logger: StructuredAgentSessionLogger, - since?: number + since?: number, + options?: JournalWriteOptions ): Promise { try { - await journal.markQueueReopen(fence, since) + await journal.markQueueReopen(fence, since, options) } catch (error) { + if (options?.background && isSqliteContentionFailure(error)) { + throw error + } logger.warn('marking a reopened queue failed', { scope: 'queue-reopen-mark', sessionId, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index 79639783edf7..6e581f1fbdff 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -192,7 +192,14 @@ async function settleEarlierProcess( const floor = journal.reopenFloor() const since = queuedMessageReopenMarkStart(floor, processOpened, journal.cursor().epoch) if ((converted || floor !== null) && since !== null) { - await markStructuredQueueReopen(sessionId, journal, fence, context.deps.logger, since) + await markStructuredQueueReopen( + sessionId, + journal, + fence, + context.deps.logger, + since, + BACKGROUND + ) } // A rewind left prepared refuses every send until settled. With no provider here, one only its // provider can prove stays, quietly, for the next acquisition, which recovers it. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts index 672a0debfb29..6f5356320735 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -6,9 +6,11 @@ // `onGenerationEnded`), an exit this host observed (whether or not its release landed), a row an // ended generation committed late, a queue send that met contention, and startup. Each one // re-derives what readers see at once (the operational revision, the published view, the queued- -// card drain) and marks the chat owed. A round visits every owed chat, in the few background slots -// with a reader's open chat first: (a) the store-wide restart reconcile, once, while any lease is -// unreconciled; then per chat, in its lane, (b) release repair, (c) settlement and what an earlier +// card drain) and marks the chat owed. A round visits every owed chat: closed ones load side by +// side in the few background slots, a reader's first, and each writes in its turn, one chat at a +// time with a yield between (`inTurn`): +// (a) the store-wide restart reconcile, once, while any lease is unreconciled; then per chat, in +// its lane, (b) release repair, (c) settlement and what an earlier // process left (`runStructuredAgentSessionReconciliationPass`), (d) the queue's next send. Every // write is background (`JournalWriteOptions`): another connection's lock fails it at once and ends // the round, since contention is connection-wide. @@ -19,6 +21,7 @@ // recovery is never decided here: its proofs wait in the set for its decision's signal. import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' +import { setImmediate as yieldToEvents } from 'node:timers/promises' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' @@ -66,6 +69,9 @@ export class StructuredAgentSessionRetry { /** The last round failed on another connection's lock, which a commit here proves gone. */ private contended = false private disposed = false + /** The round's chain of chat turns (`inTurn`), and whether contention stopped the round. */ + private turn: Promise = Promise.resolve() + private stopped = false private readonly visitHost: StructuredAgentSessionVisitHost constructor(private readonly context: StructuredAgentSessionRetryContext) { @@ -74,7 +80,8 @@ export class StructuredAgentSessionRetry { disposed: () => this.disposed, firstOpened: this.firstOpened, settle: (sessionId, chat) => this.settle(sessionId, chat), - warn: (sessionId, error) => this.warn(sessionId, error) + warn: (sessionId, error) => this.warn(sessionId, error), + inTurn: (run) => this.inTurn(run) } this.unsubscribe = context.deps.store.onGenerationEnded((ended) => this.signal(ended.sessionId, { evidence: ended.evidence }) @@ -196,11 +203,11 @@ export class StructuredAgentSessionRetry { } this.running = true this.again = false + this.stopped = false let failed = false - let contended = false const end = (visit: Visit) => { failed ||= visit === 'failed' || visit === 'contended' - contended ||= visit === 'contended' + this.stopped ||= visit === 'contended' this.again ||= visit === 'again' } if (this.context.deps.store.listRecords().some((record) => record.lease.unreconciled)) { @@ -212,14 +219,14 @@ export class StructuredAgentSessionRetry { const due = [...this.owed].filter(([, chat]) => chat.due) await Promise.all( due.map(async ([sessionId, chat]) => { - const visit = contended ? null : await this.visitInSlot(sessionId, chat, () => contended) + const visit = this.stopped ? null : await this.visitInSlot(sessionId, chat) if (visit) { end(visit) } }) ) this.running = false - this.roundEnded(failed, contended) + this.roundEnded(failed, this.stopped) // After the round's outcome is set, so a waiter reads its backoff, never a round still running. for (const [, chat] of due) { chat.attempted.splice(0).forEach((resolve) => resolve()) @@ -261,17 +268,13 @@ export class StructuredAgentSessionRetry { } /** An open chat goes straight to its lane; a closed one is replayed in a background slot. */ - private async visitInSlot( - sessionId: string, - chat: OwedChat, - stopped: () => boolean - ): Promise { + private async visitInSlot(sessionId: string, chat: OwedChat): Promise { if (this.context.sessions.has(sessionId) || chat.loaded) { return this.context.track(this.visit(sessionId, chat)) } // The replay yields a macrotask first (`loadStructuredAgentSessionForReconciliation`). const result = await this.slots.run(chat, async () => { - if (stopped() || this.disposed) { + if (this.disposed) { return null } return this.context.track(this.visit(sessionId, chat)) @@ -279,10 +282,22 @@ export class StructuredAgentSessionRetry { return result?.value ?? null } - private visit(sessionId: string, chat: OwedChat): Promise { + private visit(sessionId: string, chat: OwedChat): Promise { return visitStructuredAgentSessionOwedChat(this.visitHost, sessionId, chat) } + /** Chats load in the slots side by side, but write one at a time, in the order they are ready (a + * reader's open chat first), yielding between chats: each visit's writes stay bounded, and a + * person's operation queued meanwhile runs before the next chat's turn takes its lane. */ + private inTurn(run: () => Promise): Promise { + const next = this.turn.then(() => (this.stopped || this.disposed ? null : run())) + this.turn = next.then( + () => yieldToEvents(), + () => yieldToEvents() + ) + return next + } + private settle(sessionId: string, chat: OwedChat): 'settled' { if (this.owed.get(sessionId) === chat) { this.owed.delete(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts index baff908f8208..76216ebfaf05 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts @@ -58,6 +58,8 @@ export type StructuredAgentSessionVisitHost = { /** The chat leaves the owed set. */ settle: (sessionId: string, chat: StructuredAgentSessionOwedChat) => 'settled' warn: (sessionId: string, error: unknown) => void + /** The round's one-at-a-time turn for a chat's writes; null once the round stopped. */ + inTurn: (run: () => Promise) => Promise } type Chat = StructuredAgentSessionOwedChat @@ -67,7 +69,7 @@ export async function visitStructuredAgentSessionOwedChat( host: StructuredAgentSessionVisitHost, sessionId: string, chat: Chat -): Promise { +): Promise { const { context } = host try { const { store } = context.deps @@ -94,18 +96,7 @@ export async function visitStructuredAgentSessionOwedChat( return host.settle(sessionId, chat) } } - const visit = await context.serialize(sessionId, () => pass(host, sessionId, chat)) - if (visit !== 'settled' && visit !== 'parked') { - return visit - } - // (d) once nothing else of the chat is due: the send it owes, and the card named again. - if (chat.send) { - if ((await context.sendQueued(sessionId)) === 'contended') { - return 'contended' - } - delete chat.send - context.publishGenerationEnded(sessionId) - } + const visit = await host.inTurn(() => write(host, sessionId, chat)) return visit === 'settled' ? host.settle(sessionId, chat) : visit } catch (error) { host.warn(sessionId, error) @@ -113,6 +104,26 @@ export async function visitStructuredAgentSessionOwedChat( } } +/** The chat's writes, in its lane: the pass, then (d) once nothing else of it is due, the send it + * owes, with the card named again. */ +async function write( + host: StructuredAgentSessionVisitHost, + sessionId: string, + chat: Chat +): Promise { + const { context } = host + const visit = await context.serialize(sessionId, () => pass(host, sessionId, chat)) + if ((visit !== 'settled' && visit !== 'parked') || !chat.send) { + return visit + } + if ((await context.sendQueued(sessionId)) === 'contended') { + return 'contended' + } + delete chat.send + context.publishGenerationEnded(sessionId) + return visit +} + /** Where this process first opened the chat, if this is the first. */ export function noteStructuredAgentSessionOpened( firstOpened: Map, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts index 7b69f67e383b..c489792b334c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts @@ -22,6 +22,7 @@ import { scanRecord, seedScanJournal } from './structured-agent-session-startup-scan.test-fixture' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' let root: string let store: AgentSessionRecordStore @@ -105,3 +106,45 @@ describe('a failed startup reconcile, retried by every chat that waits on it', ( ) }) }) + +describe('a round of the retry', () => { + it("visits one chat at a time: a person's operation on a later chat runs before its visit", async () => { + const chats = await seedChats(3, true) + const order: string[] = [] + let release: () => void = () => undefined + const gate = new Promise((resolve) => (release = resolve)) + const pass = reconciliationPass.runStructuredAgentSessionReconciliationPass + vi.spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass').mockImplementation( + async (...args) => { + order.push(`visit:${args[1]}`) + if (order.length === 1) { + await gate + } + return pass(...args) + } + ) + host = openScanHost(root, store) + await host.reconcileRestartLeases() + await vi.waitFor(() => expect(order.length).toBeGreaterThan(0), { timeout: 10_000 }) + + // A chat the round has not reached: a person's operation on its lane, as a send runs. + const later = chats.find((id) => !order.includes(`visit:${id}`)) + if (!later) { + release() + throw new Error('every chat was visited at once') + } + const person = host + .collaboratorsForTests() + .serialize(later, async () => void order.push(`person:${later}`)) + await new Promise((resolve) => setTimeout(resolve, 50)) + release() + const waited = new Promise((_, reject) => + setTimeout(() => reject(new Error("the person's operation waited behind the round")), 5_000) + ) + await Promise.race([person, waited]) + await vi.waitFor(() => expect(order).toContain(`visit:${later}`), { timeout: 10_000 }) + + expect(order.indexOf(`person:${later}`)).toBeLessThan(order.indexOf(`visit:${later}`)) + expect(new Set(order.filter((entry) => entry.startsWith('visit:'))).size).toBe(3) + }) +}) From a37249171c7a8db38d88a08e17849d9483da70e9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:50:11 -0700 Subject: [PATCH 12/22] fix(native-chat): background paths write through background-bound handles - the exit's settlement and release, every coordinator visit and the queue's automatic send write through handles whose writes always fail fast on a locked database, so none can stall the app - a refused automatic send whose hold cannot be saved still shows as failed - contention is counted per round, a busy chat's lane is skipped rather than waited on, and the retry delay is capped at 2 s while work is owed - removes the deleted worker's leftovers --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + ...aude-subagent-resume-after-restart.test.ts | 4 +- .../journal-context-controller.ts | 6 +- .../journal-context-rewind.ts | 5 +- .../journal-epoch-controller.ts | 8 +- .../journal-epoch-replacement.ts | 5 +- .../journal-lifecycle-batch-appender.ts | 25 ++- .../journal-queued-messages.ts | 13 +- .../agent-session-journal/journal-store.ts | 5 +- .../journal-unsent-send-hold.ts | 22 +-- ...ured-agent-session-background-lock.test.ts | 166 ++++++++++++++++++ ...ed-agent-session-background-writes.test.ts | 55 ++---- ...uctured-agent-session-background-writes.ts | 82 +++++++++ .../structured-agent-session-child-close.ts | 16 +- .../structured-agent-session-child-exit.ts | 20 ++- ...ssion-claude-stop-exit-ends-record.test.ts | 6 +- ...tured-agent-session-crash-turn-end.test.ts | 11 +- ...uctured-agent-session-current-work.test.ts | 5 +- ...ctured-agent-session-exit-delivery.test.ts | 8 +- ...tructured-agent-session-idle-sweep.test.ts | 2 +- .../structured-agent-session-lease-release.ts | 27 ++- ...-session-leftover-settlement-scope.test.ts | 7 +- ...ssion-leftover-settlement-triggers.test.ts | 13 +- ...ession-leftover-settlement.test-fixture.ts | 18 +- ...tured-agent-session-leftover-settlement.ts | 13 +- ...gent-session-provider-child-record.test.ts | 7 +- ...ctured-agent-session-queued-abandonment.ts | 24 +-- ...session-queued-message-rig.test-fixture.ts | 5 +- ...tructured-agent-session-queued-messages.ts | 73 ++++---- .../structured-agent-session-queued-pause.ts | 15 +- ...ed-agent-session-reconciliation-backoff.ts | 7 +- ...tured-agent-session-reconciliation-load.ts | 6 +- ...ession-reconciliation-parked-debts.test.ts | 4 +- ...tured-agent-session-reconciliation-pass.ts | 53 +++--- ...agent-session-reconciliation-retry.test.ts | 25 +-- ...ured-agent-session-reconciliation-retry.ts | 52 +++--- ...ured-agent-session-reconciliation-slots.ts | 3 +- ...ured-agent-session-reconciliation-visit.ts | 59 ++++--- ...gent-session-resend-ledger-refusal.test.ts | 5 +- .../structured-agent-session-rest-test-rig.ts | 5 +- ...nt-session-restart-reconcile-retry.test.ts | 27 +++ ...uctured-agent-session-restart-reconcile.ts | 26 ++- ...tructured-agent-session-restart-restore.ts | 2 +- ...uctured-agent-session-retry-budget.test.ts | 77 ++++++++ ...ctured-agent-session-retry.test-fixture.ts | 22 +++ .../structured-agent-session-reveal.ts | 26 ++- ...ed-agent-session-roster-provenance.test.ts | 15 +- ...agent-session-send-open-stale-turn.test.ts | 5 +- ...ured-agent-session-stale-turn-open.test.ts | 13 +- ...ent-session-startup-recovery-scope.test.ts | 8 +- ...gent-session-startup-scan-lifetime.test.ts | 11 +- ...agent-session-startup-scan.test-fixture.ts | 4 +- ...uctured-agent-session-startup-scan.test.ts | 3 +- ...ctured-agent-session-startup-settlement.ts | 6 +- ...d-agent-session-stop-event-binding.test.ts | 4 +- .../structured-agent-session-task-queue.ts | 5 + ...t-session-wedged-profile-migration.test.ts | 15 +- ...d-agent-session-write-lock.test-fixture.ts | 53 ++++++ .../structured-rewind-recovery.ts | 34 +++- .../agent-session-generation-end.test.ts | 2 + .../runtime/agent-session-record-store.ts | 4 +- ...gent-session-surface-release-transition.ts | 10 +- 62 files changed, 881 insertions(+), 377 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 22fac58d8b9c..e7f8be4e3fb0 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -123,6 +123,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', diff --git a/src/main/claude/claude-subagent-resume-after-restart.test.ts b/src/main/claude/claude-subagent-resume-after-restart.test.ts index 9ac985a160a1..8e68f5a5b5d8 100644 --- a/src/main/claude/claude-subagent-resume-after-restart.test.ts +++ b/src/main/claude/claude-subagent-resume-after-restart.test.ts @@ -23,6 +23,7 @@ import { agentSessionRecordFixture } from '../../shared/agent-session-record.test-fixture' import { settleStructuredAgentSessionLeftovers } from '../native-chat/agent-session-wire/structured-agent-session-leftover-settlement' +import { backgroundSettlementWrites } from '../native-chat/agent-session-wire/structured-agent-session-background-writes' // Frame orders are real sessions', scrubbed. A resumed agent's frames still name its ORIGINAL // spawn call while the announcement names the message call that resumed it, and a new provider @@ -476,7 +477,8 @@ describe('a Claude subagent resumed after its provider restarted', () => { await settleStructuredAgentSessionLeftovers({ store: { getRecord: (sessionId) => (sessionId === 'orca-session' ? released : null) }, sessionId: 'orca-session', - journal + journal, + writes: backgroundSettlementWrites(journal) }) ).toMatchObject({ ok: true }) const [lost] = rowsListing(journal, 'agent-a')[0]?.agents ?? [] diff --git a/src/main/native-chat/agent-session-journal/journal-context-controller.ts b/src/main/native-chat/agent-session-journal/journal-context-controller.ts index 7403760e8e91..b76514743ac7 100644 --- a/src/main/native-chat/agent-session-journal/journal-context-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-context-controller.ts @@ -7,6 +7,7 @@ import type { JournalReplacementItem } from './journal-epoch-replacement' import type { JournalOperationReceipt, JournalRowWriter } from './journal-row-writer' import type { JournalReducerState } from './journal-reducer' import type { JournalQueuedMessages } from './journal-queued-messages' +import type { JournalWriteOptions } from './journal-host-database' export class JournalContextController { constructor( @@ -35,8 +36,9 @@ export class JournalContextController { floor: AgentJournalCursor, fence: number, items: readonly JournalReplacementItem[], - receipt: (cursor: AgentJournalCursor) => JournalOperationReceipt + receipt: (cursor: AgentJournalCursor) => JournalOperationReceipt, + options?: JournalWriteOptions ) { - return rewindJournalContext({ ...this.deps, floor, fence, items, receipt }) + return rewindJournalContext({ ...this.deps, floor, fence, items, receipt, options }) } } diff --git a/src/main/native-chat/agent-session-journal/journal-context-rewind.ts b/src/main/native-chat/agent-session-journal/journal-context-rewind.ts index bcfa773cbcf7..e7051cb1ed91 100644 --- a/src/main/native-chat/agent-session-journal/journal-context-rewind.ts +++ b/src/main/native-chat/agent-session-journal/journal-context-rewind.ts @@ -11,6 +11,7 @@ import { buildJournalQueueClearRow } from './journal-stop-and-resume-rows' import { journalQueuePauseRestatement } from './queued-message-pause' import type { JournalQueuedMessages } from './journal-queued-messages' import type { JournalRow } from './journal-row-schema' +import type { JournalWriteOptions } from './journal-host-database' export async function rewindJournalContext(input: { state: () => JournalReducerState @@ -20,6 +21,7 @@ export async function rewindJournalContext(input: { fence: number items: readonly JournalReplacementItem[] receipt: (cursor: AgentJournalCursor) => JournalOperationReceipt + options?: JournalWriteOptions | undefined }): Promise { let receipt: JournalOperationReceipt | undefined const rows = await input.writer.enqueueRows( @@ -92,7 +94,8 @@ export async function rewindJournalContext(input: { { write: (db) => receipt!.write(db), committed: () => receipt!.committed() - } + }, + input.options ) const last = rows.at(-1)! return { epoch: last.epoch, sequence: last.seq } diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts index ba70b0a06721..7dc70fba06a8 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-controller.ts @@ -2,7 +2,7 @@ import type { AgentJournalCursor, AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import { replaceJournalEpoch, type JournalReplacementItem } from './journal-epoch-replacement' import type { JournalQueuePauseRestatement } from './queued-message-pause' import { publishNewEpoch } from './journal-epoch-rollover' @@ -64,7 +64,8 @@ export class JournalEpochController { replace( reason: AgentJournalEpochReason, fence: number, - items: readonly JournalReplacementItem[] + items: readonly JournalReplacementItem[], + options?: JournalWriteOptions ): Promise { return this.deps.serialize(() => { assertJournalWritable(this.deps.readOnly(), this.deps.identity.sessionId) @@ -81,7 +82,8 @@ export class JournalEpochController { onPublished: (loaded) => { this.deps.founding.settled() this.deps.adopt(loaded) - } + }, + options }) return this.deps.cursor() }) diff --git a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts index 17b9f5e3b956..2b1b8af334d1 100644 --- a/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts +++ b/src/main/native-chat/agent-session-journal/journal-epoch-replacement.ts @@ -11,7 +11,7 @@ import type { AgentSessionJournalIdentity } from '../../../shared/agent-session-journal-types' import { agentSessionJournalProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' -import type { JournalHostDatabase } from './journal-host-database' +import type { JournalHostDatabase, JournalWriteOptions } from './journal-host-database' import type { JournalLoad } from './journal-open' import { applyJournalRow, createJournalReducerState } from './journal-reducer' import { buildJournalItemRow, journalRowBase } from './journal-row-builders' @@ -56,6 +56,7 @@ export function replaceJournalEpoch(input: { mintEpoch: () => string /** Called the instant the transaction commits, before any fallible follow-up. */ onPublished: (loaded: JournalLoad) => void + options?: JournalWriteOptions }): void { const epoch = input.mintEpoch() const state = createJournalReducerState(input.identity.sessionId, epoch) @@ -116,7 +117,7 @@ export function replaceJournalEpoch(input: { insertJournalRow(db, sessionId, row) } publishJournalSessionEpoch(db, input.identity, epoch) - }) + }, input.options) // COMMIT landed: on disk the superseded rows are gone and this epoch is the // live one. The caller adopts that immediately, or a later failure leaves the diff --git a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts index a1a79d92cd28..2c31cf33a96b 100644 --- a/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts +++ b/src/main/native-chat/agent-session-journal/journal-lifecycle-batch-appender.ts @@ -20,7 +20,7 @@ import type { JournalResolvedLifecycleBatchInput } from './journal-store-contracts' import type { JournalRow } from './journal-row-schema' -import type { JournalWriteOptions } from './journal-database' +import { backgroundWrite, type JournalWriteOptions } from './journal-database' import { journalQueuedRejectionRowBuilders } from './journal-pending-submission-recovery' export class JournalLifecycleBatchAppender { @@ -68,19 +68,16 @@ export class JournalLifecycleBatchAppender { /** `append`, with every row chosen at the batch's own turn in the queue. */ appendPlanned(input: JournalPlannedLifecycleBatchInput): Promise { return this.deps - .enqueueRows( - () => { - if (this.wasApplied(input.settlementId)) { - return [] - } - const { mutations, dispatches } = input.plan() - return [ - ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), - ...this.planMutations(input, mutations) - ] - }, - input.background ? { background: true } : undefined - ) + .enqueueRows(() => { + if (this.wasApplied(input.settlementId)) { + return [] + } + const { mutations, dispatches } = input.plan() + return [ + ...dispatches.map((dispatch) => journalDispatchRowBuilder(this.deps.state, dispatch)), + ...this.planMutations(input, mutations) + ] + }, backgroundWrite(input.background)) .then((rows) => this.cursorAfter(rows)) } diff --git a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts index 4041e47aede3..ca8c8101dc26 100644 --- a/src/main/native-chat/agent-session-journal/journal-queued-messages.ts +++ b/src/main/native-chat/agent-session-journal/journal-queued-messages.ts @@ -330,15 +330,12 @@ export class JournalQueuedMessages { } /** Applies owed settlements now, so a skipped live transition heals without a reopen. */ - settleOwed(): Promise { + settleOwed(options?: JournalWriteOptions): Promise { + const { sessionId, state, now } = this.deps return this.transact( - (db) => - settleOwedQueuedMessages(db, { - sessionId: this.deps.sessionId, - state: this.deps.state(), - now: this.deps.now() - }), - (settled) => settled > 0 + (db) => settleOwedQueuedMessages(db, { sessionId, state: state(), now: now() }), + (settled) => settled > 0, + options ).then(() => undefined) } diff --git a/src/main/native-chat/agent-session-journal/journal-store.ts b/src/main/native-chat/agent-session-journal/journal-store.ts index 88d9ce836f4d..997592d201d4 100644 --- a/src/main/native-chat/agent-session-journal/journal-store.ts +++ b/src/main/native-chat/agent-session-journal/journal-store.ts @@ -390,9 +390,10 @@ export class AgentSessionJournal { replaceEpochItems( reason: AgentJournalEpochReason, fence: number, - items: readonly JournalReplacementItem[] + items: readonly JournalReplacementItem[], + options?: JournalWriteOptions ): Promise { - return this.epochController.replace(reason, fence, items) + return this.epochController.replace(reason, fence, items, options) } private adoptLoadedJournal(loaded: JournalLoad): void { diff --git a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts index 6a4abb6e2306..3d5b208f0277 100644 --- a/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts +++ b/src/main/native-chat/agent-session-journal/journal-unsent-send-hold.ts @@ -89,11 +89,11 @@ export async function holdUnsentSends( /** In place of the queued sends: those a child that ended before it answered its start was * handed and never echoed. It ran none, so each is unsent as surely as a queued one. */ unrun?: true - /** Bookkeeping no person waits on (`JournalWriteOptions`): contention stops it, untouched. */ - background?: true + /** What the rows are written through: the journal, or bookkeeping's background handle. */ + writes?: Pick } ): Promise { - const background = input.background ? { background: input.background } : {} + const writes = input.writes ?? journal const { hold } = input const unsent = journal .submissions() @@ -168,16 +168,14 @@ export async function holdUnsentSends( } try { // The send names its card in the same row, so no surface draws it once the card is gone. - await journal.resolveDispatch( - { - ...(card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject), - ...background - }, + await writes.resolveDispatch( + card ? { ...reject, keptAsQueuedMessageId: clientMessageId } : reject, keep ) } catch (error) { - // Another connection holds the database: nothing was written, so nothing is lost by waiting. - if (input.background && isSqliteContentionFailure(error)) { + // Another connection holds the database: nothing was written, and the fallback would meet + // the same lock, so the caller tries again. + if (isSqliteContentionFailure(error)) { throw error } if (!keep) { @@ -192,9 +190,7 @@ export async function holdUnsentSends( cause: hold.cause, error: error instanceof Error ? error.message : String(error) }) - await journal - .resolveDispatch({ ...reject, ...background }) - .catch((fallback: unknown) => failures.push(fallback)) + await writes.resolveDispatch(reject).catch((fallback: unknown) => failures.push(fallback)) } } if (failures.length > 0) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts new file mode 100644 index 000000000000..3a3926d5d8b2 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts @@ -0,0 +1,166 @@ +// Every background path meeting another connection's write lock: an exit's own settlement and +// release, the retry's recovery of a pending rewind, and the queue's automatic drain healing its +// bookkeeping. Each fails at once, never holding the main thread or the chat's lane for the busy +// timeout, and the retry lands the work once the lock lifts. A refusal of the automatic send whose +// stored hold also fails still shows the card as not sent. + +import { afterEach, describe, expect, it, vi } from 'vitest' +import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' +import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' +import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + exitChild, + leaveUnfinishedWork, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' +import { + holdWriteLock, + longestStall, + releaseWriteLocks +} from './structured-agent-session-write-lock.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await releaseWriteLocks() + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Longer than a background write would ever wait, shorter than the busy timeout a person's has. */ +const LOCK_MS = 1_200 + +describe('under another connection’s write lock', () => { + it("an exit's own settlement and release fail at once, and the retry lands both after", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + await holdWriteLock(current.root, LOCK_MS) + + expect(await longestStall(() => exitChild(current))).toBeLessThan(60) + expect(turnState(current)).toBe('running') + + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 8_000 }) + await vi.waitFor( + () => expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('released'), + { timeout: 8_000 } + ) + }, 20_000) + + it('a pending rewind the retry recovers ends its round at once, never holding the lane', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const { reconciliation, serialize } = current.host.collaboratorsForTests() + await current.store.transitionHandoff(SESSION, (record) => ({ + ...record, + rewind: { + operationId: 'rewind-op', + callerKey: 'client-1', + itemId: agentJournalItemKey({ provider: 'claude', sessionId: 'claude-s', uuid: 'kept' }), + expectedEpoch: 'epoch-x', + phase: 'prepared', + retained: [] + } + })) + await holdWriteLock(current.root, LOCK_MS) + + let laneWaited = Infinity + const stall = await longestStall(async () => { + reconciliation.signal(SESSION) + await new Promise((resolve) => setImmediate(resolve)) + const queuedAt = performance.now() + const lane = serialize(SESSION, async () => { + laneWaited = performance.now() - queuedAt + }) + await reconciliation.attempted(SESSION) + await lane + }) + expect(stall).toBeLessThan(60) + expect(laneWaited).toBeLessThan(60) + expect(current.store.getRecord(SESSION)?.rewind?.phase).toBe('prepared') + expect(retryOwes(reconciliation, SESSION)).toBe(true) + + // Claude rewind is unsupported: once the lock lifts, the retry settles it refused. + await vi.waitFor( + () => expect(current.store.getRecord(SESSION)?.rewind?.phase).toBe('refused'), + { timeout: 8_000 } + ) + }, 20_000) + + it("the drain's bookkeeping heal fails at once, and its send waits for the retry", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + await current.send('after this', 'queue-if-active').result + // The turn's end wakes the drain, which heals skipped bookkeeping before it sends; another + // connection takes the lock just as it starts. + const owed = vi.spyOn(JournalQueuedMessages.prototype, 'settlementOwed').mockReturnValue(true) + const heal = JournalQueuedMessages.prototype.settleOwed + let waited = Infinity + vi.spyOn(JournalQueuedMessages.prototype, 'settleOwed').mockImplementationOnce(async function ( + this: JournalQueuedMessages, + ...args + ) { + owed.mockRestore() + await holdWriteLock(current.root, LOCK_MS) + const started = performance.now() + try { + return await heal.apply(this, args) + } finally { + waited = performance.now() - started + } + }) + + await current.settleAccepted(working, 'a') + await vi.waitFor(() => expect(waited).toBeLessThan(Infinity)) + expect(waited).toBeLessThan(60) + expect(current.dispatch).toHaveBeenCalledTimes(1) + + await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2), { timeout: 8_000 }) + }, 20_000) +}) + +describe('a refused automatic send whose stored hold fails too', () => { + it('reads as not sent from memory, is not named next, and is not sent again', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + const working = await current.workingSend() + const queued = await current.send('after this', 'queue-if-active').result + if (!queued.ok || !('queued' in queued.value)) { + throw new Error('expected a queued receipt') + } + const diskFull = () => Object.assign(new Error('database or disk is full'), { errcode: 13 }) + const append = vi + .spyOn(AgentSessionJournal.prototype, 'appendSubmission') + .mockRejectedValueOnce(diskFull()) + const hold = vi.spyOn(JournalQueuedMessages.prototype, 'hold').mockRejectedValueOnce(diskFull()) + + await current.settleAccepted(working, 'a') + await vi.waitFor(() => expect(hold).toHaveBeenCalledTimes(1)) + await new Promise((resolve) => setTimeout(resolve, 300)) + + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages).toEqual([ + expect.objectContaining({ + messageId: queued.value.queued.messageId, + paused: true, + pausedReason: QUEUED_MESSAGE_PAUSED_SEND_FAILED + }) + ]) + expect(page.ok && page.page.nextQueuedMessageId).toBeNull() + expect(append).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts index 8ebcb5d0c5e6..ba6199c2920d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -3,10 +3,8 @@ // waits the lock out. One episode's give-up is never a latch: a commit or a signal re-arms it, and // a queued send given up on reads as not sent even when its stored hold cannot be written. -import { Worker } from 'node:worker_threads' import { afterEach, describe, expect, it, vi } from 'vitest' import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' -import { journalDatabasePath } from '../agent-session-journal/journal-host-database' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' import { AgentSessionJournal } from '../agent-session-journal/journal-store' @@ -23,56 +21,23 @@ import { eventually, type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' +import { + holdWriteLock, + longestStall, + releaseWriteLocks +} from './structured-agent-session-write-lock.test-fixture' let rig: QueuedMessageTestRig | undefined -let locker: Worker | undefined afterEach(async () => { vi.useRealTimers() - await locker?.terminate() - locker = undefined + await releaseWriteLocks() await rig?.dispose() rig = undefined vi.restoreAllMocks() }) -/** Another connection, on its own thread, holds the write lock for `ms`: this thread's waits - * cannot release it, as another process's could not. Resolves once the lock is held. */ -function holdWriteLock(stateDirectory: string, ms: number): Promise<{ released: Promise }> { - const worker = new Worker( - `const { workerData, parentPort } = require('node:worker_threads') - const { DatabaseSync } = require('node:sqlite') - const db = new DatabaseSync(workerData.file) - db.exec('BEGIN IMMEDIATE') - parentPort.postMessage('locked') - setTimeout(() => { db.exec('ROLLBACK'); db.close(); parentPort.postMessage('released') }, workerData.ms)`, - { eval: true, workerData: { file: journalDatabasePath(stateDirectory), ms } } - ) - locker = worker - let release: () => void = () => undefined - const released = new Promise((resolve) => (release = resolve)) - return new Promise((resolve) => - worker.on('message', (message) => (message === 'locked' ? resolve({ released }) : release())) - ) -} - -/** The longest the main thread went without running a timer, while `run` ran. */ -async function longestStall(run: () => Promise): Promise { - let last = performance.now() - let longest = 0 - const probe = setInterval(() => { - const now = performance.now() - longest = Math.max(longest, now - last) - last = now - }, 2) - try { - await run() - return Math.max(longest, performance.now() - last) - } finally { - clearInterval(probe) - } -} - const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) /** A chat whose exit left its settlement owed to the retry, with nothing refusing it now. */ @@ -98,7 +63,7 @@ describe("another connection's write lock", () => { }) expect(performance.now() - started).toBeLessThan(150) expect(stall).toBeLessThan(60) - expect(reconciliation.owes(SESSION)).toBe(true) + expect(retryOwes(reconciliation, SESSION)).toBe(true) expect(turnState(current)).toBe('running') // A person's send waits the lock out (its own 5 s budget), never behind background work. @@ -195,7 +160,7 @@ describe('an episode the retry gives up', () => { await settleTurns() // Ten rounds, then no timer: the set is kept, and nothing retries on its own. expect(attempts.mock.calls.length).toBe(given) - expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(true) + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(true) attempts.mockRestore() return attempts } @@ -220,7 +185,7 @@ describe('an episode the retry gives up', () => { await settleTurns() expect(turnState(current)).toBe('interrupted') - expect(reconciliation.owes(SESSION)).toBe(false) + expect(retryOwes(reconciliation, SESSION)).toBe(false) } ) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts new file mode 100644 index 000000000000..60b38a13740e --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts @@ -0,0 +1,82 @@ +// What bookkeeping no person waits on may write: the retry's visit, the queue's automatic drain, +// and an exit's own settlement and release. Each handle carries the writes such a path makes, +// every one passing `{ background: true }` (`JournalWriteOptions`), so another connection's lock +// fails it at once instead of holding the main thread for the busy timeout. The paths read through +// types without these writes, so none can reach a write that waits. + +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { JournalContextController } from '../agent-session-journal/journal-context-controller' +import type { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' + +const BACKGROUND = { background: true } as const + +type Background = { readonly background: true } + +/** A settlement's one write (`settleStructuredAgentSessionLeftovers`). */ +export type BackgroundSettlementWrites = Pick & + Background + +export type BackgroundJournalWrites = BackgroundSettlementWrites & + Pick< + AgentSessionJournal, + 'resolveDispatch' | 'appendSubmission' | 'markQueueReopen' | 'replaceEpochItems' + > & { + queuedMessages: Pick + context: Pick + } + +/** A lease release's one write. */ +export type BackgroundLeaseWrites = Pick & Background + +export type BackgroundStoreWrites = BackgroundLeaseWrites & + Pick + +export function backgroundSettlementWrites( + journal: Pick +): BackgroundSettlementWrites { + return { + ...BACKGROUND, + appendPlannedLifecycleBatch: (input) => + journal.appendPlannedLifecycleBatch({ ...input, ...BACKGROUND }) + } +} + +export function backgroundJournalWrites(journal: AgentSessionJournal): BackgroundJournalWrites { + const { queuedMessages: queued, context } = journal + return { + ...backgroundSettlementWrites(journal), + resolveDispatch: (input, hook) => journal.resolveDispatch({ ...input, ...BACKGROUND }, hook), + appendSubmission: (input, consume, receipt) => + journal.appendSubmission(input, consume && { ...consume, ...BACKGROUND }, receipt), + markQueueReopen: (fence, since) => journal.markQueueReopen(fence, since, BACKGROUND), + replaceEpochItems: (reason, fence, items) => + journal.replaceEpochItems(reason, fence, items, BACKGROUND), + queuedMessages: { + hold: (input) => queued.hold({ ...input, ...BACKGROUND }), + repairAndPrune: () => queued.repairAndPrune(BACKGROUND), + settleOwed: () => queued.settleOwed(BACKGROUND) + }, + context: { + rewind: (floor, fence, items, receipt) => + context.rewind(floor, fence, items, receipt, BACKGROUND) + } + } +} + +export function backgroundLeaseWrites( + store: Pick +): BackgroundLeaseWrites { + return { + ...BACKGROUND, + transitionHandoff: (sessionId, transition) => + store.transitionHandoff(sessionId, transition, BACKGROUND) + } +} + +export function backgroundStoreWrites(store: AgentSessionRecordStore): BackgroundStoreWrites { + return { + ...backgroundLeaseWrites(store), + recordOperationOutcome: (args) => store.recordOperationOutcome(args, BACKGROUND) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts index 34ef73dea479..158c6c0b048c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-close.ts @@ -15,7 +15,7 @@ import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { stopAgentSessionProviderRoot } from './structured-agent-session-provider-exit-proof' -import type { JournalWriteOptions } from '../agent-session-journal/journal-database' +import type { BackgroundLeaseWrites } from './structured-agent-session-background-writes' import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { releaseStoredStructuredAgentSessionOwnerAfterExit } from './structured-agent-session-lease-release' import { isSurfaceReleasableAgentSessionRecord } from '../../runtime/agent-session-surface-release-transition' @@ -124,12 +124,13 @@ export function structuredAgentSessionEndedChildHoldsLease( } /** Writes the release a proven exit allows when the exit handler could not: a start and the - * handle's close re-derive it, and a failure is reported, never anyone's refusal. Resolves - * whether the lease still names that child. */ + * handle's close re-derive it, and a failure is reported, never anyone's refusal. The retry + * writes it through its background handle, where another connection's lock throws for its next + * round. Resolves whether the lease still names that child. */ export async function releaseLeaseOfEndedStructuredAgentSessionChild( context: Pick, sessionId: string, - options?: JournalWriteOptions + background?: BackgroundLeaseWrites ): Promise { const ended = context.sessions.get(sessionId)?.lastEndedChild if (!ended || !structuredAgentSessionEndedChildHoldsLease(context, sessionId)) { @@ -137,14 +138,13 @@ export async function releaseLeaseOfEndedStructuredAgentSessionChild( } await releaseStoredStructuredAgentSessionOwnerAfterExit({ store: context.deps.store, + writes: background ?? context.deps.store, sessionId, expectedFence: ended.fence, now: context.now(), - ...(ended.reason ? { exitReason: ended.reason } : {}), - ...options + ...(ended.reason ? { exitReason: ended.reason } : {}) }).catch((error: unknown) => { - // Background work stops on contention, for its retry; anything else is only reported. - if (options?.background && isSqliteContentionFailure(error)) { + if (background && isSqliteContentionFailure(error)) { throw error } context.deps.logger.warn("releasing an exited agent's lease failed", { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 808703a120bc..16774b3e0950 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -15,6 +15,10 @@ import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { recordProviderChildEnd } from './structured-agent-session-provider-child' +import { + backgroundLeaseWrites, + backgroundSettlementWrites +} from './structured-agent-session-background-writes' import { releaseStoredStructuredAgentSessionOwnerAfterExit, type StructuredAgentSessionLeaseStore @@ -53,8 +57,8 @@ export type StructuredAgentSessionChildExitSession = Pick< 'child' | 'lastEndedChild' > & { journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal } -/** The chat's reconciliation worker's signal (`StructuredAgentSessionReconciliation.signal`): the - * one edge every way a generation ends goes through. `exit`: the exit's own settlement, owed when +/** The host retry's signal (`StructuredAgentSessionRetry.signal`): the one edge every way a + * generation ends goes through. `exit`: the exit's own settlement, owed when * it did not land. */ export type StructuredAgentSessionGenerationEnded = ( sessionId: string, @@ -69,8 +73,7 @@ export type StructuredAgentSessionChildExitContext< flushLifecycle: (sessionId: string) => Promise publishStatus?: (sessionId: string) => void /** A generation ended: every reader re-derives current work, the queued-card drain is - * scheduled with no journal write needed, and the chat's reconciliation worker settles what is - * owed. `restate`: each chat re-baselines at the moved fence. */ + * scheduled with no journal write needed, and the host's retry settles what is owed. `restate`: each chat re-baselines at the moved fence. */ generationEnded: StructuredAgentSessionGenerationEnded /** The delivery loop hands over whatever is queued once the child is off the record. */ wakeDelivery?: (sessionId: string) => void @@ -246,9 +249,10 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< } } finally { // What the exited child left is settled first, at its own fence, as its own closing tail lands: - // a row its closed sink still delivers lands too, and wakes the chat's worker to settle it once - // the release below moves the fence. A settlement or a release that cannot commit stays owed: - // the chat's reconciliation worker retries it. + // a row its closed sink still delivers lands too, and wakes the retry to settle it once the + // release below moves the fence. Both are bookkeeping (`structured-agent-session-background- + // writes.ts`): another connection's lock fails them at once, and what did not commit stays owed + // to the retry, the settlement as the exit's debt and the release as its repair. let released = false let owed: StructuredAgentSessionExitSettlement | null = null if (settlement) { @@ -256,6 +260,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< store: context.store, sessionId, journal: session.journal, + writes: backgroundSettlementWrites(session.journal), exit: settlement, // Ended whether or not the release lands: the host saw the root go. ended: { fence: child.fence, rootGone: true } @@ -268,6 +273,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< try { await releaseStoredStructuredAgentSessionOwnerAfterExit({ store: context.store, + writes: backgroundLeaseWrites(context.store), sessionId, expectedFence: child.fence, now: context.now(), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts index af06ae1ff2e5..3c5ca8cd311a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-claude-stop-exit-ends-record.test.ts @@ -397,7 +397,7 @@ it('sends after a proven exit whose resume-point write and lease release both fa expect.arrayContaining(['claude-close-resume-point', 'exit-owner-release']) ) ) - // The chat's worker writes the release from this host's proof of that exit; no send waits on it. + // The host's retry writes the release from this host's proof of that exit; no send waits on it. await eventually(() => expect(lease()?.claimStatus).toBe('released')) await send('Carry on.') @@ -419,7 +419,7 @@ it('sends after a crash with a long reason whose own lease release failed', asyn ) await eventually(() => expect(child()).toBeNull()) await eventually(() => expect(scopes()).toContain('exit-owner-release')) - // The chat's worker writes the release the exit's own write could not. + // The host's retry writes the release the exit's own write could not. await eventually(() => expect(lease()?.claimStatus).toBe('released')) await send('Carry on.') @@ -527,7 +527,7 @@ it('has the idle reaper join a close still unverifiable, ending the record with expect(connection.closeCount).toBe(2) expect(lease()).toMatchObject({ claimStatus: 'released', ownerProcess: null }) - // Closed once the chat's worker has settled what the ended child left. + // Closed once the host's retry has settled what the ended child left. await eventually(() => expect(host.hasSession(SESSION)).toBe(false)) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts index 98b097b72fec..a83cf093eb88 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-crash-turn-end.test.ts @@ -50,6 +50,7 @@ import { createStructuredAgentSessionLogger } from './structured-agent-session-l import { recordingStructuredAgentSessionLogger } from './structured-agent-session-logger-test-support' import { claudeProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const PROVIDER_SESSION = 'provider-session-alpha-1' /** The tool call's row: the last thing the provider wrote before the crash. */ @@ -304,9 +305,9 @@ async function drainSession(): Promise { await reconciled() } -/** Once the chat's reconciliation worker settled what a generation's end left. */ +/** Once the host's retry settled what a generation's end left. */ function reconciled(): Promise { - return host.collaboratorsForTests().reconciliation.idle(SESSION) + return retryIdle(host.collaboratorsForTests().reconciliation, SESSION) } /** Startup: the reconcile, then the scan it begins. */ @@ -514,7 +515,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () ]) }) - it('stays as it was when the startup settlement cannot be written, and the worker retries it', async () => { + it('stays as it was when the startup settlement cannot be written, and the retry retries it', async () => { const log = recordingStructuredAgentSessionLogger() openHost({ probeOwner: async () => ({ outcome: 'pid-absent' }), logger: log.logger }) await host.history({ sessionId: SESSION, direction: 'tail' }) @@ -528,7 +529,7 @@ describe('a turn a read reached before the reconcile proved its owner dead', () expect(new Set(log.scopes())).toEqual(new Set(['reconciliation'])) expect(await settledTurn()).toEqual(RUNNING_TURN) full.mockRestore() - // Nothing stored says it is owed: the worker re-derives it after its backoff, from the proof + // Nothing stored says it is owed: the retry re-derives it after its backoff, from the proof // the record holds. await vi.waitFor( async () => @@ -601,7 +602,7 @@ const PROVEN_TURN = { completedAt: LAST_RENEWED_AT } -// The relaunch proved the fence-13 owner gone, and the chat's worker settles the turn by that proof, +// The relaunch proved the fence-13 owner gone, and the host's retry settles the turn by that proof, // held across the starts that clear it from the record; what the record holds after a start // reserving fence 15 fails is about the start's own child. describe('a turn a newer start could not settle before it failed', () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts index d27d29e64128..1554fac43f9b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts @@ -36,6 +36,7 @@ import { sendText, turnState } from './structured-agent-session-leftover-settlement.test-fixture' +import { retryIdle } from './structured-agent-session-retry.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -237,7 +238,7 @@ describe('an observed exit whose release write failed', () => { vi.spyOn(current.store, 'transitionHandoff').mockImplementation((sessionId, change) => transition(sessionId, (record) => { const next = change(record) - // The exit's own release, and the chat worker's first repair of it. + // The exit's own release, and the retry's first repair of it. if (refused < 2 && next.lease.claimStatus === 'released') { refused += 1 throw new Error('disk full') @@ -259,7 +260,7 @@ describe('an observed exit whose release write failed', () => { acquisitionGeneration: child.generation }) await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) - await current.host.collaboratorsForTests().reconciliation.idle(SESSION) + await retryIdle(current.host.collaboratorsForTests().reconciliation, SESSION) expect(refused).toBe(2) // The release never landed: the lease still names the gone child. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts index 0d617782cdd1..373ca6a3da37 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts @@ -26,6 +26,7 @@ import { } from './structured-agent-session-queued-message-rig.test-fixture' import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' import { isResumableStructuredAgentSessionRecord } from './structured-agent-session-resume-eligibility' +import { retryOwes } from './structured-agent-session-retry.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -131,9 +132,12 @@ describe('an exit that settles and releases cleanly', () => { await current.host.collaboratorsForTests().serialize(SESSION, async () => {}) await eventually(async () => expect(await current.handoff(draftId)).toBeDefined()) - // Inside one turn of the chat's lane: no retry round, no backoff. + // Inside one turn of the chat's lane, no backoff; the retry's visit once the lane frees finds + // nothing left. expect(Date.now() - started).toBeLessThan(250) - expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(false) + await eventually(() => + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(false) + ) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts index b96313dbcb38..93d3a3c4fc8a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts @@ -225,7 +225,7 @@ describe('the idle sweep', () => { await vi.waitFor(() => expect(rig.adapter.closeSession).toHaveBeenCalledWith(SESSION)) }) - it('never stops a worker whose orchestration dispatch is open, and stops it once it settles (P2-19 i)', async () => { + it('never stops the retry whose orchestration dispatch is open, and stops it once it settles (P2-19 i)', async () => { await rig.dispose() let open = true const hasOpenDispatch = vi.fn(() => open) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts index e752f02ed840..99bd45df8406 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-lease-release.ts @@ -17,17 +17,20 @@ export type StructuredAgentSessionLeaseStore = Pick< 'getRecord' | 'transitionHandoff' > +/** What the release writes through: the store itself, or bookkeeping's background handle + * (`BackgroundLeaseWrites`). */ +export type StructuredAgentSessionLeaseWrites = Pick + /** Releases the lease of the child whose exit this host observed, with that exit's evidence. * Throws `agent_session_checkpoint_stale` when the record no longer names that child. */ export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { - store: StructuredAgentSessionLeaseStore + store: Pick + writes: StructuredAgentSessionLeaseWrites sessionId: string expectedFence: number now: number exitObservedAt?: number exitReason?: string - /** Bookkeeping no person waits on (`JournalWriteOptions`). */ - background?: true }): Promise { const record = input.store.getRecord(input.sessionId) if ( @@ -37,15 +40,11 @@ export async function releaseStoredStructuredAgentSessionOwnerAfterExit(input: { ) { throw new Error('agent_session_checkpoint_stale') } - return releaseStoredAgentSessionOwnerAfterSurfaceClose( - input.store, - { - sessionId: input.sessionId, - expectedFence: input.expectedFence, - now: input.now, - ...(input.exitObservedAt === undefined ? {} : { exitObservedAt: input.exitObservedAt }), - ...(input.exitReason ? { exitReason: input.exitReason } : {}) - }, - input.background ? { background: true } : undefined - ) + return releaseStoredAgentSessionOwnerAfterSurfaceClose(input.writes, { + sessionId: input.sessionId, + expectedFence: input.expectedFence, + now: input.now, + ...(input.exitObservedAt === undefined ? {} : { exitObservedAt: input.exitObservedAt }), + ...(input.exitReason ? { exitReason: input.exitReason } : {}) + }) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts index f95c2b346e9e..4d5ab34f4cdb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -23,6 +23,7 @@ import { sendText, turnState } from './structured-agent-session-leftover-settlement.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -56,7 +57,7 @@ describe('reading a chat', () => { // Answered, so no send left pending keeps the conversation open. await current.settleAccepted(await current.workingSend(), 'work') await leaveUnfinishedWork(current, { prompt: true }) - // The storage fault stays: the worker keeps retrying in the background, and every attempt + // The storage fault stays: the retry keeps retrying in the background, and every attempt // rolls back, so any write the readers made would be the only one logged. const { database } = await exitWhileSettlementFails(current) await current.host.close(SESSION, 'evict') @@ -74,8 +75,8 @@ describe('reading a chat', () => { unsubscribe() expect(writes()).toEqual([]) - // Still owed, and still as the dead generation left it: only the chat's worker settles it. - expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(true) + // Still owed, and still as the dead generation left it: only the host's retry settles it. + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(true) expect(turnState(current)).toBe('running') expect(holds(current)).toEqual({ working: false, queueHeld: false }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts index 138fb3cb1377..8893748c0f48 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -48,6 +48,7 @@ import { turnState, watchSettlementCommits } from './structured-agent-session-leftover-settlement.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -58,7 +59,7 @@ afterEach(async () => { }) describe('an observed exit whose settlement the database refused', () => { - it('is settled by the worker in one commit, while the next send starts and delivers', async () => { + it('is settled by the retry in one commit, while the next send starts and delivers', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig const firstSend = await current.workingSend() @@ -119,7 +120,7 @@ describe('an observed exit whose settlement the database refused', () => { expect(turnState(current)).toBe('interrupted') expect(commits()).toHaveLength(1) const settled = recoveredRows(database.db) - // The next start finds nothing left: the worker its end signals writes no row. + // The next start finds nothing left: the retry its end signals writes no row. expect((await sendText(current, 'next').result).ok).toBe(true) await vi.waitFor(() => expect(current.dispatch).toHaveBeenCalledTimes(2)) await reconciled(current) @@ -219,7 +220,7 @@ describe('the retry', () => { const { database } = await exitWhileSettlementFails(current) const reconciliation = current.host.collaboratorsForTests().reconciliation const attempts = vi.spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') - expect(reconciliation.owes(SESSION)).toBe(true) + expect(retryOwes(reconciliation, SESSION)).toBe(true) await vi.advanceTimersByTimeAsync(990) await settleTurns() @@ -246,7 +247,7 @@ describe('the retry', () => { expect(attempts).toHaveBeenCalledTimes(3) await vi.advanceTimersByTimeAsync(10) await settleTurns() - expect(reconciliation.owes(SESSION)).toBe(false) + expect(retryOwes(reconciliation, SESSION)).toBe(false) expect(turnState(current)).toBe('interrupted') } finally { vi.useRealTimers() @@ -314,7 +315,7 @@ describe('a stale prompt', () => { }) describe('a late write from an ended generation', () => { - it('is settled by the worker its commit wakes', async () => { + it('is settled by the retry its commit wakes', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig // Answered, so the exit leaves nothing to settle and writes no row at the moved fence. @@ -359,7 +360,7 @@ describe('startup', () => { await current.crashReloadHostProcess() await reconciled(current) const database = openTestJournalHostDatabase(current.root) - // The worker opened the chat itself, so it closes it again: nothing else had it open. + // The retry opened the chat itself, so it closes it again: nothing else had it open. await vi.waitFor(() => expect(current.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts index f24c78e8af4f..300958ee3b5e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.test-fixture.ts @@ -33,6 +33,7 @@ import { QUEUED_RIG_CALLER as CALLER, type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' +import { retryIdle, retryOwes } from './structured-agent-session-retry.test-fixture' export const TURN: AgentJournalItemIdentity = { provider: 'codex', @@ -128,8 +129,8 @@ export async function leaveUnfinishedWork( } /** The child dies on its own while the database refuses every settlement row: the exit releases - * its lease (proof written, fence moved) and its work stays saved as running. The chat's - * reconciliation worker is left backing off. */ + * its lease (proof written, fence moved) and its work stays saved as running. The host's + * retry is left backing off. */ export async function exitWhileSettlementFails(current: QueuedMessageTestRig) { const context = current.host.collaboratorsForTests() const session = context.sessions.get(SESSION) @@ -150,6 +151,8 @@ export async function exitWhileSettlementFails(current: QueuedMessageTestRig) { acquisitionGeneration: child.generation }) await context.serialize(SESSION, async () => {}) + // The retry's first visit waited for the exit's lane; it fails too, and backs off. + await context.reconciliation.attempted(SESSION) expect(session.child).toBeNull() expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ claimStatus: 'released', @@ -249,10 +252,11 @@ export function watchSettlementCommits(): () => JournalRow[][] { ) } -/** Once the chat's reconciliation worker has nothing owed: it retired. */ +/** Once the host's retry owes the chat nothing. */ export function reconciled(current: QueuedMessageTestRig, timeout = 10_000): Promise { return vi.waitFor( - () => expect(current.host.collaboratorsForTests().reconciliation.owes(SESSION)).toBe(false), + () => + expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(false), { timeout } ) } @@ -278,7 +282,7 @@ export function logEveryWrite(db: Database.Database): () => string[] { } /** Startup as the runtime runs it, on any host: the restart reconcile, then the scan of every chat - * it begins, with no stored mark of what is owed, until the chat's worker is idle. */ + * it begins, with no stored mark of what is owed, until the host's retry is idle. */ export async function startUpHost( host: Pick< StructuredAgentSessionHost, @@ -287,11 +291,11 @@ export async function startUpHost( ): Promise { await host.reconcileRestartLeases() await host.startupSettled() - await host.collaboratorsForTests().reconciliation.idle(SESSION) + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) } /** Startup as the runtime runs it: the restart reconcile, then the scan of every chat it begins, - * until the chat's worker has nothing owed. */ + * until the host's retry owes it nothing. */ export async function startUp( current: QueuedMessageTestRig, host: Pick = current.host diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts index ed84e30119c2..1965da4327bc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement.ts @@ -1,6 +1,7 @@ // The one settlement of what ended generations left unfinished. An observed exit runs it first, -// with what it saw; the chat's reconciliation worker runs it after every generation end and at -// startup (`structured-agent-session-reconciliation-worker.ts`). Opening a chat never runs it. +// with what it saw; the host's retry runs it after every generation end and at startup +// (`structured-agent-session-reconciliation-retry.ts`). Opening a chat never runs it. Bookkeeping +// no person waits on, so it writes only through a background handle (`BackgroundSettlementWrites`). // // Re-derived each time from the journal, the lease and this host's sight of an exit: every turn, // call, prompt, reasoning row, subagent roster entry and background task whose execution belongs @@ -22,6 +23,7 @@ import { type StaleStructuredAgentSessionStateJournal } from './structured-agent-session-stale-state-settlement' import type { DeadGenerationJournal } from './structured-agent-session-unfinished-work' +import type { BackgroundSettlementWrites } from './structured-agent-session-background-writes' import { STALE_SESSION_ROW_PREFIX } from '../../../shared/agent-session-stop-row-identity' import { structuredAgentSessionFailureWordsContext } from './structured-agent-session-send-preparation' import { @@ -40,7 +42,9 @@ export type StructuredAgentSessionExitSettlement = Omit< export type StructuredAgentSessionLeftoverSettlementInput = { store: StructuredAgentSessionLeftoverStore sessionId: string + /** Read for the plan; the settlement's write goes through `writes`. */ journal: DeadGenerationJournal & StaleStructuredAgentSessionStateJournal + writes: BackgroundSettlementWrites exit?: StructuredAgentSessionExitSettlement /** This host's sight of a child's root exit, which ends its generation even when the release * write failed (`StructuredAgentSessionHostSession.lastEndedChild`). */ @@ -49,8 +53,6 @@ export type StructuredAgentSessionLeftoverSettlementInput = { * proven-dead owner clears it, and a later release writes its own): only what that generation * and earlier ones left is settled, judged by it. Absent: the lease's own evidence judges. */ proof?: AgentSessionDeathEvidence & { ownerFence: number } - /** Bookkeeping no person waits on (`JournalWriteOptions`). */ - background?: true } export type StructuredAgentSessionLeftoverSettlement = @@ -83,13 +85,12 @@ export async function settleStructuredAgentSessionLeftovers( : (liveFence ?? fence + 1) const failureTextContext = structuredAgentSessionFailureWordsContext(record) let planned = 0 - await journal.appendPlannedLifecycleBatch({ + await input.writes.appendPlannedLifecycleBatch({ settlementId: exit ? `dead-generation:${exit.settlementId}` : `${STALE_SESSION_ROW_PREFIX}${sessionId}:${fence}:${proof ? `proof-${proof.ownerFence}:` : ''}seq-${journal.cursor().sequence}`, fence, recovered: true, - ...(input.background ? { background: true } : {}), plan: () => { const exited = exit ? planStructuredAgentSessionDeadGeneration( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts index 350e80f83072..194c219cf7dc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child-record.test.ts @@ -44,6 +44,7 @@ import { openTestJournalHostDatabase } from '../agent-session-journal/journal-ho import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const CALLER = { callerKey: 'client-1' } const CHAT_CLOSED = agentSessionFailureWords(agentSessionFailureFact('chatClosed'), { @@ -130,9 +131,9 @@ beforeEach(async () => { startHost() expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) await host.close(SESSION, 'evict') - // The stop's release wakes the chat's worker, which opens the chat and closes it again; the + // The stop's release wakes the host's retry, which opens the chat and closes it again; the // tests below write its journal directly, so it must be shut first. - await host.collaboratorsForTests().reconciliation.idle(SESSION) + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) }) @@ -368,7 +369,7 @@ describe('settling an earlier child before the next one takes its message', () = await restartHost() await host.reconcileRestartLeases() await host.startupSettled() - await host.collaboratorsForTests().reconciliation.idle(SESSION) + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) const id = await accept('for the next child') await eventually(async () => expect((await submission(id))?.dispatchState).toBe('accepted')) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts index 8aecff8b6b01..b7197d8379c2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-abandonment.ts @@ -1,14 +1,21 @@ -// A queued card whose automatic send the retry gave up on (`StructuredAgentSessionRetry`): it reads -// as not sent at once, from memory, while its stored hold is tried once. Memory only: a restart's +// A queued card whose automatic send failed for good: the retry gave up on it +// (`StructuredAgentSessionRetry`), or the journal refused it. It reads as not sent at once, from +// memory, while its stored hold is tried once, in the background. Memory only: a restart's // earlier-process settlement converts the card. import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' -import type { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { backgroundJournalWrites } from './structured-agent-session-background-writes' export class QueuedSendAbandonment { /** By chat, until a person moves the card, its hold lands or a later attempt sends it. */ private readonly cards = new Map() + /** `publish`: re-sends what clients read, so the card shows as not sent and is not named next. */ + constructor( + private readonly publish: (sessionId: string, journal: AgentSessionJournal) => void + ) {} + card(sessionId: string): string | undefined { return this.cards.get(sessionId) } @@ -18,19 +25,16 @@ export class QueuedSendAbandonment { } /** Marks the card, then tries its stored hold once; should that fail too, the mark still shows. */ - mark(sessionId: string, messageId: string, queued: Pick): void { + mark(sessionId: string, messageId: string, journal: AgentSessionJournal): void { this.cards.set(sessionId, messageId) - void queued - .hold({ - messageIds: [messageId], - reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED, - background: true - }) + void backgroundJournalWrites(journal) + .queuedMessages.hold({ messageIds: [messageId], reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED }) .then(() => { if (this.cards.get(sessionId) === messageId) { this.cards.delete(sessionId) } }) .catch(() => undefined) + this.publish(sessionId, journal) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts index 0a8fd8434606..d8fc1a359cbc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-message-rig.test-fixture.ts @@ -40,6 +40,7 @@ import { type QueuedRigProviderOptions } from './structured-agent-session-queued-message-rig-provider.test-fixture' import { claudeAndCodexDeclared } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' export const QUEUED_RIG_CALLER = { callerKey: 'client-1' } type RigSendOptions = { internal?: true; from?: AgentMessageSource } @@ -234,12 +235,12 @@ export async function createQueuedMessageTestRig( /** A host process that dies with no close: a new process loads the same state directory, its * records with it, and runs its startup as the runtime does before any client reaches it: the * restart reconcile and its settlement, then the restore of the chat's tab, which decides a - * lease latched in recovery; that decision's release wakes the chat's worker once more. */ + * lease latched in recovery; that decision's release wakes the retry once more. */ async function crashRestartHostProcess(beforeStartup?: () => unknown): Promise { await crashReloadHostProcess(beforeStartup) await host.restoreReadableSessions([SESSION]) await host.startupSettled() - await host.collaboratorsForTests().reconciliation.idle(SESSION) + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) } /** The same crash, with only the restart reconcile and its settlement: no tab is restored. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 8a0719479e85..25ceb7c50e7a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -8,10 +8,9 @@ import { randomUUID } from 'node:crypto' import type { AgentJournalMessageItem } from '../../../shared/agent-session-journal-types' -import { - QUEUED_MESSAGE_PAUSED_SEND_FAILED, - type AgentSessionSendResult, - type AgentSessionWireRefusal +import type { + AgentSessionSendResult, + AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { createStructuredAgentSessionOperationId } from '../../../shared/structured-agent-session-mutation' import { agentSessionSendBodyFingerprint } from '../../../shared/structured-agent-session-send-mutation' @@ -31,6 +30,7 @@ import type { StructuredAgentSessionHostSession } from './structured-agent-sessi import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' import { QueuedSendAbandonment } from './structured-agent-session-queued-abandonment' +import { backgroundJournalWrites } from './structured-agent-session-background-writes' import { structuredAgentSessionHostInstance, structuredQueuePauses @@ -257,7 +257,9 @@ export type QueuedMessageDrainDeps = { export class StructuredAgentSessionQueuedMessageDrain { private readonly scheduled = new Set() private disposed = false - private readonly abandoned = new QueuedSendAbandonment() + private readonly abandoned = new QueuedSendAbandonment((sessionId, journal) => + this.deps.publish(sessionId, journal) + ) constructor(private readonly deps: QueuedMessageDrainDeps) {} @@ -281,16 +283,13 @@ export class StructuredAgentSessionQueuedMessageDrain { this.abandoned.clear(sessionId) } - /** The retry gave up on the chat's automatic send: its card reads as not sent at once, and the - * stored hold is tried once; should that fail too, this mark still shows it. */ + /** The retry gave up on the chat's automatic send. */ abandon(sessionId: string): void { const journal = this.deps.sessions.get(sessionId)?.journal const next = journal ? oldestActionableQueuedMessage(journal) : null - if (!journal || !next) { - return + if (journal && next) { + this.abandoned.mark(sessionId, next.messageId, journal) } - this.abandoned.mark(sessionId, next.messageId, journal.queuedMessages) - this.deps.publish(sessionId, journal) } schedule(sessionId: string): void { @@ -322,21 +321,22 @@ export class StructuredAgentSessionQueuedMessageDrain { this.scheduled.delete(sessionId) return this.step(sessionId, false) }) + // Another connection holds the database: the retry sends it in its next round, nothing shown. + .then((outcome) => outcome === 'contended' && this.deps.retry().sendContended(sessionId)) .catch((error: unknown) => { this.scheduled.delete(sessionId) this.warn(sessionId, error) }) } - /** The retry's own send of the chat's next card (`StructuredAgentSessionRetry`): whether another - * connection's lock refused it, which ends the retry's round. */ + /** The retry's own send of the chat's next card (`StructuredAgentSessionRetry`), in the chat's + * lane, which its visit already holds: whether another connection's lock refused it, which ends + * the retry's round. */ sendForRetry(sessionId: string): Promise<'contended' | 'done'> { - return this.deps - .serialize(sessionId, () => this.step(sessionId, true)) - .catch((error: unknown) => { - this.warn(sessionId, error) - return 'done' as const - }) + return this.step(sessionId, true).catch((error: unknown) => { + this.warn(sessionId, error) + return 'done' as const + }) } private warn(sessionId: string, error: unknown): void { @@ -354,28 +354,37 @@ export class StructuredAgentSessionQueuedMessageDrain { return 'done' } const journal = session.journal + // The automatic send is bookkeeping no person waits on: it writes only through this handle. + const writes = backgroundJournalWrites(journal) if (journal.queuedMessages.settlementOwed() || journal.queuedMessages.deliveredByEchoOwed()) { // A live per-row hook was skipped; heal now, before a draft sends, rather than at reopen. - await journal.queuedMessages.settleOwed().catch((error: unknown) => { + try { + await writes.queuedMessages.settleOwed() + } catch (error) { + if (isSqliteContentionFailure(error)) { + return 'contended' + } this.deps.logger.warn('settling owed queued-message bookkeeping failed', { scope: 'queued-settle-owed', sessionId, error }) - }) + } } const fence = this.deps.conversationFence(sessionId) // Whatever clears a hold publishes or commits, which re-derives this step. const record = this.deps.getRecord(sessionId) const work = this.deps.currentWork(sessionId) - const next = work ? nextStructuredQueuedMessage({ journal, record, work }) : null - if (this.disposed || !next || (!forRetry && this.deps.retry().sendWaits(sessionId))) { + // The retry's own attempt sends what the drain alone waits on (`waits`). + const drainWaits = (messageId: string) => !forRetry && this.waits(sessionId, messageId) + const next = work ? nextStructuredQueuedMessage({ journal, record, work, drainWaits }) : null + if (this.disposed || !next) { return 'done' } // Always a fresh id: the submission names its draft by `queuedMessageId`, never by id equality. const submissionId = createStructuredAgentSessionOperationId(randomUUID) try { - await journal.appendSubmission( + await writes.appendSubmission( { clientMessageId: submissionId, // The queue's own automatic send, never kept as a card by a restart or a close. @@ -390,8 +399,7 @@ export class StructuredAgentSessionQueuedMessageDrain { expect: 'waiting', settledByOp: null, hostInstance: structuredAgentSessionHostInstance(), - yieldsToPause: true, - background: true + yieldsToPause: true } ) } catch (error) { @@ -399,19 +407,12 @@ export class StructuredAgentSessionQueuedMessageDrain { // Lost a race with a Send-now, a Delete or a Stop; their transition stands. return 'done' } - // Another connection holds the database: the retry sends it in its next round, nothing shown. if (isSqliteContentionFailure(error)) { - if (!forRetry) { - this.deps.retry().sendContended(sessionId) - } return 'contended' } - // A refusal: the draft stays waiting, held with the marker on the card (a stored fact, so it - // survives eviction and restart). The hold's own commit notification publishes it. An - // explicit Send retries. - await journal.queuedMessages - .hold({ messageIds: [next.messageId], reason: QUEUED_MESSAGE_PAUSED_SEND_FAILED }) - .catch(() => {}) + // A refusal: the draft stays waiting, shown as not sent and held (a stored fact once its hold + // lands, so it survives eviction and restart). An explicit Send retries. + this.abandoned.mark(sessionId, next.messageId, journal) throw error } this.abandoned.clear(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts index a6df16b8efae..06f210bc4d6d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-pause.ts @@ -6,7 +6,6 @@ import { randomUUID } from 'node:crypto' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { DerivedQueuePause } from '../agent-session-journal/queued-message-pause' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import type { JournalWriteOptions } from '../agent-session-journal/journal-host-database' import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' /** A per-process id, minted once per host process like the runtime's own @@ -36,20 +35,20 @@ export function structuredQueuePauses(journal: PauseJournal): DerivedQueuePause[ * chat with cards the earlier process left, and so does a person's close: the idle sweep never closes a chat with cards waiting, so its * eviction never reopens one. `since`: where the chat stopped, for a mark written after a send * that came later, which must still lift it. Bookkeeping, so a failure is reported and never - * thrown; the pause then starts where the mark would have gone, holding no less. A background - * mark (`JournalWriteOptions`) that met another connection's lock throws, ending the retry's round. */ + * thrown; the pause then starts where the mark would have gone, holding no less. One written + * through the retry's background handle that met another connection's lock throws, ending the + * retry's round. */ export async function markStructuredQueueReopen( sessionId: string, - journal: Pick, + journal: Pick & { background?: true }, fence: number, logger: StructuredAgentSessionLogger, - since?: number, - options?: JournalWriteOptions + since?: number ): Promise { try { - await journal.markQueueReopen(fence, since, options) + await journal.markQueueReopen(fence, since) } catch (error) { - if (options?.background && isSqliteContentionFailure(error)) { + if (journal.background && isSqliteContentionFailure(error)) { throw error } logger.warn('marking a reopened queue failed', { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts index c79b04ec0e57..919df4d0c684 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts @@ -1,7 +1,8 @@ -// The one backoff every in-memory retry of background bookkeeping uses: from 1 s to 30 s, given up -// after a bounded run (about three minutes), so nothing is owed forever. +// The one backoff the host's retry of background bookkeeping uses: from 1 s to 2 s, given up after +// ten rounds in a row, so nothing is owed forever. Short: a background write fails at once, so +// waiting long only delays work a lifted lock already allows, and nothing announces the lift. -export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 30_000 } as const +export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 2_000 } as const /** Consecutive failed attempts after which a retry gives up. */ export const RECONCILIATION_MAX_FAILED_ATTEMPTS = 10 diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts index aba47487c80a..ef3df020d479 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-load.ts @@ -1,6 +1,6 @@ -// The reconciliation worker's own read of a chat nobody has open: replayed outside the chat's lane, -// and never indexed as the chat's conversation, so what its pass writes reaches no reader and no -// status surface. A reader that opens the chat replays it fresh; the worker then uses theirs. +// The host retry's own read of a chat nobody has open: replayed outside the chat's lane, and never +// indexed as the chat's conversation, so what its pass writes reaches no reader and no status +// surface. A reader that opens the chat replays it fresh; the retry then uses theirs. import { setImmediate as yieldToEventLoop } from 'node:timers/promises' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts index e208e2658dcf..10b02bba8ce4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-parked-debts.test.ts @@ -110,7 +110,7 @@ describe('a chat whose lease recovers', () => { const current = await recoveringAt14() const { reconciliation } = current.collaboratorsForTests() - // Generation 13's exit could not be settled; the lease recovers, so the worker retires. + // Generation 13's exit could not be settled; the lease recovers, so the retry retires. reconciliation.signal(CHAT, { evidence: proof13, exit: exit13 }) await idle(current, [CHAT]) expect(reconciliation['owed'].get(CHAT)).toMatchObject({ @@ -138,7 +138,7 @@ describe('a chat waiting out a backoff with an exit account', () => { it("drops it once a later generation acquired, so generation 14's send stays in doubt", async () => { const current = await recoveringAt14() const { reconciliation } = current.collaboratorsForTests() - // The pass fails once (storage busy): the worker backs off still holding generation 13's. + // The pass fails once (storage busy): the retry backs off still holding generation 13's. vi.spyOn(console, 'warn').mockImplementation(() => undefined) vi.spyOn(JournalQueuedMessages.prototype, 'repairAndPrune').mockRejectedValueOnce( Object.assign(new Error('database is locked'), { errcode: 5 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts index 6e581f1fbdff..1ae83048ddbc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-pass.ts @@ -1,5 +1,6 @@ -// One attempt of a chat's reconciliation worker, for a caller inside the chat's action lane: the -// short recheck and the writes. Everything owed is re-derived here from the lease, this host's +// One visit of the host's retry to a chat, for a caller inside the chat's action lane: the short +// recheck and the writes, all through the background handles (`structured-agent-session- +// background-writes.ts`), so another connection's lock ends the round instead of waiting. Everything owed is re-derived here from the lease, this host's // sight of an exit and the journal, so a pass that finds nothing owed writes nothing; only what // cannot be derived rides in from the signals (an exit's own account, a proof the lease no longer // holds). A lease latched in recovery still gets (b), which touches no process and no lease, as @@ -15,6 +16,12 @@ // An empty item plan says nothing of (a) or (b). import { holdUnsentSends } from '../agent-session-journal/journal-unsent-send-hold' +import { + backgroundJournalWrites, + backgroundStoreWrites, + type BackgroundJournalWrites, + type BackgroundStoreWrites +} from './structured-agent-session-background-writes' import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { queuedMessageReopenMarkStart } from '../agent-session-journal/queued-message-reopen-floor' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' @@ -60,10 +67,7 @@ export type StructuredAgentSessionReconciliationPass = { recovering: boolean } -/** Every write a pass makes is bookkeeping no person waits on (`JournalWriteOptions`). */ -const BACKGROUND = { background: true } as const - -/** `session`: the chat's open conversation, or the worker's own read of a closed one. +/** `session`: the chat's open conversation, or the retry's own read of a closed one. * `processOpened`: where this host first opened the chat's journal; a send at or before it was an * earlier process's. A later handle's own open cannot tell: this process may have accepted sends * through an earlier handle, which its close or its delivery loop still owns. */ @@ -77,6 +81,11 @@ export async function runStructuredAgentSessionReconciliationPass( const lease = context.deps.store.getRecord(sessionId)?.lease const recovering = lease?.handoffStage === 'recovering' const pass: StructuredAgentSessionReconciliationPass = { failed: [], wrote: false, recovering } + // Every write a pass makes is bookkeeping no person waits on: only these handles write. + const writes = { + store: backgroundStoreWrites(context.deps.store), + journal: backgroundJournalWrites(session.journal) + } // An exit's account judges only its own generation: once a later one acquired, it is spent. if (debts.exit && (!lease || lease.runtimeFence > debts.exit.ownerFence + 1)) { delete debts.exit @@ -84,11 +93,11 @@ export async function runStructuredAgentSessionReconciliationPass( if ( !recovering && structuredAgentSessionEndedChildHoldsLease(context, sessionId) && - (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId, BACKGROUND)) + (await releaseLeaseOfEndedStructuredAgentSessionChild(context, sessionId, writes.store)) ) { pass.failed.push(new Error('agent_session_exit_release_owed')) } - await settleEarlierProcess(context, sessionId, session.journal, processOpened, pass) + await settleEarlierProcess(context, sessionId, session.journal, writes, processOpened, pass) if (recovering || pass.failed.some(isSqliteContentionFailure)) { // What ended is not known until its recovery is decided; the debts wait for that signal. return pass @@ -98,10 +107,10 @@ export async function runStructuredAgentSessionReconciliationPass( store: context.deps.store, sessionId, journal: session.journal, + writes: writes.journal, ...(debts.exit ? { exit: debts.exit } : {}), ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), - ...(proof ? { proof } : {}), - background: true + ...(proof ? { proof } : {}) }) // Each proof the lease no longer holds judges what its own generation left, oldest first. const leaseEvidence = context.deps.store.getRecord(sessionId)?.lease.deathEvidence ?? null @@ -149,6 +158,7 @@ async function settleEarlierProcess( context: StructuredAgentSessionReconciliationPassContext, sessionId: string, journal: StructuredAgentSessionHostSession['journal'], + writes: { store: BackgroundStoreWrites; journal: BackgroundJournalWrites }, processOpened: AgentJournalCursor, pass: StructuredAgentSessionReconciliationPass ): Promise { @@ -169,7 +179,7 @@ async function settleEarlierProcess( } ) // Another connection's lock ends the pass: the round ends, and every step is tried again. - if (!(await attempt(() => journal.queuedMessages.repairAndPrune(BACKGROUND)))) { + if (!(await attempt(() => writes.journal.queuedMessages.repairAndPrune()))) { return } // Before a Stop can withdraw one: a Stop never withdraws a card. @@ -180,7 +190,7 @@ async function settleEarlierProcess( fence, hostInstance: structuredAgentSessionHostInstance(), hold: { cause: 'hostRestarted', which: (entry) => earlier(entry.acceptedSequence) }, - background: true + writes: writes.journal })) !== null }) if (!held && pass.failed.some(isSqliteContentionFailure)) { @@ -192,24 +202,23 @@ async function settleEarlierProcess( const floor = journal.reopenFloor() const since = queuedMessageReopenMarkStart(floor, processOpened, journal.cursor().epoch) if ((converted || floor !== null) && since !== null) { - await markStructuredQueueReopen( - sessionId, - journal, - fence, - context.deps.logger, - since, - BACKGROUND - ) + await markStructuredQueueReopen(sessionId, writes.journal, fence, context.deps.logger, since) } // A rewind left prepared refuses every send until settled. With no provider here, one only its // provider can prove stays, quietly, for the next acquisition, which recovers it. if (!structuredRewindNeedsProvider(record.rewind)) { - await recoverStructuredRewind(context.deps, sessionId, journal, fence).catch((error: unknown) => + const deps = { ...context.deps, background: writes } + await recoverStructuredRewind(deps, sessionId, journal, fence).catch((error: unknown) => { + // Another connection's lock ends the round, as every other write here does. + if (isSqliteContentionFailure(error)) { + pass.failed.push(error) + return + } context.deps.logger.warn('recovering a rewind an earlier process left did not finish', { scope: 'reconciliation-rewind', sessionId, error }) - ) + }) } } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts index a2749db3a012..f4987835ca92 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts @@ -38,6 +38,8 @@ import { turnState, watchSettlementCommits } from './structured-agent-session-leftover-settlement.test-fixture' +import { retryIdle, retryOwes } from './structured-agent-session-retry.test-fixture' +import { backgroundSettlementWrites } from './structured-agent-session-background-writes' let rig: QueuedMessageTestRig | undefined @@ -47,7 +49,7 @@ afterEach(async () => { vi.restoreAllMocks() }) -function worker(current: QueuedMessageTestRig) { +function retryOf(current: QueuedMessageTestRig) { return current.host.collaboratorsForTests().reconciliation } @@ -64,7 +66,7 @@ describe('a retry dies', () => { openTestJournalHostDatabase(current.root).db.exec(REJECT_RECOVERED_ROWS) ) const database = openTestJournalHostDatabase(current.root) - expect(worker(current).owes(SESSION)).toBe(true) + expect(retryOwes(retryOf(current), SESSION)).toBe(true) expect(turnState(current)).toBe('running') database.db.exec('DROP TRIGGER reject_recovered') @@ -73,7 +75,8 @@ describe('a retry dies', () => { await settleStructuredAgentSessionLeftovers({ store: current.store, sessionId: SESSION, - journal: currentJournal(current) + journal: currentJournal(current), + writes: backgroundSettlementWrites(currentJournal(current)) }) ).toMatchObject({ ok: true }) expect(turnState(current)).toBe('unverifiable') @@ -113,7 +116,7 @@ describe('a retry dies', () => { current.host.stopDelivery() - expect(worker(current).owes(SESSION)).toBe(false) + expect(retryOwes(retryOf(current), SESSION)).toBe(false) await new Promise((resolve) => setTimeout(resolve, 1_500)) expect(recoveredRows(database.db)).toEqual([]) expect(turnState(current)).toBe('running') @@ -121,7 +124,7 @@ describe('a retry dies', () => { }) describe('an observed exit whose release write failed', () => { - it('is Working for no reader at once, restated with no row, and the worker owes the release until it lands', async () => { + it('is Working for no reader at once, restated with no row, and the retry owes the release until it lands', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig await current.workingSend() @@ -164,8 +167,8 @@ describe('an observed exit whose release write failed', () => { // item plan never retires it. database.db.exec('DROP TRIGGER reject_recovered') await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 10_000 }) - await worker(current).idle(SESSION) - expect(worker(current).owes(SESSION)).toBe(true) + await retryIdle(retryOf(current), SESSION) + expect(retryOwes(retryOf(current), SESSION)).toBe(true) expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('live') release.mockRestore() @@ -184,7 +187,7 @@ describe('a journal no retry can load', () => { ['damaged', '}{'], ['saved by a newer Orca', null] ] as const)( - '%s: the startup worker retires instead of replaying it again', + '%s: the startup retry settles it instead of replaying it again', async (_why, json) => { // The owner is proven gone at the restart, so no recovery waits on a person's attach. rig = await createQueuedMessageTestRig({ @@ -212,7 +215,7 @@ describe('a journal no retry can load', () => { }) }) - expect(worker(current).owes(SESSION)).toBe(false) + expect(retryOwes(retryOf(current), SESSION)).toBe(false) await new Promise((resolve) => setTimeout(resolve, 1_200)) expect(opens).toBe(1) } @@ -255,7 +258,7 @@ describe('the reopen mark after a restart', () => { }) describe("an ended generation's closing tail", () => { - it('lands at its own fence after the exit settled, and its worker settles what it left running', async () => { + it('lands at its own fence after the exit settled, and the retry settles what it left running', async () => { rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig await current.workingSend() @@ -285,7 +288,7 @@ describe("an ended generation's closing tail", () => { ) expect(currentJournal(current).itemBody(agentJournalItemKey(tail(1)))).not.toBeNull() - // Below the moved fence, so its commit woke the worker, which settles it by the exit's proof. + // Below the moved fence, so its commit woke the retry, which settles it by the exit's proof. await vi.waitFor(() => expect( readAgentJournalTurn( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts index 6f5356320735..4632eb001557 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -51,8 +51,6 @@ export type StructuredAgentSessionReconciliationSignal = { restate?: boolean } -const BACKGROUND = { background: true } as const - export class StructuredAgentSessionRetry { private readonly owed = new Map() /** Where this process first opened each chat's journal: every send at or before it was accepted @@ -81,7 +79,13 @@ export class StructuredAgentSessionRetry { firstOpened: this.firstOpened, settle: (sessionId, chat) => this.settle(sessionId, chat), warn: (sessionId, error) => this.warn(sessionId, error), - inTurn: (run) => this.inTurn(run) + inTurn: (run) => this.inTurn(run), + // Once the lane's own bookkeeping has cleared too, so the next visit finds it free. + afterLane: (sessionId) => + void context + .serialize(sessionId, async () => undefined) + .then(() => yieldToEvents()) + .then(() => this.roundSoon()) } this.unsubscribe = context.deps.store.onGenerationEnded((ended) => this.signal(ended.sessionId, { evidence: ended.evidence }) @@ -126,12 +130,14 @@ export class StructuredAgentSessionRetry { } } - /** The queue's automatic send met another connection's lock: the next round sends it again. */ + /** The queue's automatic send met another connection's lock: the next round sends it again. + * Owed, not a failure: only a round counts against the budget, however many sends collided. */ sendContended = (sessionId: string): void => { if (!this.disposed) { this.owe(sessionId).send = true + this.contended = true this.context.publishGenerationEnded(sessionId) - this.roundEnded(true, true) + this.roundSoon() } } @@ -144,16 +150,6 @@ export class StructuredAgentSessionRetry { return chat?.due ? new Promise((resolve) => chat.attempted.push(resolve)) : Promise.resolve() } - /** Whether a visit is owed: something was signalled, or failed, and has not settled. */ - owes = (sessionId: string): boolean => this.owed.get(sessionId)?.due === true - - /** Resolves once the chat owes nothing, or waits out a failed round's backoff. */ - idle = async (sessionId: string): Promise => { - while (this.owes(sessionId) && (this.running || this.timer === 'soon' || this.failures === 0)) { - await this.attempted(sessionId) - } - } - /** Shutdown: the timer stops and every slot wait ends; a visit already writing finishes, * tracked, and one that has not started work starts none. */ dispose = (): void => { @@ -166,10 +162,6 @@ export class StructuredAgentSessionRetry { } } - processOpened(sessionId: string): AgentJournalCursor | undefined { - return this.firstOpened.get(sessionId) - } - private owe(sessionId: string): OwedChat { let chat = this.owed.get(sessionId) if (!chat) { @@ -191,6 +183,15 @@ export class StructuredAgentSessionRetry { this.contended = false if (!this.running && this.timer !== 'soon') { this.stopTimer() + this.roundSoon() + } + } + + /** A round now unless one runs, is due or waits out a backoff; the budget stays as it is. */ + private roundSoon(): void { + if (this.running) { + this.again = true + } else if (!this.disposed && this.timer === null) { this.timer = 'soon' queueMicrotask(() => void this.round()) } @@ -205,15 +206,20 @@ export class StructuredAgentSessionRetry { this.again = false this.stopped = false let failed = false + let contended = false const end = (visit: Visit) => { failed ||= visit === 'failed' || visit === 'contended' + contended ||= visit === 'contended' this.stopped ||= visit === 'contended' this.again ||= visit === 'again' } if (this.context.deps.store.listRecords().some((record) => record.lease.unreconciled)) { - // Store-wide: once a round, for every chat, never once per chat. - if (!(await this.context.reconcile('retry', BACKGROUND))) { - end('contended') + // Store-wide: once a round, for every chat, never once per chat. No chat is visited while a + // lease is unreconciled. + const reconciled = await this.context.reconcile() + if (reconciled !== 'settled') { + end(reconciled) + this.stopped = true } } const due = [...this.owed].filter(([, chat]) => chat.due) @@ -226,7 +232,7 @@ export class StructuredAgentSessionRetry { }) ) this.running = false - this.roundEnded(failed, this.stopped) + this.roundEnded(failed, contended) // After the round's outcome is set, so a waiter reads its backoff, never a round still running. for (const [, chat] of due) { chat.attempted.splice(0).forEach((resolve) => resolve()) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts index f651affba300..2ad0787bfc56 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-slots.ts @@ -1,5 +1,4 @@ -// The reconciliation workers' few background slots, for a chat's replay and its startup recovery -// step: outside every chat's lane, so a send or a read never waits behind the scan. A chat a reader +// The host retry's few background slots, for a chat's replay and its startup recovery step: outside every chat's lane, so a send or a read never waits behind the scan. A chat a reader // opened goes ahead of the rest; a quit ends every wait, and a task it has not started never runs. import { PrioritySemaphore } from '../../../shared/priority-semaphore' diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts index 76216ebfaf05..29cd6273fdfe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts @@ -18,22 +18,31 @@ import type { StructuredAgentSessionReconciliationSlotWaiter } from './structure export type StructuredAgentSessionRetryContext = StructuredAgentSessionReconciliationPassContext & { deps: StructuredAgentSessionReconciliationPassContext['deps'] - /** The chat's action lane. */ + /** The chat's action lane, and whether anything holds it now. */ serialize: (sessionId: string, task: () => Promise) => Promise + laneBusy: (sessionId: string) => boolean /** Lets a quit wait for a visit already writing. */ track: (operation: Promise) => Promise /** Publishes what is current now and wakes the queued-card drain. */ publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void - /** Startup's store-wide lease reconcile (`createReaderReconcile`): whether every lease settled. */ - reconcile: (sessionId: string, options: { background: true }) => Promise - /** The queue's automatic send of the chat's next card, and an episode giving up on it - * (`StructuredAgentSessionQueuedMessageDrain.sendForRetry`, `.abandon`). */ + /** Startup's store-wide lease reconcile, as background bookkeeping: whether every lease settled, + * or why not. */ + reconcile: () => Promise<'settled' | 'contended' | 'failed'> + /** The queue's automatic send of the chat's next card, inside the lane the visit holds, and an + * episode giving up on it (`StructuredAgentSessionQueuedMessageDrain.sendForRetry`, `.abandon`). */ sendQueued: (sessionId: string) => Promise<'contended' | 'done'> abandonSend: (sessionId: string) => void } -/** `again`: it wrote, or was signalled meanwhile, so the next round verifies nothing is left. */ -export type StructuredAgentSessionVisit = 'settled' | 'again' | 'parked' | 'failed' | 'contended' +/** `again`: it wrote, or was signalled meanwhile, so the next round verifies nothing is left. + * `busy`: a person's operation held its lane, so it waits, owed, for that lane to free. */ +export type StructuredAgentSessionVisit = + | 'settled' + | 'again' + | 'parked' + | 'failed' + | 'contended' + | 'busy' /** A chat with something owed, until a visit finds nothing left. */ export type StructuredAgentSessionOwedChat = StructuredAgentSessionReconciliationSlotWaiter & { @@ -60,6 +69,8 @@ export type StructuredAgentSessionVisitHost = { warn: (sessionId: string, error: unknown) => void /** The round's one-at-a-time turn for a chat's writes; null once the round stopped. */ inTurn: (run: () => Promise) => Promise + /** A round once the chat's busy lane frees. */ + afterLane: (sessionId: string) => void } type Chat = StructuredAgentSessionOwedChat @@ -80,9 +91,6 @@ export async function visitStructuredAgentSessionOwedChat( } return host.settle(sessionId, chat) } - if (store.getRecord(sessionId)?.lease.unreconciled) { - return 'failed' - } if (!context.sessions.has(sessionId) && !chat.loaded) { const loaded = await loadStructuredAgentSessionForReconciliation( context.deps, @@ -97,6 +105,10 @@ export async function visitStructuredAgentSessionOwedChat( } } const visit = await host.inTurn(() => write(host, sessionId, chat)) + if (visit === null) { + // The round stopped before this chat's turn: its read is not kept open until the next. + dropStructuredAgentSessionLoaded(chat) + } return visit === 'settled' ? host.settle(sessionId, chat) : visit } catch (error) { host.warn(sessionId, error) @@ -105,23 +117,30 @@ export async function visitStructuredAgentSessionOwedChat( } /** The chat's writes, in its lane: the pass, then (d) once nothing else of it is due, the send it - * owes, with the card named again. */ + * owes, with the card named again. Never waits for a lane inside the round's turn: a person's + * operation holding it leaves the chat owed for when it frees. */ async function write( host: StructuredAgentSessionVisitHost, sessionId: string, chat: Chat ): Promise { const { context } = host - const visit = await context.serialize(sessionId, () => pass(host, sessionId, chat)) - if ((visit !== 'settled' && visit !== 'parked') || !chat.send) { - return visit - } - if ((await context.sendQueued(sessionId)) === 'contended') { - return 'contended' + if (context.laneBusy(sessionId)) { + host.afterLane(sessionId) + return 'busy' } - delete chat.send - context.publishGenerationEnded(sessionId) - return visit + return context.serialize(sessionId, async () => { + const visit = await pass(host, sessionId, chat) + if ((visit !== 'settled' && visit !== 'parked') || !chat.send) { + return visit + } + if ((await context.sendQueued(sessionId)) === 'contended') { + return 'contended' + } + delete chat.send + context.publishGenerationEnded(sessionId) + return visit + }) } /** Where this process first opened the chat, if this is the first. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts index 955992c36dca..4c2343ae9dae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-resend-ledger-refusal.test.ts @@ -19,6 +19,7 @@ import { HOST_TEST_SESSION as SESSION, hostTestMessage } from './structured-agent-session-host-test-data' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const EXPIRED = { ok: false, @@ -76,8 +77,8 @@ describe('an expired send id', () => { it('is answered expired while its chat is closed, not as a chat this host lacks', async () => { await attach() await host.close(SESSION, 'evict') - // The stop's release wakes the chat's worker, which opens it, finds nothing owed, and closes it. - await host.collaboratorsForTests().reconciliation.idle(SESSION) + // The stop's release wakes the host's retry, which opens it, finds nothing owed, and closes it. + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) await vi.waitFor(() => expect(host.hasSession(SESSION)).toBe(false)) await expect(host.send(CALLER, expiredParams('long gone'))).resolves.toMatchObject(EXPIRED) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts index 1d14597a546b..0cc5a8cf3d0e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rest-test-rig.ts @@ -36,6 +36,7 @@ import { STRUCTURED_AGENT_SESSION_IDLE_MS } from './structured-agent-session-idl import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' +import { retryIdle } from './structured-agent-session-retry.test-fixture' export const REST_TEST_CALLER = { callerKey: 'client-1' } export const IDLE_MS = STRUCTURED_AGENT_SESSION_IDLE_MS @@ -228,8 +229,8 @@ export async function foundRestTestChat(rig: RestTestRig): Promise { export async function sweepOnce(host: StructuredAgentSessionHost): Promise { const { lifetime, reconciliation, serialize } = host.collaboratorsForTests() await lifetime.idleSweep.tick() - // A close the sweep left to the chat's reconciliation worker lands once that retires. - await reconciliation.idle(SESSION) + // A close the sweep left to the host's retry lands once that retires. + await retryIdle(reconciliation, SESSION) await serialize(SESSION, async () => {}) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts index c489792b334c..8f612a061600 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts @@ -147,4 +147,31 @@ describe('a round of the retry', () => { expect(order.indexOf(`person:${later}`)).toBeLessThan(order.indexOf(`visit:${later}`)) expect(new Set(order.filter((entry) => entry.startsWith('visit:'))).size).toBe(3) }) + + it("never holds the round for a chat's busy lane: a person's long operation delays no other chat", async () => { + const chats = await seedChats(3, true) + const started = performance.now() + const visits: Record = {} + const pass = reconciliationPass.runStructuredAgentSessionReconciliationPass + vi.spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass').mockImplementation( + async (...args) => { + visits[args[1]] ??= performance.now() - started + return pass(...args) + } + ) + host = openScanHost(root, store) + // A person's operation holds the first chat's lane for 3 s (a Stop waiting out its deadline). + const [busy, ...others] = chats + const person = host + .collaboratorsForTests() + .serialize(busy!, () => new Promise((resolve) => setTimeout(resolve, 3_000))) + await host.reconcileRestartLeases() + await vi.waitFor(() => expect(Object.keys(visits)).toHaveLength(2), { timeout: 10_000 }) + + expect(Math.max(...others.map((id) => visits[id] ?? Infinity))).toBeLessThan(1_500) + expect(visits[busy!]).toBeUndefined() + // The busy chat is visited once its lane frees. + await person + await vi.waitFor(() => expect(visits[busy!]).toBeGreaterThan(2_900), { timeout: 10_000 }) + }, 20_000) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index a841a649ba82..0ac228fc5f16 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -22,7 +22,10 @@ export function createRestartReconciler(deps: { records: readonly AgentSessionRecord[] ) => Promise> now: () => number -}): (sessionId: string, options?: JournalWriteOptions) => Promise { +}): ( + sessionId: string | null, + options?: JournalWriteOptions +) => Promise { let pending: Promise | null = null return async (sessionId, options) => { if (!deps.store.listRecords().some((record) => record.lease.unreconciled)) { @@ -44,11 +47,8 @@ export function createRestartReconciler(deps: { await pending return null } catch (error) { - return classifyStoreFailure( - error, - deps.store.getRecord(sessionId)?.lease.runtimeFence ?? null, - deps.store.getRecord(sessionId) - ) + const record = sessionId === null ? null : deps.store.getRecord(sessionId) + return classifyStoreFailure(error, record?.lease.runtimeFence ?? null, record) } } } @@ -61,8 +61,7 @@ export type ReaderBookkeepingFailures = { clear: () => void } -/** Startup and the read carry on past a failure: each chat's reconciliation worker retries it with - * a backoff, and the next attach or send reconciles and resolves recovery again before it acts. */ +/** Startup and the read carry on past a failure: the host's retry tries it again with a backoff, and the next attach or send reconciles and resolves recovery again before it acts. */ export function reportEachFailureOnce( logger: StructuredAgentSessionLogger ): ReaderBookkeepingFailures { @@ -89,16 +88,13 @@ export function reportEachFailureOnce( * reconciles again before it acts. * Answers whether every lease is settled. */ export function createReaderReconcile( - reconcile: ( - sessionId: string, - options?: JournalWriteOptions - ) => Promise, + reconcile: (sessionId: string) => Promise, failures: ReaderBookkeepingFailures -): (sessionId: string, options?: JournalWriteOptions) => Promise { - return async (sessionId, options) => { +): (sessionId: string) => Promise { + return async (sessionId) => { let failure: unknown try { - const refusal = await reconcile(sessionId, options) + const refusal = await reconcile(sessionId) if (!refusal) { failures.clear() return true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts index a91904964819..3c4ca6fbffbe 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-restore.ts @@ -80,7 +80,7 @@ export async function restoreStructuredAgentSessionsOnRestart( } // One check for the pass. Each chat checks again while it holds, since another writer can mark // leases unreconciled mid-pass; after the first failure, retrying per chat only waits on the - // same store again; each chat's reconciliation worker retries it with a backoff instead, and the + // same store again; the host's retry tries it once a round with a backoff instead, and the // next attach or send reconciles before it acts. let settled = await input.reconcile(first.sessionId) const settleLeases = async (sessionId: string): Promise => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts new file mode 100644 index 000000000000..620e92561963 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts @@ -0,0 +1,77 @@ +// The retry's budget counts rounds, never collisions: many chats whose automatic sends meet one +// lock at once cost one failed round, not one each, and the work lands once the lock lifts. + +import { afterEach, expect, it, vi } from 'vitest' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' +import { + StructuredAgentSessionRetry, + type StructuredAgentSessionRetryContext +} from './structured-agent-session-reconciliation-retry' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +async function settleTurns(): Promise { + for (let turn = 0; turn < 60; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + +it('ten chats whose automatic sends meet one lock cost one round, and send once it lifts', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + vi.spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass').mockResolvedValue({ + failed: [], + wrote: false, + recovering: false + }) + const ids = Array.from({ length: 10 }, (_, index) => `chat-${index}`) + let locked = true + const sendQueued = vi.fn(async () => (locked ? ('contended' as const) : ('done' as const))) + const abandonSend = vi.fn() + const context = { + deps: { + store: { + onGenerationEnded: () => () => undefined, + getRecord: () => ({ lease: { unreconciled: false, runtimeFence: 1 } }), + listRecords: () => [], + readOnly: false + }, + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() } + }, + sessions: new Map( + ids.map((id) => [id, { journal: { openedAt: () => ({ epoch: 'e', sequence: 0 }) } }]) + ), + now: () => 0, + serialize: (_id: string, task: () => Promise) => task(), + laneBusy: () => false, + track: (operation: Promise) => operation, + publishGenerationEnded: vi.fn(), + reconcile: async () => 'settled' as const, + sendQueued, + abandonSend + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retry reads only the fields above; the host's full deps would add nothing this test exercises. + const typed = context as unknown as StructuredAgentSessionRetryContext + const retry = new StructuredAgentSessionRetry(typed) + + // Each chat's own drain met the lock at the same moment. + for (const id of ids) { + retry.sendContended(id) + } + await settleTurns() + // One round, ended by the first send that met the lock: one failure, nobody given up on. + expect(sendQueued).toHaveBeenCalledTimes(1) + expect(abandonSend).not.toHaveBeenCalled() + expect(retry['failures']).toBe(1) + + locked = false + await vi.advanceTimersByTimeAsync(2_000) + await settleTurns() + expect(sendQueued).toHaveBeenCalledTimes(11) + expect(ids.filter((id) => retryOwes(retry, id))).toEqual([]) + expect(abandonSend).not.toHaveBeenCalled() + retry.dispose() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts new file mode 100644 index 000000000000..bfa96e55d5f6 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry.test-fixture.ts @@ -0,0 +1,22 @@ +// What a test may ask of the host's retry (`StructuredAgentSessionRetry`) that no production caller +// does: whether a chat is still owed, and waiting until it is not. + +import type { StructuredAgentSessionRetry } from './structured-agent-session-reconciliation-retry' + +/** Whether a visit is owed: something was signalled, or failed, and has not settled. */ +export function retryOwes(retry: StructuredAgentSessionRetry, sessionId: string): boolean { + return retry['owed'].get(sessionId)?.due === true +} + +/** Resolves once the chat owes nothing, or a failed round left it waiting out a backoff. */ +export async function retryIdle( + retry: StructuredAgentSessionRetry, + sessionId: string +): Promise { + while ( + retryOwes(retry, sessionId) && + (retry['running'] || retry['timer'] === 'soon' || retry['failures'] === 0) + ) { + await retry.attempted(sessionId) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index efa3239d9683..7e1b8f0218a9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -13,6 +13,8 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { journalOpenRefusal } from '../agent-session-journal/journal-open-failure' +import type { JournalWriteOptions } from '../agent-session-journal/journal-host-database' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' import { @@ -68,10 +70,13 @@ export function createStructuredAgentSessionHostRestore( ConstructorParameters[0], 'openDeps' | 'reconcile' | 'resolveRecovery' > & { - reconcileLeases: (sessionId: string) => Promise + reconcileLeases: ( + sessionId: string | null, + options?: JournalWriteOptions + ) => Promise resolveRecovery: (sessionId: string) => Promise startup: Pick & { - tasks: Pick + tasks: Pick clientDelivery: Pick queue: { sendForRetry: (sessionId: string) => Promise<'contended' | 'done'> @@ -94,11 +99,26 @@ export function createStructuredAgentSessionHostRestore( sessions: startup.sessions, now: () => deps.now?.() ?? Date.now(), serialize: rest.serialize, + laneBusy: (sessionId) => startup.tasks.busy(sessionId), // Tracked like a start, so a quit waits for an attempt before it closes what it writes to. track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => startup.clientDelivery.publishGenerationEnded(sessionId, options), - reconcile, + reconcile: () => + reconcileLeases(null, { background: true }).then( + (refusal) => { + if (!refusal) { + failures.clear() + return 'settled' + } + failures.report(refusal) + return 'failed' + }, + (error: unknown) => { + failures.report(error) + return isSqliteContentionFailure(error) ? 'contended' : 'failed' + } + ), sendQueued: (sessionId) => startup.queue.sendForRetry(sessionId), abandonSend: (sessionId) => startup.queue.abandon(sessionId) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts index b81e9aec3bd1..0b9ea4c273e0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts @@ -1,7 +1,7 @@ // A roster row and a background-task row are revised in place across generations, so each entry // carries the generation that last observed it. Settling an ended generation settles only the // entries it observed; a later live generation's children stay as they are. And a proof the -// reconciliation worker holds for an ended generation revises what an earlier settle of it could +// retry holds for an ended generation revises what an earlier settle of it could // only call unverifiable. import { mkdtemp, rm } from 'node:fs/promises' @@ -39,6 +39,7 @@ import type { AgentSessionJournal } from '../agent-session-journal/journal-store import { createDeferredStructuredAgentSessionEventSink } from './structured-agent-session-event-sink' import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' import { testEventSinkLogging } from './structured-agent-session-logger-test-support' +import { backgroundSettlementWrites } from './structured-agent-session-background-writes' const SESSION = 'orca-session' const GROUP = 'claude-session:turn-a' @@ -202,7 +203,8 @@ describe('settling an ended generation’s roster', () => { await settleStructuredAgentSessionLeftovers({ store: { getRecord: (sessionId) => (sessionId === SESSION ? live : null) }, sessionId: SESSION, - journal + journal, + writes: backgroundSettlementWrites(journal) }) ).toMatchObject({ ok: true }) @@ -219,7 +221,8 @@ describe('settling an ended generation’s roster', () => { await settleStructuredAgentSessionLeftovers({ store: { getRecord: () => live }, sessionId: SESSION, - journal + journal, + writes: backgroundSettlementWrites(journal) }) ).toEqual({ ok: true, planned: 0 }) }) @@ -255,12 +258,13 @@ describe('a held proof', () => { await settleStructuredAgentSessionLeftovers({ store: { getRecord: () => released }, sessionId: SESSION, - journal + journal, + writes: backgroundSettlementWrites(journal) }) expect(turn()).toMatchObject({ state: 'unverifiable' }) // A later probe proved that owner gone; a reservation since cleared the lease's copy, and the - // worker holds the proof it was handed. + // retry holds the proof it was handed. const live = agentSessionRecordFixture( agentSessionLeaseFixture({ sessionId: SESSION, runtimeFence: LIVE }) ) @@ -269,6 +273,7 @@ describe('a held proof', () => { store: { getRecord: () => live }, sessionId: SESSION, journal, + writes: backgroundSettlementWrites(journal), proof: { kind: 'pid-absent', detail: 'owner gone', observedAt: 2_000, ownerFence: ENDED } }) ).toMatchObject({ ok: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts index fb7f7ef81e18..ac865bf60276 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-send-open-stale-turn.test.ts @@ -28,6 +28,7 @@ import { import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' import { createStructuredAgentSessionLogger } from './structured-agent-session-logger' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' /** Delivery runs on its own serialized steps; under a loaded runner they take more than a second. */ function eventually(assertion: () => unknown): Promise { @@ -112,9 +113,9 @@ it.each(PROBES)( 'settles a turn a dead generation left running when a send opens the chat and its start fails, when %s', async (_when, probe, settled) => { const { host, acquire } = await relaunchAfterCrashMidTurn(probe) - // The startup's worker opens the chat in the background and closes it again once done. + // The startup retry opens the chat in the background and closes it again once done. await host.startupSettled() - await host.collaboratorsForTests().reconciliation.idle(SESSION) + await retryIdle(host.collaboratorsForTests().reconciliation, SESSION) await eventually(() => expect(host.hasSession(SESSION)).toBe(false)) const body = hostTestMessage('sent to a chat nobody has open') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts index 85432b2b901a..c2423682722c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stale-turn-open.test.ts @@ -1,5 +1,5 @@ // A turn an exited agent left running, whose settlement write failed, is settled by the chat's -// reconciliation worker once storage takes it, or by the next startup. A reader opening the chat +// retry once storage takes it, or by the next startup. A reader opening the chat // in between writes nothing, and the turn holds nothing: it is an ended generation's. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -16,6 +16,7 @@ import { sweepOnce, type RestTestRig } from './structured-agent-session-rest-test-rig' +import { retryIdle, retryOwes } from './structured-agent-session-retry.test-fixture' let rig: RestTestRig @@ -83,12 +84,12 @@ describe('a turn its gone agent left running', () => { expect(rig.adapter.acquire).toHaveBeenCalledOnce() }) - it('is settled by its chat’s worker once storage takes the write, with no send; a reader writes nothing meanwhile', async () => { - // The exit's own write and the worker's first attempt both fail; the backoff's retry lands. + it('is settled by the host retry once storage takes the write, with no send; a reader writes nothing meanwhile', async () => { + // The exit's own write and the retry's first attempt both fail; the backoff's retry lands. await exitWithUnwrittenSettlement(2) const { reconciliation } = rig.host.collaboratorsForTests() - await reconciliation.idle(SESSION) - expect(reconciliation.owes(SESSION)).toBe(true) + await retryIdle(reconciliation, SESSION) + expect(retryOwes(reconciliation, SESSION)).toBe(true) expect(await workingTurnState()).toBe('running') const cursor = rig.host.collaboratorsForTests().sessions.get(SESSION)!.journal.cursor() @@ -105,7 +106,7 @@ describe('a turn its gone agent left running', () => { ) expect(rig.adapter.acquire).toHaveBeenCalledOnce() // Nothing owed once it landed: the idle sweep closes the chat as it would any other. - await reconciliation.idle(SESSION) + await retryIdle(reconciliation, SESSION) rig.clock.now += IDLE_MS + 1 await sweepOnce(rig.host) expect(rig.host.collaboratorsForTests().sessions.has(SESSION)).toBe(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts index def0d0f52183..4e3624d0b6dd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-recovery-scope.test.ts @@ -1,6 +1,6 @@ // What startup decides about a lease latched in recovery: nothing, for a chat nobody looks at. // Deciding it signals a process that may still run, so only the visible-tab restore, a start or an -// attach does; that decision's release then wakes the chat's worker, which settles what it left. +// attach does; that decision's release then wakes the host's retry, which settles what it left. // What the earlier process left unsent is kept as a card meanwhile, as every open did on main. import { mkdtemp, rm } from 'node:fs/promises' @@ -83,7 +83,7 @@ describe('a hidden chat whose agent outlived the earlier process', () => { expect(stopOwnerProcess).toHaveBeenCalledWith(OWNER_PID, 'SIGTERM') expect(store.getRecord(CHAT)?.lease).toMatchObject({ handoffStage: null, ownerProcess: null }) - // Its release woke the worker, which settles the turn the owner left by that proof. + // Its release woke the retry, which settles the turn the owner left by that proof. await idle(current, [CHAT]) expect(await turnState(current, CHAT)).toBe('interrupted') }) @@ -137,12 +137,12 @@ describe('a visible chat whose agent outlived the earlier process', () => { }) describe('a chat whose startup lease reconcile failed (storage was busy)', () => { - it('is reconciled by its worker after a backoff, and its turn settles with no attach or send', async () => { + it('is reconciled by the retry after a backoff, and its turn settles with no attach or send', async () => { await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) await seedScanJournal(root, CHAT) store = await openTestAgentSessionRecordStore(root) const busy = () => new Error('database is locked') - // Startup's reconcile and the worker's first retry meet a locked store; the next one lands. + // Startup's reconcile and the retry's first retry meet a locked store; the next one lands. const reconcile = vi .spyOn(store, 'reconcileOnRestart') .mockRejectedValueOnce(busy()) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts index 8b1a3e95cea2..fcf8f6f37638 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan-lifetime.test.ts @@ -1,4 +1,4 @@ -// The startup scan's reconciliation workers over their whole life: what a quit cuts short, what +// The startup scan's retrys over their whole life: what a quit cuts short, what // they never publish or close, and what ends a retry that cannot succeed. import { mkdtemp, rm } from 'node:fs/promises' @@ -30,6 +30,7 @@ import { scanTurnState as turnState, seedScanJournal } from './structured-agent-session-startup-scan.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' const CHAT = 'chat-aaaaaaa1' @@ -169,7 +170,7 @@ describe('a chat the startup scan settles', () => { }) }) -describe('a startup worker whose step cannot succeed', () => { +describe('a startup retry whose step cannot succeed', () => { it('retires at once on a journal no retry can load, and replays it once', async () => { await seed([CHAT]) // The last row is one no build wrote. @@ -183,7 +184,7 @@ describe('a startup worker whose step cannot succeed', () => { await current.reconcileRestartLeases() await current.startupSettled() - expect(current.collaboratorsForTests().reconciliation.owes(CHAT)).toBe(false) + expect(retryOwes(current.collaboratorsForTests().reconciliation, CHAT)).toBe(false) await new Promise((resolve) => setTimeout(resolve, 1_200)) expect(counter.opens).toBe(1) expect(warnings).toContain("a chat's history cannot be loaded, so nothing is settled") @@ -230,7 +231,7 @@ describe('a startup worker whose step cannot succeed', () => { await current.reconcileRestartLeases() await current.startupSettled() - expect(current.collaboratorsForTests().reconciliation.owes(CHAT)).toBe(false) + expect(retryOwes(current.collaboratorsForTests().reconciliation, CHAT)).toBe(false) await new Promise((resolve) => setTimeout(resolve, 1_200)) expect(counter.opens).toBe(1) }) @@ -246,6 +247,6 @@ describe('a startup worker whose step cannot succeed', () => { // Attempts at 0 s, 1 s and 3 s: a read each, never a loop. expect(counter.opens).toBeGreaterThanOrEqual(2) expect(counter.opens).toBeLessThanOrEqual(3) - expect(current.collaboratorsForTests().reconciliation.owes(CHAT)).toBe(true) + expect(retryOwes(current.collaboratorsForTests().reconciliation, CHAT)).toBe(true) }) }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts index 45cb244af8ef..4f840a9b1063 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture.ts @@ -18,6 +18,7 @@ import { type StructuredAgentSessionLogger } from './structured-agent-session-logger' import type { StructuredAgentSessionStatusSink } from './structured-agent-session-status-ownership' +import { retryOwes } from './structured-agent-session-retry.test-fixture' export const SCAN_NOW = 1_800_000_000_000 export const SCAN_CHATS = [ @@ -178,7 +179,8 @@ export async function scanTurnState(current: StructuredAgentSessionHost, session export async function scanIdle(current: StructuredAgentSessionHost, sessionIds: readonly string[]) { const { reconciliation } = current.collaboratorsForTests() await vi.waitFor( - () => expect(sessionIds.filter((sessionId) => reconciliation.owes(sessionId))).toEqual([]), + () => + expect(sessionIds.filter((sessionId) => retryOwes(reconciliation, sessionId))).toEqual([]), { timeout: 10_000 } ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts index 6403eae1f1c9..07daaffacef7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts @@ -27,6 +27,7 @@ import { scanTurnState as turnState, seedScanJournal } from './structured-agent-session-startup-scan.test-fixture' +import { retryOwes } from './structured-agent-session-retry.test-fixture' let root: string let store: AgentSessionRecordStore @@ -196,7 +197,7 @@ describe('the startup scan', () => { await current.reconcileRestartLeases() await current.startupSettled() const { reconciliation } = current.collaboratorsForTests() - expect(reconciliation.owes(RELEASED)).toBe(true) + expect(retryOwes(reconciliation, RELEASED)).toBe(true) database.db.exec('DROP TRIGGER reject_conversion') await idle(current, [RELEASED]) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts index 45539542f109..87fd4618723e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-startup-settlement.ts @@ -1,11 +1,11 @@ // What the host owes its chats when it comes up as a new instance: what the earlier process left. // Run after the whole-host restart reconcile, in the background, for EVERY chat on record: no -// stored "settled" mark says which are owed, so each chat's worker derives it from its journal -// (`structured-agent-session-reconciliation-pass.ts`). The scan only wakes the workers. +// stored "settled" mark says which are owed, so the host's retry derives each from its journal +// (`structured-agent-session-reconciliation-pass.ts`). The scan only wakes the retry. // // It never decides a lease latched in recovery: that signals a process that may still run, which // only the visible-tab restore, a start or an attach does, as on every build before this one. -// Their release then wakes the worker again. A worker replays a closed chat in a background slot as +// Their release then wakes the retry again. The retry replays a closed chat in a background slot as // its own read, published to no status surface, and takes the chat's lane only to write, so a send, // start or read of a chat late in the scan never waits behind it, and an opened chat goes first. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts index 56a1da6f5dd1..cfd68b683c17 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-stop-event-binding.test.ts @@ -122,7 +122,7 @@ async function stopOfStart(options: { held?: true } = {}): Promise { const handedOver = rig.dispatch.mock.calls.length - const mail = rig.send('mail for the worker') + const mail = rig.send('mail for the retry') await mail.result await rig.proveStart() await eventually(() => expect(rig.dispatch).toHaveBeenCalledTimes(handedOver + 1)) @@ -167,7 +167,7 @@ describe('a Stop of a start that never landed binds no later turn', () => { it('reads a mail turn the child end cut, with no verdict of its own, as news', async () => { await stopOfStart() - const mail = rig.send('mail for the worker') + const mail = rig.send('mail for the retry') await mail.result await turnOpenedBy(mail.id) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts index f0237835c756..2bc2ed3ab304 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-task-queue.ts @@ -8,6 +8,11 @@ export class StructuredAgentSessionTaskQueue { return runKeyedSerializedOperation(this.chains, sessionId, task) } + /** Whether anything holds or waits for the chat's lane now. */ + busy(sessionId: string): boolean { + return this.chains.has(sessionId) + } + trackAttach(operation: Promise): Promise { this.attaching.add(operation) void operation.then( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts index f8bc1afa6716..65091e8eee71 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-wedged-profile-migration.test.ts @@ -58,6 +58,7 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import { NO_STRUCTURED_AGENTS } from './structured-agent-session-adapter-router-test-support' +import { retryIdle } from './structured-agent-session-retry.test-fixture' const CALLER = { callerKey: 'client-1' } const DEAD_OWNER: AgentSessionProcessIdentity = { @@ -150,9 +151,9 @@ function openHost(overrides: Partial = {}): void }) } -/** Startup as the runtime runs it: the restart reconcile, every chat's worker in the background, - * and the boot sweep that opens each chat for its readers once that worker closed it again. A - * lease latched in recovery is decided by the sweep, whose release wakes the worker again. */ +/** Startup as the runtime runs it: the restart reconcile, the retry's visit to every chat in the + * background, and the boot sweep that opens each chat for its readers once the retry closed it. A + * lease latched in recovery is decided by the sweep, whose release wakes the retry again. */ async function startUp(): Promise { await host.reconcileRestartLeases() await host.startupSettled() @@ -162,9 +163,9 @@ async function startUp(): Promise { await reconciled() } -/** The chat's reconciliation worker has retired, or waits out a backoff. */ +/** The host's retry has retired, or waits out a backoff. */ function reconciled(): Promise { - return host.collaboratorsForTests().reconciliation.idle(SESSION) + return retryIdle(host.collaboratorsForTests().reconciliation, SESSION) } /** The host starting the agent with no message to deliver, as an operation that needs it does. */ @@ -405,7 +406,7 @@ describe('already-wedged profiles become usable on load', () => { const fence = stale.ok ? 13 : (stale.refusal.currentFence ?? 13) expect(stale.ok || stale.refusal.code === 'agent_session_checkpoint_stale').toBe(true) expect(stale.ok || (await host.attach(CALLER, hostTestAttachParams(fence))).ok).toBe(true) - // The reservation over the proven-dead owner ended its generation: the worker settles it. + // The reservation over the proven-dead owner ended its generation: the retry settles it. await reconciled() expect(acquire).toHaveBeenCalledOnce() @@ -488,7 +489,7 @@ describe('already-wedged profiles become usable on load', () => { : { outcome: 'pid-absent' }, stopOwnerProcess }) - // The worker's settlement fails until storage takes it again; it backs off and retries. + // The retry's settlement fails until storage takes it again; it backs off and retries. const failing = vi .spyOn(JournalLifecycleBatchAppender.prototype, 'appendPlanned') .mockRejectedValue(new Error('journal unavailable')) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts new file mode 100644 index 000000000000..807d4fb7f10d --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts @@ -0,0 +1,53 @@ +// Another connection's write lock, held from its own thread, and a watch on the main thread while +// this one meets it: for tests of bookkeeping that must fail at once instead of waiting it out. + +import { Worker } from 'node:worker_threads' +import { journalDatabasePath } from '../agent-session-journal/journal-host-database' + +const lockers = new Set() + +/** Another connection, on its own thread, holds the write lock for `ms`: this thread's waits + * cannot release it, as another process's could not. Resolves once the lock is held. */ +export function holdWriteLock( + stateDirectory: string, + ms: number +): Promise<{ released: Promise }> { + const worker = new Worker( + `const { workerData, parentPort } = require('node:worker_threads') + const { DatabaseSync } = require('node:sqlite') + const db = new DatabaseSync(workerData.file) + db.exec('BEGIN IMMEDIATE') + parentPort.postMessage('locked') + setTimeout(() => { db.exec('ROLLBACK'); db.close(); parentPort.postMessage('released') }, workerData.ms)`, + { eval: true, workerData: { file: journalDatabasePath(stateDirectory), ms } } + ) + lockers.add(worker) + let release: () => void = () => undefined + const released = new Promise((resolve) => (release = resolve)) + return new Promise((resolve) => + worker.on('message', (message) => (message === 'locked' ? resolve({ released }) : release())) + ) +} + +/** Ends every lock a test still holds. */ +export async function releaseWriteLocks(): Promise { + await Promise.all([...lockers].map((worker) => worker.terminate())) + lockers.clear() +} + +/** The longest the main thread went without running a timer, while `run` ran. */ +export async function longestStall(run: () => Promise): Promise { + let last = performance.now() + let longest = 0 + const probe = setInterval(() => { + const now = performance.now() + longest = Math.max(longest, now - last) + last = now + }, 2) + try { + await run() + return Math.max(longest, performance.now() - last) + } finally { + clearInterval(probe) + } +} diff --git a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts index ca5f9ca393e3..25bbf9905678 100644 --- a/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts +++ b/src/main/native-chat/agent-session-wire/structured-rewind-recovery.ts @@ -16,11 +16,21 @@ import { rewindRefusal } from './structured-rewind-refusal' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' import type { JournalOperationReceipt } from '../agent-session-journal/journal-row-writer' import type { AgentJournalCursor } from '../../../shared/agent-session-journal-types' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' +import type { + BackgroundJournalWrites, + BackgroundStoreWrites +} from './structured-agent-session-background-writes' -type RewindRecoveryDeps = { store: AgentSessionRecordStore; logger: StructuredAgentSessionLogger } +type RewindRecoveryDeps = { + store: AgentSessionRecordStore + logger: StructuredAgentSessionLogger + /** The retry's own recovery writes through its handles: another connection's lock throws. */ + background?: { store: BackgroundStoreWrites; journal: BackgroundJournalWrites } +} export function persistRewindRecord( - store: AgentSessionRecordStore, + store: Pick, sessionId: string, fence: number, rewind: AgentSessionRewindRecord @@ -38,12 +48,13 @@ export function persistRewindRecord( * settled refused rather than proven. Bookkeeping only: the chat is already attached either way. */ async function settleUnsupportedClaudeRewind( - { store, logger }: RewindRecoveryDeps, + { store: reads, logger, background }: RewindRecoveryDeps, sessionId: string, fence: number, rewind: AgentSessionRewindRecord ): Promise { const refusal = rewindRefusal('unsupported').refusal + const store = background?.store ?? reads try { await persistRewindRecord(store, sessionId, fence, { ...rewind, @@ -57,6 +68,9 @@ async function settleUnsupportedClaudeRewind( outcome: { status: 'failed', code: refusal.code, rewindReason: 'unsupported' } }) } catch (error) { + if (background && isSqliteContentionFailure(error)) { + throw error + } logger.warn('a pending Claude rewind was not settled', { scope: 'rewind-unsupported-settlement', sessionId, @@ -91,8 +105,10 @@ export async function recoverStructuredRewind( cursor: AgentJournalCursor ) => JournalOperationReceipt ): Promise { - const { store } = deps - let rewind = store.getRecord(sessionId)?.rewind + const { store: reads } = deps + const store = deps.background?.store ?? reads + const writes = deps.background?.journal ?? journal + let rewind = reads.getRecord(sessionId)?.rewind if (rewind?.phase !== 'provider-succeeded' && rewind?.phase !== 'prepared') { return } @@ -167,7 +183,7 @@ export async function recoverStructuredRewind( const replacement = rewind.retained.map(retainedRowReplacement) if (rewind.contextClearOperationId && rewind.contextClearSequence) { if ( - store.getRecord(sessionId)?.providerContextBoundary?.operationId !== + reads.getRecord(sessionId)?.providerContextBoundary?.operationId !== rewind.contextClearOperationId || journal.cursor().epoch !== rewind.expectedEpoch || journal.context.floor()?.sequence !== rewind.contextClearSequence @@ -175,13 +191,13 @@ export async function recoverStructuredRewind( throw new Error('agent_session_rewind:stale-context') } const completed = rewind - await journal.context.rewind( + await writes.context.rewind( { epoch: completed.expectedEpoch, sequence: completed.contextClearSequence! }, fence, replacement, (cursor) => receipt?.(completed, fence, cursor) ?? - store.conversationReceipts.rewind(sessionId, fence, completed, cursor) + reads.conversationReceipts.rewind(sessionId, fence, completed, cursor) ) return } @@ -201,7 +217,7 @@ export async function recoverStructuredRewind( } const cursor = alreadyReplaced ? journal.cursor() - : await journal.replaceEpochItems('handle_forked', fence, replacement) + : await writes.replaceEpochItems('handle_forked', fence, replacement) await persistRewindRecord(store, sessionId, fence, { ...rewind, phase: 'completed', diff --git a/src/main/runtime/agent-session-generation-end.test.ts b/src/main/runtime/agent-session-generation-end.test.ts index 30ef76df9fda..17dffef026f4 100644 --- a/src/main/runtime/agent-session-generation-end.test.ts +++ b/src/main/runtime/agent-session-generation-end.test.ts @@ -118,6 +118,7 @@ describe('a generation that ends', () => { await releaseStoredStructuredAgentSessionOwnerAfterExit({ store, + writes: store, sessionId: SESSION, expectedFence: fence, now: NOW + 1_000, @@ -251,6 +252,7 @@ describe('a write that ends nothing', () => { const first = await liveOwner() await releaseStoredStructuredAgentSessionOwnerAfterExit({ store, + writes: store, sessionId: SESSION, expectedFence: first.fence, now: NOW + 1_000, diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index 80e0b949cc55..ce2647a92a7e 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -315,8 +315,8 @@ export class AgentSessionRecordStore { claimAfter: ClaimAfterAdmission ) => this.transact((draft) => admitAndClaimAgentSessionOperationInto(draft, args, claimAfter)) - recordOperationOutcome = (args: AgentSessionOperationSettlement): Promise => - this.transact((draft) => settleAgentSessionOperationInto(draft, args)) + recordOperationOutcome = (args: AgentSessionOperationSettlement, options?: JournalWriteOptions) => + this.transact((draft) => settleAgentSessionOperationInto(draft, args), options) /** The same settlement, committed by the journal write that makes it true. It changes only the * ledger, so no record listener is owed. */ diff --git a/src/main/runtime/agent-session-surface-release-transition.ts b/src/main/runtime/agent-session-surface-release-transition.ts index 0c9906b624c6..1099e8d6c045 100644 --- a/src/main/runtime/agent-session-surface-release-transition.ts +++ b/src/main/runtime/agent-session-surface-release-transition.ts @@ -8,7 +8,6 @@ // The fence still moves, so the next owner is a new generation: an attach or settlement still // holding the stopped owner's fence is refused as stale rather than acting on its successor. -import type { JournalWriteOptions } from '../native-chat/agent-session-journal/journal-database' import { agentSessionRefusalError } from '../../shared/agent-session-wire-refusals' import { MAX_AGENT_SESSION_DEATH_DETAIL_CHARS, @@ -73,12 +72,9 @@ export function releaseStoredAgentSessionOwnerAfterSurfaceClose( now: number exitObservedAt?: number exitReason?: string - }, - options?: JournalWriteOptions + } ): Promise { - return store.transitionHandoff( - args.sessionId, - (record) => releaseAgentSessionOwnerAfterSurfaceClose({ ...args, record }), - options + return store.transitionHandoff(args.sessionId, (record) => + releaseAgentSessionOwnerAfterSurfaceClose({ ...args, record }) ) } From 28cd5214a39029613228bcd46591f152db8e3d82 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:39:48 -0700 Subject: [PATCH 13/22] fix(native-chat): startup bookkeeping fails fast; a dead turn reads interrupted before its cleanup lands - the restore's reconcile, recovery resolution and tab-index seed write as background work, and the retry's reconcile resolves recovery the restore had to skip - a turn whose agent this host saw exit reads interrupted, and the queued card offers Send, from the moment of the exit rather than once its cleanup is written --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + ...ured-agent-session-background-lock.test.ts | 69 +++++++++++- ...uctured-agent-session-background-writes.ts | 4 +- .../structured-agent-session-child-exit.ts | 7 +- ...uctured-agent-session-current-work.test.ts | 3 +- .../structured-agent-session-current-work.ts | 34 +++++- ...red-agent-session-dead-turn-frames.test.ts | 106 ++++++++++++++++++ ...ctured-agent-session-host-runtime-state.ts | 7 +- .../structured-agent-session-host-tabs.ts | 4 +- .../structured-agent-session-host.ts | 2 +- ...structured-agent-session-provider-child.ts | 7 ++ ...agent-session-reconciliation-retry.test.ts | 2 +- ...ured-agent-session-reconciliation-retry.ts | 22 +++- ...tured-agent-session-recovery-resolution.ts | 29 +++-- ...uctured-agent-session-restart-reconcile.ts | 15 ++- .../structured-agent-session-reveal.ts | 46 +++++--- ...agent-session-status-journal-projection.ts | 29 ++++- ...d-agent-session-write-lock.test-fixture.ts | 29 +++-- .../runtime/agent-session-record-store.ts | 4 +- ...structured-agent-session-tab-index-seed.ts | 4 +- ...uctured-agent-session-queued-cards.test.ts | 15 +++ .../structured-agent-session-queued-cards.ts | 16 ++- ...tructured-agent-session-queued-messages.ts | 9 +- 23 files changed, 391 insertions(+), 73 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index e7f8be4e3fb0..6200aac2036e 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -124,6 +124,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-exit-delivery.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts index 3a3926d5d8b2..30d04e723627 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts @@ -13,6 +13,8 @@ import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-te import { exitChild, leaveUnfinishedWork, + REASONING, + TURN_KEY, turnState } from './structured-agent-session-leftover-settlement.test-fixture' import { @@ -23,7 +25,8 @@ import { retryOwes } from './structured-agent-session-retry.test-fixture' import { holdWriteLock, longestStall, - releaseWriteLocks + releaseWriteLocks, + stallsOver } from './structured-agent-session-write-lock.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -57,6 +60,70 @@ describe('under another connection’s write lock', () => { ) }, 20_000) + it("an agent dying mid-row costs one wait, its last row's, and the exit's bookkeeping none", async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let sinkFailedAt = Infinity + vi.spyOn(console, 'error').mockImplementation((message: unknown) => { + if (String(message).includes('writing provider events to the chat journal failed')) { + sinkFailedAt = performance.now() + } + }) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + // Longer than a person's write waits (`JOURNAL_BUSY_TIMEOUT_MS`), so the row's write gives up. + await holdWriteLock(current.root, 6_500) + + const stalls = await stallsOver(60, async () => { + // The agent's last words are a person's content: written as on main, waiting its 5 s. + current + .providerEvents() + .appendItem( + REASONING, + { kind: 'message', role: 'assistant', blocks: [{ type: 'text', text: 'last words' }] }, + { turnScope: { kind: 'turn', turnItemId: TURN_KEY } } + ) + await exitChild(current) + }) + + // One wait, the row's own busy timeout (5 s, plus SQLite's sleep granularity on a loaded + // machine), and it is the row's: the sink reports its failure as that wait ends. + expect(stalls).toHaveLength(1) + const [wait] = stalls + expect(wait!.ms).toBeLessThanOrEqual(5_500) + expect(sinkFailedAt).toBeGreaterThanOrEqual(wait!.endedAt - wait!.ms) + expect(sinkFailedAt).toBeLessThanOrEqual(wait!.endedAt) + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 10_000 }) + await vi.waitFor( + () => expect(current.store.getRecord(SESSION)?.lease.claimStatus).toBe('released'), + { timeout: 8_000 } + ) + }, 30_000) + + it('a relaunch under the lock never holds the main thread, and its startup lands after', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + + // Another process holds the lock from before this one starts: the restart reconcile, the + // restore's reconcile and recovery, and the startup scan all meet it. + expect( + await longestStall(() => + current.crashRestartHostProcess(() => holdWriteLock(current.root, LOCK_MS)) + ) + ).toBeLessThan(60) + // Once it lifts, the retry reconciles, resolves the recovery the restore could not, and settles + // the turn as an unlocked startup does: the rig's probe proves nothing, so unverifiable. + await vi.waitFor(() => expect(turnState(current)).toBe('unverifiable'), { timeout: 8_000 }) + expect(current.store.getRecord(SESSION)?.lease).toMatchObject({ + unreconciled: false, + handoffStage: null + }) + }, 20_000) + it('a pending rewind the retry recovers ends its round at once, never holding the lane', async () => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) rig = await createQueuedMessageTestRig({ restartable: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts index 60b38a13740e..b1e8dbae5e01 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.ts @@ -9,7 +9,9 @@ import type { JournalContextController } from '../agent-session-journal/journal- import type { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -const BACKGROUND = { background: true } as const +/** Bookkeeping's write options, for a call that takes them directly. */ +export const BACKGROUND_WRITE = { background: true } as const +const BACKGROUND = BACKGROUND_WRITE type Background = { readonly background: true } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts index 16774b3e0950..47312959a38e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-child-exit.ts @@ -15,6 +15,7 @@ import type { StructuredAgentSessionProviderChild } from './structured-agent-session-host-types' import { recordProviderChildEnd } from './structured-agent-session-provider-child' +import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { backgroundLeaseWrites, backgroundSettlementWrites @@ -255,6 +256,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // to the retry, the settlement as the exit's debt and the release as its repair. let released = false let owed: StructuredAgentSessionExitSettlement | null = null + let contended = false if (settlement) { const settled = await settleStructuredAgentSessionLeftovers({ store: context.store, @@ -267,6 +269,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< }) if (!settled.ok) { owed = settlement + contended = isSqliteContentionFailure(settled.error) logExitFailure(context, sessionId, 'exit-settlement', settled.error) } } @@ -282,6 +285,7 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< }) released = true } catch (error) { + contended ||= isSqliteContentionFailure(error) logExitFailure(context, sessionId, 'exit-owner-release', error) } if (context.route) { @@ -301,7 +305,8 @@ export async function endExitedStructuredAgentSessionChildUnderSerialize< // and the queued-card drain runs, whether or not the release or the settlement was written. context.generationEnded(sessionId, { restate: released && !expected, - ...(owed ? { exit: owed } : {}) + ...(owed ? { exit: owed } : {}), + ...(contended ? { contended } : {}) }) context.wakeDelivery?.(sessionId) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts index 1554fac43f9b..6f2f3c25b472 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts @@ -129,7 +129,8 @@ describe("a dead generation's leftovers, its settlement refused and its lease re expect(current.host.readStatusSummary(SESSION)?.status).not.toBe('working') // A client hydrating now, and the one already watching, read the same. const page = await hydration(current) - expect(page.latestTurn).toBeNull() + // Read interrupted, as its settlement will write it: never running, never finished. + expect(page.latestTurn?.turn.state).toBe('interrupted') expect(page.actionablePromptIds).toEqual([]) await vi.waitFor(() => expect(view.chat().actionablePromptIds).toEqual([])) expect(runningStructuredAgentSessionTurnId(view.chat())).toBeNull() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index bcb1fc0c6a96..a5ac9dc56ace 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -33,11 +33,13 @@ export type StructuredAgentSessionCurrentWorkJournal = Pick< export type StructuredAgentSessionWorkEvidence = { record: Pick | null /** The last child this host saw end (`StructuredAgentSessionHostSession.lastEndedChild`). */ - ended?: { fence: number; rootGone?: boolean } + ended?: StructuredAgentSessionSeenEnd /** `StructuredAgentSessionHostSession.operationalRevision`. */ revision?: number } +type StructuredAgentSessionSeenEnd = { fence: number; rootGone?: boolean; cause?: string } + /** The live or reserved generation's fence, or null when no generation is live. With no record * nothing has ended: fence 0, under which every item is current, as before records scoped work. */ export function structuredAgentSessionLiveFence( @@ -82,7 +84,8 @@ export class StructuredAgentSessionCurrentWork { readonly liveFence: number | null, /** Moves with every generation end this host saw, lease write or not: a reader that caches * what it derived from this answer keys it here and on the journal's cursor. */ - readonly revision = 0 + readonly revision = 0, + private readonly ended?: StructuredAgentSessionSeenEnd ) {} /** What a reader that published this answer compares to learn it changed with no row: the live @@ -165,10 +168,30 @@ export class StructuredAgentSessionCurrentWork { return this.activeTurnId() !== null || this.owesAnySend() } + /** The running turn of the generation this host saw die on its own (its root gone, nobody asked): + * that death's settlement writes it interrupted, so it reads so before that lands. */ + diedTurn(): { item: AgentJournalRenderItem; turnId: string } | null { + const running = this.journal.runningTurn() + const { ended } = this + return running && + ended?.rootGone === true && + ended.cause === 'exit' && + this.journal.itemFence(running.item.itemId) === ended.fence && + !this.isCurrentItem(running.item.itemId) + ? running + : null + } + /** The newest turn record as a client is told it: a running one an ended generation opened is - * none, so no client reads it as working or steers into it. */ + * none, or interrupted when it died on its own (`diedTurn`), so no client reads it as working, + * steers into it or reads it finished. */ publishedLatestTurn(latest: AgentSessionLatestTurn | null): AgentSessionLatestTurn | null { - return latest?.turn.state === 'running' && !this.isCurrentItem(latest.itemId) ? null : latest + if (latest?.turn.state !== 'running' || this.isCurrentItem(latest.itemId)) { + return latest + } + return this.diedTurn()?.item.itemId === latest.itemId + ? { ...latest, turn: { ...latest.turn, state: 'interrupted' } } + : null } } @@ -179,7 +202,8 @@ export function structuredAgentSessionCurrentWork( return new StructuredAgentSessionCurrentWork( journal, structuredAgentSessionLiveFence(evidence), - evidence.revision + evidence.revision, + evidence.ended ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts new file mode 100644 index 000000000000..1e02f9b1ff25 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts @@ -0,0 +1,106 @@ +// An agent that dies while another connection holds the database: its exit's settlement fails at +// once and waits for the retry, so the journal still says the turn runs. Until the settlement +// lands, every frame a reader gets reads that turn as the settlement will write it, interrupted: +// the chat's latest turn, the session list's row (which the tab and sidebar show), and the host's +// working answer that decides a queued card's Send. Never Working, never a finish with no verdict. + +import { afterEach, expect, it, vi } from 'vitest' +import type { AgentSessionStatusSummary } from '../../../shared/agent-session-wire' +import { runningStructuredAgentSessionTurnId } from '../../../shared/structured-agent-session-live-turn' +import { + EMPTY_STRUCTURED_AGENT_SESSION, + reduceStructuredAgentSession, + type StructuredAgentSessionState +} from '../../../shared/structured-agent-session-reducer' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + exitChild, + leaveUnfinishedWork, + turnState +} from './structured-agent-session-leftover-settlement.test-fixture' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + holdWriteLock, + releaseWriteLocks +} from './structured-agent-session-write-lock.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await releaseWriteLocks() + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +/** Every frame a chat client and the session list receive, as each reduces it. */ +function recordFrames(current: QueuedMessageTestRig) { + const chats: StructuredAgentSessionState[] = [] + const rows: AgentSessionStatusSummary[] = [] + let chat = EMPTY_STRUCTURED_AGENT_SESSION + current.host.subscribe({ + id: 'chat-1', + sessionId: SESSION, + emit: (event) => { + chat = reduceStructuredAgentSession(chat, { type: 'event', event: structuredClone(event) }) + chats.push(chat) + } + }) + current.host.subscribeStatus({ + id: 'list-1', + emit: (event) => { + const row = + event.type === 'status' + ? event.session + : event.type === 'snapshot' + ? event.sessions.find((session) => session.sessionId === SESSION) + : undefined + if (row?.sessionId === SESSION) { + rows.push(structuredClone(row)) + } + } + }) + return { chats, rows } +} + +it('reads the dead turn interrupted in every frame until its settlement lands', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await current.send('after this', 'queue-if-active').result + await leaveUnfinishedWork(current) + const frames = recordFrames(current) + await vi.waitFor(() => expect(frames.chats.at(-1)?.working).toBe(true)) + const { released } = await holdWriteLock(current.root, 1_200) + const before = { chats: frames.chats.length, rows: frames.rows.length } + + await exitChild(current) + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + await released + // Nothing landed: what the frames read is derived from the death alone. + expect(turnState(current)).toBe('running') + + const chats = frames.chats.slice(before.chats) + const rows = frames.rows.slice(before.rows) + expect(chats.length).toBeGreaterThan(0) + expect(rows.length).toBeGreaterThan(0) + for (const chat of [...chats, page.ok ? page.page : null]) { + expect(chat?.latestTurn?.turn.state).toBe('interrupted') + expect(chat?.working).toBe(false) + // So no turn runs to steer into: the card's control is Send. + expect(chat?.nextQueuedMessageId ?? null).toBeNull() + } + for (const chat of chats) { + expect(runningStructuredAgentSessionTurnId(chat)).toBeNull() + } + for (const row of rows) { + expect(row).toMatchObject({ status: 'idle', turnOutcome: 'interruption' }) + } + + // Once the lock lifts, the retry writes what the frames already said. + await vi.waitFor(() => expect(turnState(current)).toBe('interrupted'), { timeout: 8_000 }) +}, 20_000) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts index d604b805eced..201d057bbd3f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-runtime-state.ts @@ -14,6 +14,7 @@ import { } from './structured-agent-session-provider-child' import { StructuredAgentSessionOptionRevisions } from './structured-agent-session-option-revisions' import { resolveStructuredSessionRecovery } from './structured-agent-session-recovery-resolution' +import type { StructuredAgentSessionLeaseWrites } from './structured-agent-session-lease-release' import { recordAgentSessionStartup } from '../../observability/agent-session-instrumentation' import { StructuredAgentSessionStartupAttempts, @@ -154,10 +155,14 @@ export class StructuredAgentSessionHostRuntimeState { } /** Exit from a latched recovery stage when present-time evidence permits one. */ - resolveRecovery(sessionId: string): Promise<'resolved' | 'unresolved' | 'not-applicable'> { + resolveRecovery( + sessionId: string, + writes?: StructuredAgentSessionLeaseWrites + ): Promise<'resolved' | 'unresolved' | 'not-applicable'> { return resolveStructuredSessionRecovery( { store: this.deps.store, + ...(writes ? { writes } : {}), probeRecord: (record) => this.probeRecord(record), now: () => this.deps.now?.() ?? Date.now(), ...(this.deps.stopOwnerProcess ? { stopOwnerProcess: this.deps.stopOwnerProcess } : {}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts index 2fc2bc28d735..d007029296c3 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-tabs.ts @@ -1,6 +1,7 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' +import type { JournalWriteOptions } from '../agent-session-journal/journal-database' /** * Chat-tab visibility is the deletion funnel: every path that removes a chat as a user-facing @@ -97,7 +98,8 @@ export function createStructuredAgentSessionTabSurface( getPersistedVisibleSessionTabIndex: () => host.deps.store.getVisibleSessionTabIndex(), getSessionTabId: (sessionId: string): string | null => host.deps.store.getSessionTabId(sessionId), - showSessionTabs: (sessionIds: readonly string[]) => host.deps.store.showSessionTabs(sessionIds), + showSessionTabs: (sessionIds: readonly string[], options?: JournalWriteOptions) => + host.deps.store.showSessionTabs(sessionIds, options), notifySessionTabHidden: (sessionId: string): void => notifyTabHidden(host, sessionId), setSessionTabVisibility: async ( sessionId: string, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts index c9a57d137529..67609ecc51c9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host.ts @@ -130,7 +130,7 @@ export class StructuredAgentSessionHost { }) this.restore = reveal.createStructuredAgentSessionHostRestore(deps, { reconcileLeases: this.reconcileLeases, - resolveRecovery: (sessionId) => this.runtimeState.resolveRecovery(sessionId), + resolveRecovery: (sessionId, writes) => this.runtimeState.resolveRecovery(sessionId, writes), serialize: (sessionId, task) => this.serialize(sessionId, task), hasSession: this.hasSession, // Site 10: cannot overwrite a live entry — the restorer returns early on diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts index 080a70dfb7ff..8f9d2e42ea42 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-provider-child.ts @@ -113,6 +113,13 @@ export function recordProviderChildEnd( return true } +/** The child's exit is still settling: it ended, and its exit has not let it go (`endChild`). */ +export function providerChildExitSettling( + session: Pick | undefined +): boolean { + return session?.child != null && session.lastEndedChild?.fence === session.child.fence +} + /** The conversation's last start died before it proved itself, and nothing started since. Only a * send retries it: a view or an exit recovery would respawn into the same failure, adding a row. */ export function failedProviderChildStart( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts index f4987835ca92..f78f530a2e27 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts @@ -159,7 +159,7 @@ describe('an observed exit whose release write failed', () => { type: 'batch', batch: { items: [], cursor }, working: false, - latestTurn: null + latestTurn: { turn: { state: 'interrupted' } } }) ) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts index 4632eb001557..e5e9c78f4020 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -30,6 +30,7 @@ import { reconciliationBackoffDelay } from './structured-agent-session-reconciliation-backoff' import { StructuredAgentSessionReconciliationSlots } from './structured-agent-session-reconciliation-slots' +import { providerChildExitSettling } from './structured-agent-session-provider-child' import { dropStructuredAgentSessionLoaded, noteStructuredAgentSessionOpened, @@ -49,6 +50,8 @@ export type StructuredAgentSessionReconciliationSignal = { exit?: StructuredAgentSessionExitSettlement /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ restate?: boolean + /** The exit's own settlement met another connection's lock. */ + contended?: boolean } export class StructuredAgentSessionRetry { @@ -104,6 +107,10 @@ export class StructuredAgentSessionRetry { if (chat && signal.exit) { chat.debts.exit = signal.exit } + if (chat && signal.contended) { + // The chat's queued send would meet the same lock: it waits for the round, which sends it. + chat.send = true + } this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) this.freshEpisode() } @@ -120,13 +127,17 @@ export class StructuredAgentSessionRetry { } /** A reader opened the chat: its handle marks the process boundary if it is the first, and what - * it owes goes ahead of the scan, in a fresh episode. */ + * it owes goes ahead of the scan, in a fresh episode unless a backoff runs. */ opened = (sessionId: string, journal: Pick): void => { noteStructuredAgentSessionOpened(this.firstOpened, sessionId, journal) const chat = this.owed.get(sessionId) if (chat) { this.slots.prioritize(chat) - this.freshEpisode() + // Backing off after a failed round, it goes first in the next, which an open does not + // hurry: a store that keeps refusing costs a write per round, never one per chat opened. + if (this.failures === 0 || this.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { + this.freshEpisode() + } } } @@ -141,8 +152,11 @@ export class StructuredAgentSessionRetry { } } - /** Whether the chat's queued send waits for a round (or for a fresh episode, given up). */ - sendWaits = (sessionId: string): boolean => this.owed.get(sessionId)?.send === true + /** Whether the chat's queued send waits for a round (or for a fresh episode, given up): it met a + * lock, or the exit it follows is still settling. */ + sendWaits = (sessionId: string): boolean => + this.owed.get(sessionId)?.send === true || + providerChildExitSettling(this.context.sessions.get(sessionId)) /** Resolves once the chat's next visit finished, or at once when it owes nothing. */ attempted = (sessionId: string): Promise => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts index 257072f7c23b..7c2eb4af060d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovery-resolution.ts @@ -17,13 +17,19 @@ import { } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { PROVIDER_SUPERVISOR_MAX_STOP_MS } from '../../provider-process/provider-process-supervisor' -import { releaseUnprovenAgentSessionOwner } from '../../runtime/agent-session-lease-transitions' +import { + evictAgentSessionOwner, + releaseUnprovenAgentSessionOwner +} from '../../runtime/agent-session-lease-transitions' import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import type { StructuredAgentSessionLeaseWrites } from './structured-agent-session-lease-release' export type StructuredSessionRecoveryStopSignal = 'SIGTERM' | 'SIGKILL' export type StructuredSessionRecoveryResolutionDeps = { store: AgentSessionRecordStore + /** The store itself, or a startup restore's background handle (`backgroundLeaseWrites`). */ + writes?: StructuredAgentSessionLeaseWrites probeRecord: (record: AgentSessionRecord) => Promise now: () => number stopOwnerProcess?: (pid: number, signal: StructuredSessionRecoveryStopSignal) => void @@ -69,21 +75,14 @@ export async function resolveStructuredSessionRecovery( probe = await stopOwnerAndReprobe(deps, record, owner.pid) } } + const expectedFence = record.lease.runtimeFence + const now = deps.now() try { - await (owner && !isProvenDeadProbe(probe) - ? deps.store.transitionHandoff(sessionId, (latest) => - releaseUnprovenAgentSessionOwner({ - record: latest, - expectedFence: record.lease.runtimeFence, - now: deps.now() - }) - ) - : deps.store.evictProvenDeadOwner({ - sessionId, - expectedFence: record.lease.runtimeFence, - probe, - now: deps.now() - })) + await (deps.writes ?? deps.store).transitionHandoff(sessionId, (latest) => + owner && !isProvenDeadProbe(probe) + ? releaseUnprovenAgentSessionOwner({ record: latest, expectedFence, now }) + : evictAgentSessionOwner({ record: latest, expectedFence, probe, now }) + ) return 'resolved' } catch (error) { const code = error instanceof Error ? error.message : String(error) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts index 0ac228fc5f16..6fbbe5b14b1e 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile.ts @@ -27,14 +27,18 @@ export function createRestartReconciler(deps: { options?: JournalWriteOptions ) => Promise { let pending: Promise | null = null + let background: Promise | null = null return async (sessionId, options) => { if (!deps.store.listRecords().some((record) => record.lease.unreconciled)) { return null } - // Background bookkeeping joins a reader's run, never the reverse: no reader inherits its - // refusal to wait for another connection's lock. + // Background bookkeeping joins a person's run, or another background one, never the reverse: + // no person's run inherits its refusal to wait for another connection's lock. if (options?.background && !pending) { - await reconcileCurrentLeases(deps, options) + background ??= reconcileCurrentLeases(deps, options).finally(() => { + background = null + }) + await background return null } if (!pending) { @@ -83,9 +87,8 @@ export function reportEachFailureOnce( } } -/** The reconcile a reader runs, at startup, before each restored read and in the reconciliation - * worker: it never throws, since an unreconciled lease grants no writer and the next send - * reconciles again before it acts. +/** The reconcile a reader runs, at startup and before each restored read: it never throws, since + * an unreconciled lease grants no writer and the next send reconciles again before it acts. * Answers whether every lease is settled. */ export function createReaderReconcile( reconcile: (sessionId: string) => Promise, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 7e1b8f0218a9..350903351b8b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -14,6 +14,11 @@ import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { journalOpenRefusal } from '../agent-session-journal/journal-open-failure' import type { JournalWriteOptions } from '../agent-session-journal/journal-host-database' +import { + BACKGROUND_WRITE, + backgroundLeaseWrites, + type BackgroundLeaseWrites +} from './structured-agent-session-background-writes' import { isSqliteContentionFailure } from '../../sqlite/sqlite-read-failure' import { StructuredAgentSessionReadableRestorer } from './structured-agent-session-readable-restorer' import { StructuredAgentSessionRestartRestoreGate } from './structured-agent-session-restart-restore-gate' @@ -74,7 +79,7 @@ export function createStructuredAgentSessionHostRestore( sessionId: string | null, options?: JournalWriteOptions ) => Promise - resolveRecovery: (sessionId: string) => Promise + resolveRecovery: (sessionId: string, writes: BackgroundLeaseWrites) => Promise startup: Pick & { tasks: Pick clientDelivery: Pick @@ -93,7 +98,13 @@ export function createStructuredAgentSessionHostRestore( } { const { reconcileLeases, resolveRecovery, startup, ...rest } = wiring const failures = reportEachFailureOnce(deps.logger) - const reconcile = createReaderReconcile(reconcileLeases, failures) + // Startup's and a restored read's bookkeeping, which neither waits on: another connection's lock + // fails it at once, and the retry's round tries it again. + const reconcile = createReaderReconcile( + (sessionId) => reconcileLeases(sessionId, BACKGROUND_WRITE), + failures + ) + const recoveryWrites = backgroundLeaseWrites(deps.store) const reconciliation = new StructuredAgentSessionRetry({ deps, sessions: startup.sessions, @@ -104,21 +115,26 @@ export function createStructuredAgentSessionHostRestore( track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => startup.clientDelivery.publishGenerationEnded(sessionId, options), - reconcile: () => - reconcileLeases(null, { background: true }).then( - (refusal) => { - if (!refusal) { - failures.clear() - return 'settled' - } + reconcile: async () => { + try { + const refusal = await reconcileLeases(null, BACKGROUND_WRITE) + if (refusal) { failures.report(refusal) return 'failed' - }, - (error: unknown) => { - failures.report(error) - return isSqliteContentionFailure(error) ? 'contended' : 'failed' } - ), + // What a restore whose own reconcile failed left latched: its open chats' recovery. + for (const { sessionId, lease } of deps.store.listRecords()) { + if (lease.handoffStage === 'recovering' && startup.sessions.has(sessionId)) { + await resolveRecovery(sessionId, recoveryWrites) + } + } + failures.clear() + return 'settled' + } catch (error) { + failures.report(error) + return isSqliteContentionFailure(error) ? 'contended' : 'failed' + } + }, sendQueued: (sessionId) => startup.queue.sendForRetry(sessionId), abandonSend: (sessionId) => startup.queue.abandon(sessionId) }) @@ -128,7 +144,7 @@ export function createStructuredAgentSessionHostRestore( reconcile, // The next attach or send resolves recovery again, strictly, before it acts. resolveRecovery: (sessionId) => - resolveRecovery(sessionId).then( + resolveRecovery(sessionId, recoveryWrites).then( () => true, (error: unknown) => { failures.report(error) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index 81f104bb0d6a..50e3aeb9dcb4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -108,11 +108,14 @@ export class StructuredAgentSessionJournalProjections { workRevision: work.revision, stopRevision, // The host's projection of current work: an ended generation's is not Working. - state: projectStructuredAgentSessionStatusState( - snapshot.items, - snapshot.submissions, - fence, - work.scope + state: withDiedTurnVerdict( + projectStructuredAgentSessionStatusState( + snapshot.items, + snapshot.submissions, + fence, + work.scope + ), + work.diedTurn() ), acceptedSendKey: newestAcceptedSendKey(cursor.epoch, snapshot.submissions), stopping: structuredAgentSessionStopping( @@ -127,3 +130,19 @@ export class StructuredAgentSessionJournalProjections { return projection } } + +/** A turn whose generation died on its own reads as its settlement writes it, interrupted, before + * that lands: never a finish with no verdict (`StructuredAgentSessionCurrentWork.diedTurn`). */ +function withDiedTurnVerdict( + state: StructuredAgentSessionStatusState, + died: { turnId: string } | null +): StructuredAgentSessionStatusState { + const request = state.latestRequest + return died && + state.summary.status === 'idle' && + request?.kind === 'turn' && + request.id === died.turnId && + request.turnState === 'running' + ? { ...state, summary: { ...state.summary, turnOutcome: 'interruption' } } + : state +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts index 807d4fb7f10d..8cbc919e6faf 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-write-lock.test-fixture.ts @@ -35,19 +35,34 @@ export async function releaseWriteLocks(): Promise { lockers.clear() } -/** The longest the main thread went without running a timer, while `run` ran. */ -export async function longestStall(run: () => Promise): Promise { +/** Each gap between the main thread's timer turns while `run` ran: how long, and when it ended. */ +async function timerGaps(run: () => Promise): Promise<{ ms: number; endedAt: number }[]> { let last = performance.now() - let longest = 0 - const probe = setInterval(() => { + const gaps: { ms: number; endedAt: number }[] = [] + const gap = (): void => { const now = performance.now() - longest = Math.max(longest, now - last) + gaps.push({ ms: now - last, endedAt: now }) last = now - }, 2) + } + const probe = setInterval(gap, 2) try { await run() - return Math.max(longest, performance.now() - last) + gap() + return gaps } finally { clearInterval(probe) } } + +/** The longest the main thread went without running a timer, while `run` ran. */ +export async function longestStall(run: () => Promise): Promise { + return Math.max(0, ...(await timerGaps(run)).map((gap) => gap.ms)) +} + +/** Every time the main thread went longer than `ms` without running a timer, while `run` ran. */ +export async function stallsOver( + ms: number, + run: () => Promise +): Promise<{ ms: number; endedAt: number }[]> { + return (await timerGaps(run)).filter((gap) => gap.ms > ms) +} diff --git a/src/main/runtime/agent-session-record-store.ts b/src/main/runtime/agent-session-record-store.ts index ce2647a92a7e..842dc210cd5c 100644 --- a/src/main/runtime/agent-session-record-store.ts +++ b/src/main/runtime/agent-session-record-store.ts @@ -155,8 +155,8 @@ export class AgentSessionRecordStore { /** Shows each session that still has a record, in one write: an index written part way would * read as complete at the next launch and drop the rest. */ - showSessionTabs(sessionIds: readonly string[]): Promise { - return this.transact((draft) => showAgentSessionTabs(draft, sessionIds)) + showSessionTabs(sessionIds: readonly string[], options?: JournalWriteOptions): Promise { + return this.transact((draft) => showAgentSessionTabs(draft, sessionIds), options) } listByScope(location: AgentSessionExecutionLocation): AgentSessionRecord[] { diff --git a/src/main/runtime/structured-agent-session-tab-index-seed.ts b/src/main/runtime/structured-agent-session-tab-index-seed.ts index 3e2245148a26..cd1063490211 100644 --- a/src/main/runtime/structured-agent-session-tab-index-seed.ts +++ b/src/main/runtime/structured-agent-session-tab-index-seed.ts @@ -3,6 +3,7 @@ // the next launch and drop the rest. With an index present, every restored chat is already listed. import type { StructuredAgentSessionHost } from '../native-chat/agent-session-wire/structured-agent-session-host' +import { BACKGROUND_WRITE } from '../native-chat/agent-session-wire/structured-agent-session-background-writes' type TabIndexHost = Pick< StructuredAgentSessionHost, @@ -26,7 +27,8 @@ export async function seedStructuredAgentSessionTabIndex( (sessionId) => opened.has(sessionId) && !listed.has(sessionId) ) if (unlisted.length > 0) { - await host.showSessionTabs(unlisted).catch((error: unknown) => + // Startup's bookkeeping: another connection's lock fails it at once, and the next launch seeds. + await host.showSessionTabs(unlisted, BACKGROUND_WRITE).catch((error: unknown) => host.deps.logger.warn('recording restored chat tabs failed', { scope: 'tab-index-seed', sessionIds: unlisted, diff --git a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts index 73a3c3ab1bed..3053d95ee586 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts @@ -171,6 +171,21 @@ describe('queued message cards', () => { expect(cards.map((card) => queuedMessageCardSteers(card))).toEqual([true, true]) }) + it('a card offers Send, not Steer, while the host says no turn runs; an older host keeps Steer', () => { + // An agent that died while its settlement waits: the host names no turn running. + const idle = projectQueuedMessageCards([draft('waiting', 1)], [], { + hasPendingPrompt: false, + agentWorking: false, + hostIdle: true + }) + expect(idle.map((card) => [card.hold, queuedMessageCardSteers(card)])).toEqual([ + ['turn', false] + ]) + // An older host gives no answer of its own: its cards steer as they always did. + const older = projectQueuedMessageCards([draft('waiting', 1)], [], IDLE) + expect(older.map((card) => queuedMessageCardSteers(card))).toEqual([true]) + }) + it("the header names the queue's pause while it holds a card, and none over cards Resume would not send", () => { const stopped = { reason: 'stopped' } as const const project = (messages: AgentSessionQueuedMessage[], queuePaused = true) => diff --git a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts index 4192b552d791..8d24463a33fe 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts @@ -39,6 +39,8 @@ export type QueuedMessageCard = { command?: true /** A command card while the agent works: it offers no send until the agent is idle. */ waitsForAgent?: true + /** No turn runs, by the host's own answer, so there is nothing to steer into. */ + agentIdle?: true pausedReason?: string returnedReason?: string | null /** The typed fact the returned card's submission settled with; read like its `rejection`. */ @@ -61,7 +63,13 @@ function queuedMessageCardText(body: AgentSessionQueuedMessage['body']): string export function projectQueuedMessageCards( queuedMessages: readonly AgentSessionQueuedMessage[] | null | undefined, submissions: readonly AgentJournalSubmission[], - session: { hasPendingPrompt: boolean; queuePaused?: boolean; agentWorking?: boolean } + session: { + hasPendingPrompt: boolean + queuePaused?: boolean + agentWorking?: boolean + /** The host said nothing runs; an older host never says, and its cards read as before. */ + hostIdle?: boolean + } ): QueuedMessageCard[] { const handedOff = handedOffQueuedMessageIds( submissions.filter((submission) => submission.dispatchState !== 'rejected') @@ -97,6 +105,7 @@ export function projectQueuedMessageCards( ...(session.agentWorking ? { waitsForAgent: true as const } : {}) } : {}), + ...(session.hostIdle ? { agentIdle: true as const } : {}), ...(message.pausedReason !== undefined ? { pausedReason: message.pausedReason } : {}), ...(message.returnedReason !== undefined ? { returnedReason: message.returnedReason } : {}), ...(message.returnedRejection !== undefined @@ -128,9 +137,10 @@ export function queuedMessagesQueuePause( } /** Steer names the mid-turn jump, also while the whole queue is paused; a card held on its own or - * returned is not waiting on the turn, so its action is plainly Send. A command never steers. */ + * returned is not waiting on the turn, nor is any card while no turn runs, so its action is + * plainly Send. A command never steers. */ export function queuedMessageCardSteers(card: QueuedMessageCard): boolean { - return card.hold !== 'paused' && card.hold !== 'returned' && !card.command + return card.hold !== 'paused' && card.hold !== 'returned' && !card.command && !card.agentIdle } /** The card Cmd/Ctrl+Enter steers: the newest one, unless it is a command, which never steers. */ diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts index b56d5b0eb916..6d638dea9d37 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts @@ -81,6 +81,8 @@ export function useStructuredAgentSessionQueuedMessages(args: { hasPendingPrompt: boolean /** A turn is running, whoever started it, or the queue is about to send its next card. */ isWorking: boolean + /** The host's own answer (`working`); absent from an older host. */ + hostWork?: { working?: boolean } /** This pane's sends without a host record yet show as sending cards. */ sending?: readonly StructuredAgentSessionPendingSend[] composerScopeKey: string | undefined @@ -88,6 +90,8 @@ export function useStructuredAgentSessionQueuedMessages(args: { }): StructuredAgentSessionQueuedMessagesController { const { composerScopeKey, enabled, hasPendingPrompt, mutate, queuedMessages, submissions } = args const { isWorking, queuePause } = args + // Only the host's own answer: an older host's cards steer as they always did. + const hostIdle = args.hostWork?.working === false && !isWorking const sending = args.sending ?? NO_SENDS const cards = useMemo( @@ -96,7 +100,8 @@ export function useStructuredAgentSessionQueuedMessages(args: { hasPendingPrompt, // A command card offers no send while the agent works. agentWorking: isWorking, - queuePaused: queuePause !== null + queuePaused: queuePause !== null, + hostIdle }), ...sendingQueuedMessageCards( pendingQueueSendsOnTheirWay( @@ -107,7 +112,7 @@ export function useStructuredAgentSessionQueuedMessages(args: { ) ) ], - [hasPendingPrompt, isWorking, sending, queuePause, queuedMessages, submissions] + [hasPendingPrompt, hostIdle, isWorking, sending, queuePause, queuedMessages, submissions] ) const cardsRef = useRef(cards) useEffect(() => { From ff5f1a8decbb5a235770b298e62b74b1c10df9c8 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:20:33 -0700 Subject: [PATCH 14/22] fix(native-chat): retries give up by time, and recovery stays with the tabs a restart restored - the retry gives up after about three minutes of failure rather than ten fast rounds - recovery the restore could not finish is retried each round, only for the chats it was asked to restore, so a chat only read in the background never has its agent stopped - the phone offers Send on a card when the host says nothing is running, like the desktop --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + .../MobileNativeChatQueuedMessages.test.tsx | 9 ++ .../MobileNativeChatQueuedMessages.tsx | 4 +- ...le-structured-queued-message-cards.test.ts | 13 +++ .../mobile-structured-queued-message-cards.ts | 5 + .../use-mobile-structured-agent-session.ts | 2 + ...bile-structured-queued-message-controls.ts | 5 + ...ured-agent-session-background-lock.test.ts | 19 +++ ...ed-agent-session-background-writes.test.ts | 12 +- ...ed-agent-session-reconciliation-backoff.ts | 10 +- ...ured-agent-session-reconciliation-retry.ts | 34 +++--- ...ured-agent-session-reconciliation-visit.ts | 6 +- ...ent-session-restore-recovery-scope.test.ts | 110 ++++++++++++++++++ ...uctured-agent-session-retry-budget.test.ts | 1 + .../structured-agent-session-reveal.ts | 36 ++++-- 15 files changed, 231 insertions(+), 36 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 6200aac2036e..a06b1dfa6bed 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -125,6 +125,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', diff --git a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx index b01a0fa66c18..22dd447d0e99 100644 --- a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx +++ b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx @@ -209,6 +209,15 @@ describe('MobileNativeChatQueuedMessages', () => { expect(nodeTypes(rows[1]!)).toContain('CornerDownRight') }) + it('reads Send, not Steer, on a waiting card while the host says nothing runs', async () => { + const mounted = await mount({ + cards: [card({ messageId: 'waiting', agentIdle: true })], + onSend: vi.fn(async () => true) + }) + expect(texts(mounted).filter((text) => text === 'Send' || text === 'Steer')).toEqual(['Send']) + expect(mounted.root.findByProps({ accessibilityLabel: 'Send this message' })).toBeTruthy() + }) + it('a command card never steers: Send only while the agent is idle, and no menu', async () => { const onSend = vi.fn(async () => true) const mounted = await mount({ diff --git a/mobile/src/session/MobileNativeChatQueuedMessages.tsx b/mobile/src/session/MobileNativeChatQueuedMessages.tsx index 80476207c04d..d32c8815dc49 100644 --- a/mobile/src/session/MobileNativeChatQueuedMessages.tsx +++ b/mobile/src/session/MobileNativeChatQueuedMessages.tsx @@ -109,8 +109,8 @@ export function MobileNativeChatQueuedMessages({ const busy = busyIds.has(card.messageId) const returned = card.state === 'returned' // "Steer" submits beside the running turn, the paused queue's cards too; a card whose - // own send failed, or a returned one, is sent again. - const steers = !returned && !card.paused && !card.command + // own send failed, or a returned one, is sent again, as is any while nothing runs. + const steers = !returned && !card.paused && !card.command && !card.agentIdle return ( { ) }) + it("marks every card idle only on the host's own word that nothing runs", () => { + const idle = mobileQueuedMessageCards([draft({ messageId: 'waiting' })], [], { + pendingPrompt: false, + hostIdle: true + }) + expect(idle.map((card) => card.agentIdle)).toEqual([true]) + // An older host says nothing: its cards read as before. + const older = mobileQueuedMessageCards([draft({ messageId: 'waiting' })], [], { + pendingPrompt: false + }) + expect(older.map((card) => card.agentIdle)).toEqual([undefined]) + }) + it('renders nothing without a published list', () => { expect(mobileQueuedMessageCards(null, [], { pendingPrompt: false })).toEqual([]) expect(mobileQueuedMessageCards([], [], { pendingPrompt: false })).toEqual([]) diff --git a/mobile/src/session/mobile-structured-queued-message-cards.ts b/mobile/src/session/mobile-structured-queued-message-cards.ts index 42883b4ed4df..50904833ca15 100644 --- a/mobile/src/session/mobile-structured-queued-message-cards.ts +++ b/mobile/src/session/mobile-structured-queued-message-cards.ts @@ -38,6 +38,8 @@ export type MobileQueuedMessageCard = { command?: true /** A command card while the agent works: it offers no send until the agent is idle. */ waitsForAgent?: true + /** No turn runs, by the host's own answer, so there is nothing to steer into. */ + agentIdle?: true } function queuedMessageBodyText(body: AgentSessionQueuedMessage['body']): string { @@ -115,6 +117,8 @@ export function mobileQueuedMessageCards( pendingPrompt: boolean queuePaused?: boolean agentWorking?: boolean + /** The host said nothing runs; an older host never says, and its cards read as before. */ + hostIdle?: boolean agentName?: string statedFailures?: readonly AgentSessionFailureFact[] } @@ -162,6 +166,7 @@ export function mobileQueuedMessageCards( (paused && draft.pausedReason === QUEUED_MESSAGE_PAUSED_SEND_FAILED), caption, attribution: agentMessageAttribution('From', readAgentMessageSource(draft.body.from)), + ...(facts.hostIdle ? { agentIdle: true as const } : {}), ...(draft.body.command !== undefined ? { command: true as const, diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index 1b1aab57c176..fa519eebd201 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -237,6 +237,8 @@ export function useMobileStructuredAgentSession(args: { submissions: state.submissions, pendingPrompt: approvalPrompt !== null || questionPrompt !== null, agentWorking: isWorking, + // The host's own answer, as desktop reads it: an older host gives none, and its cards steer. + hostIdle: state.working === false && (state.nextQueuedMessageId ?? null) === null, mutate, appendComposerText, onSendError, diff --git a/mobile/src/session/use-mobile-structured-queued-message-controls.ts b/mobile/src/session/use-mobile-structured-queued-message-controls.ts index 1a73756841f1..1858cca5b637 100644 --- a/mobile/src/session/use-mobile-structured-queued-message-controls.ts +++ b/mobile/src/session/use-mobile-structured-queued-message-controls.ts @@ -58,6 +58,8 @@ export function useMobileStructuredQueuedMessageControls(args: { pendingPrompt: boolean /** The chat shows the agent working: a command card offers no send then. */ agentWorking?: boolean + /** The host said nothing runs, so no card steers; absent from an older host. */ + hostIdle?: boolean mutate: MobileStructuredAgentMutate /** The active pane's live composer, Edit's copy target; absent = Edit refuses. False when * nothing was copied. */ @@ -78,11 +80,13 @@ export function useMobileStructuredQueuedMessageControls(args: { submissions } = args const agentWorking = args.agentWorking === true + const hostIdle = args.hostIdle === true const cards = useMemo( () => mobileQueuedMessageCards(queuedMessages, submissions, { pendingPrompt, agentWorking, + hostIdle, agentName: args.agentName, statedFailures: queuedMessages?.some((draft) => draft.state === 'returned') ? agentSessionVisibleFailureFacts(args.journalItems ?? []) @@ -91,6 +95,7 @@ export function useMobileStructuredQueuedMessageControls(args: { }), [ agentWorking, + hostIdle, args.agentName, args.journalItems, pendingPrompt, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts index 30d04e723627..ce0c754a437f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts @@ -124,6 +124,25 @@ describe('under another connection’s write lock', () => { }) }, 20_000) + it('a relaunch under a lock longer than a minute of rounds still settles once it lifts, unasked', async () => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + rig = await createQueuedMessageTestRig({ restartable: true }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current, { prompt: true }) + let released: Promise = Promise.resolve() + + await current.crashRestartHostProcess(async () => { + released = (await holdWriteLock(current.root, 40_000)).released + }) + await released + // Nothing else writes or reads: only the retry's own backoff, at most 2 s, comes back to it. + await vi.waitFor(() => expect(turnState(current)).toBe('unverifiable'), { timeout: 3_000 }) + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.working).toBe(false) + expect(page.ok && page.page.latestTurn?.turn.state).not.toBe('running') + }, 60_000) + it('a pending rewind the retry recovers ends its round at once, never holding the lane', async () => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) rig = await createQueuedMessageTestRig({ restartable: true }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts index ba6199c2920d..570fe8db9beb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -151,14 +151,15 @@ describe('an episode the retry gives up', () => { const attempts = vi .spyOn(currentJournal(current), 'appendPlannedLifecycleBatch') .mockRejectedValue(locked()) - for (let round = 0; round < 12; round += 1) { - await vi.advanceTimersByTimeAsync(31_000) + // Past the episode's budget of rounds failing in a row (`RECONCILIATION_GIVE_UP_MS`). + for (let round = 0; round < 100; round += 1) { + await vi.advanceTimersByTimeAsync(2_000) await settleTurns() } const given = attempts.mock.calls.length await vi.advanceTimersByTimeAsync(120_000) await settleTurns() - // Ten rounds, then no timer: the set is kept, and nothing retries on its own. + // Then no timer: the set is kept, and nothing retries on its own. expect(attempts.mock.calls.length).toBe(given) expect(retryOwes(current.host.collaboratorsForTests().reconciliation, SESSION)).toBe(true) attempts.mockRestore() @@ -205,8 +206,9 @@ describe('a queued send the retry gives up on', () => { const hold = vi.spyOn(JournalQueuedMessages.prototype, 'hold').mockRejectedValue(locked()) await current.settleAccepted(working, 'a') - for (let round = 0; round < 12; round += 1) { - await vi.advanceTimersByTimeAsync(31_000) + for (let round = 0; round < 100; round += 1) { + await vi.advanceTimersByTimeAsync(2_000) + await new Promise((resolve) => setImmediate(resolve)) } expect(hold).toHaveBeenCalledTimes(1) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts index 919df4d0c684..89aa02c718d7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts @@ -1,10 +1,12 @@ // The one backoff the host's retry of background bookkeeping uses: from 1 s to 2 s, given up after -// ten rounds in a row, so nothing is owed forever. Short: a background write fails at once, so -// waiting long only delays work a lifted lock already allows, and nothing announces the lift. +// about 3 minutes of rounds failing in a row, so nothing is owed forever. Short: a background write +// fails at once, so a round is cheap, waiting long only delays work a lifted lock already allows, +// and nothing announces the lift. export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 2_000 } as const -/** Consecutive failed attempts after which a retry gives up. */ -export const RECONCILIATION_MAX_FAILED_ATTEMPTS = 10 +/** The backoffs a retry waits out with every round failing, summed, after which it gives up: a + * round fails at once, so this is the time it kept failing. */ +export const RECONCILIATION_GIVE_UP_MS = 180_000 /** The wait before the next try, after `failures` consecutive failed ones. */ export function reconciliationBackoffDelay(failures: number): number { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts index e5e9c78f4020..d8490fef5bf4 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -9,16 +9,18 @@ // card drain) and marks the chat owed. A round visits every owed chat: closed ones load side by // side in the few background slots, a reader's first, and each writes in its turn, one chat at a // time with a yield between (`inTurn`): -// (a) the store-wide restart reconcile, once, while any lease is unreconciled; then per chat, in +// (a) the store-wide restart reconcile and the restore's latched recovery, once, while either is +// owed (`reconcileOwed`); then per chat, in // its lane, (b) release repair, (c) settlement and what an earlier // process left (`runStructuredAgentSessionReconciliationPass`), (d) the queue's next send. Every // write is background (`JournalWriteOptions`): another connection's lock fails it at once and ends // the round, since contention is connection-wide. // -// A failed round backs off from 1 s to 30 s; ten in a row end the episode: the timer stops, the set -// is kept, and a queue send still owed shows as not sent (`abandonSend`). A new signal, a reader's -// open, or a commit on the connection after contention starts a fresh episode. A lease latched in -// recovery is never decided here: its proofs wait in the set for its decision's signal. +// A failed round backs off from 1 s to 2 s; about 3 minutes of them in a row end the episode +// (`RECONCILIATION_GIVE_UP_MS`): the timer stops, the set is kept, and a queue send still owed +// shows as not sent (`abandonSend`). A new signal, a reader's open, or a commit on the connection +// after contention starts a fresh episode. A lease latched in recovery is decided in (a) only for a +// chat the restart restore made readable; any other one's proofs wait in the set for its decision. import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' import { setImmediate as yieldToEvents } from 'node:timers/promises' @@ -26,7 +28,7 @@ import type { AgentJournalCursor } from '../../../shared/agent-session-journal-t import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionExitSettlement } from './structured-agent-session-leftover-settlement' import { - RECONCILIATION_MAX_FAILED_ATTEMPTS, + RECONCILIATION_GIVE_UP_MS, reconciliationBackoffDelay } from './structured-agent-session-reconciliation-backoff' import { StructuredAgentSessionReconciliationSlots } from './structured-agent-session-reconciliation-slots' @@ -65,8 +67,10 @@ export class StructuredAgentSessionRetry { private running = false /** A chat became due while a round ran: another follows at once. */ private again = false - /** Consecutive failed rounds in this episode; at the budget the episode ends. */ + /** Consecutive failed rounds in this episode, and the backoffs they waited out; at the budget + * (`RECONCILIATION_GIVE_UP_MS`) the episode ends. */ private failures = 0 + private failingFor = 0 /** The last round failed on another connection's lock, which a commit here proves gone. */ private contended = false private disposed = false @@ -135,7 +139,7 @@ export class StructuredAgentSessionRetry { this.slots.prioritize(chat) // Backing off after a failed round, it goes first in the next, which an open does not // hurry: a store that keeps refusing costs a write per round, never one per chat opened. - if (this.failures === 0 || this.failures >= RECONCILIATION_MAX_FAILED_ATTEMPTS) { + if (this.failures === 0 || this.failingFor >= RECONCILIATION_GIVE_UP_MS) { this.freshEpisode() } } @@ -194,6 +198,7 @@ export class StructuredAgentSessionRetry { return } this.failures = 0 + this.failingFor = 0 this.contended = false if (!this.running && this.timer !== 'soon') { this.stopTimer() @@ -227,9 +232,9 @@ export class StructuredAgentSessionRetry { this.stopped ||= visit === 'contended' this.again ||= visit === 'again' } - if (this.context.deps.store.listRecords().some((record) => record.lease.unreconciled)) { - // Store-wide: once a round, for every chat, never once per chat. No chat is visited while a - // lease is unreconciled. + if (this.context.reconcileOwed()) { + // Store-wide: once a round, for every chat, never once per chat. No chat is visited while it + // owes anything. const reconciled = await this.context.reconcile() if (reconciled !== 'settled') { end(reconciled) @@ -259,6 +264,7 @@ export class StructuredAgentSessionRetry { } if (!failed) { this.failures = 0 + this.failingFor = 0 this.contended = false if (this.again && this.timer === null) { this.freshEpisode() @@ -268,8 +274,10 @@ export class StructuredAgentSessionRetry { this.failures += 1 this.contended = contended this.stopTimer() - if (this.failures < RECONCILIATION_MAX_FAILED_ATTEMPTS) { - const timer = setTimeout(() => void this.round(), reconciliationBackoffDelay(this.failures)) + if (this.failingFor < RECONCILIATION_GIVE_UP_MS) { + const delay = reconciliationBackoffDelay(this.failures) + this.failingFor += delay + const timer = setTimeout(() => void this.round(), delay) // A backoff alone never keeps the process alive. timer.unref?.() this.timer = timer diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts index 29cd6273fdfe..d66066cdd96a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts @@ -25,8 +25,10 @@ export type StructuredAgentSessionRetryContext = StructuredAgentSessionReconcili track: (operation: Promise) => Promise /** Publishes what is current now and wakes the queued-card drain. */ publishGenerationEnded: (sessionId: string, options?: { restate?: boolean }) => void - /** Startup's store-wide lease reconcile, as background bookkeeping: whether every lease settled, - * or why not. */ + /** Whether the store-wide step owes anything: a lease unreconciled, or the restart restore's + * latched recovery its own write could not decide. */ + reconcileOwed: () => boolean + /** That step, as background bookkeeping: whether it all settled, or why not. */ reconcile: () => Promise<'settled' | 'contended' | 'failed'> /** The queue's automatic send of the chat's next card, inside the lane the visit holds, and an * episode giving up on it (`StructuredAgentSessionQueuedMessageDrain.sendForRetry`, `.abandon`). */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts new file mode 100644 index 000000000000..00b2ee9954b9 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts @@ -0,0 +1,110 @@ +// Startup owes a latched recovery only to the chats its restart restore makes readable, the visible +// tabs: deciding it may stop a surviving agent. When the restore's own reconcile or recovery write +// meets another connection's lock, the host's retry decides it instead, round after round, until it +// lands; and a chat a reader only opened behind them (an orchestration worker's history, a mailbox +// pointer, an outline, a naming read) is never decided, so its agent is never signalled. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { closeTestJournalHostDatabases } from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + openScanHost, + scanRecord as record, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-restore-recovery-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +const busy = () => + Object.assign(new Error('database is locked'), { errcode: 5, code: 'SQLITE_BUSY' }) + +/** A chat whose agent outlived the earlier process, and whose first `locked` reconciles meet a + * lock: its lease reconciles to `recovering`, and only stopping the agent decides it. */ +async function survivingAgent(locked: number) { + await seedTestAgentSessionRecordStore(root, { records: [record(CHAT, false)] }) + await seedScanJournal(root, CHAT) + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const reconcile = vi.spyOn(store, 'reconcileOnRestart') + for (let index = 0; index < locked; index += 1) { + reconcile.mockRejectedValueOnce(busy()) + } + let alive = true + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + const probeOwner = async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + return { current: host, stopOwnerProcess } +} + +it("never stops a hidden chat's agent that readers only read, after reconciles a lock refused", async () => { + const { current, stopOwnerProcess } = await survivingAgent(2) + await current.reconcileRestartLeases() + expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(true) + + // Every reader that opens a chat no tab shows. + await current.history({ sessionId: CHAT, direction: 'tail' }, 'own-agent') // worker-show + await current.journalSnapshot(CHAT) // a mailbox pointer + await current.journalSnapshot(CHAT) // the conversation outline + await current.journalSnapshot(CHAT) // a naming read + await vi.waitFor(() => expect(store.getRecord(CHAT)?.lease.unreconciled).toBe(false), { + timeout: 8_000 + }) + // Rounds enough for anything that would decide it. + await new Promise((resolve) => setTimeout(resolve, 3_000)) + + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') +}, 20_000) + +it("retries a visible chat's recovery its write could not land, until it does", async () => { + // Startup's, the restore's and the first round's reconciles meet the lock. + const { current, stopOwnerProcess } = await survivingAgent(3) + const transition = store.transitionHandoff + let refused = 0 + vi.spyOn(store, 'transitionHandoff').mockImplementation((...args) => { + // The lock is back for the recovery's own write, once. + if (args[2]?.background && refused === 0) { + refused += 1 + return Promise.reject(busy()) + } + return transition(...args) + }) + + await current.reconcileRestartLeases() + await current.restoreReadableSessions([CHAT]) + + await vi.waitFor(() => expect(refused).toBe(1), { timeout: 8_000 }) + await vi.waitFor(() => expect(store.getRecord(CHAT)?.lease.handoffStage).toBeNull(), { + timeout: 8_000 + }) + expect(stopOwnerProcess).toHaveBeenCalled() + expect(current.currentWork(CHAT)?.working()).toBe(false) +}, 30_000) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts index 620e92561963..d039fc31f471 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts @@ -49,6 +49,7 @@ it('ten chats whose automatic sends meet one lock cost one round, and send once laneBusy: () => false, track: (operation: Promise) => operation, publishGenerationEnded: vi.fn(), + reconcileOwed: () => false, reconcile: async () => 'settled' as const, sendQueued, abandonSend diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index 350903351b8b..b06b67a372cb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -65,10 +65,10 @@ export async function revealStructuredAgentSession( } } -/** The host's startup and its reconciliation: reconcile, then hand every chat to its reconciliation - * worker in the background (no chat's tab or send waits on it), and the readable-restore sweep - * that resolves and opens each visible chat. Its lease bookkeeping is a reader's, which never fails a read - * or startup; startup shares it. */ +/** The host's startup and its reconciliation: reconcile, then hand every chat to the host's retry + * in the background (no chat's tab or send waits on it), and the readable-restore sweep that + * resolves and opens each visible chat. Its lease bookkeeping is a reader's, which never fails a + * read or startup; startup shares it. */ export function createStructuredAgentSessionHostRestore( deps: StructuredAgentSessionHostDeps, wiring: Omit< @@ -105,6 +105,15 @@ export function createStructuredAgentSessionHostRestore( failures ) const recoveryWrites = backgroundLeaseWrites(deps.store) + // The chats the restart restore was asked to make readable, the visible tabs: theirs is the one + // latched recovery startup owes. A chat a reader opened behind them never is: a read stops no one. + const restoreTargets = new Set() + let restoresAll = false + const latchedTargets = () => + (restoresAll + ? deps.store.listRecords().map((record) => record.sessionId) + : [...restoreTargets] + ).filter((sessionId) => deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering') const reconciliation = new StructuredAgentSessionRetry({ deps, sessions: startup.sessions, @@ -115,6 +124,9 @@ export function createStructuredAgentSessionHostRestore( track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => startup.clientDelivery.publishGenerationEnded(sessionId, options), + reconcileOwed: () => + deps.store.listRecords().some((record) => record.lease.unreconciled) || + latchedTargets().length > 0, reconcile: async () => { try { const refusal = await reconcileLeases(null, BACKGROUND_WRITE) @@ -122,11 +134,9 @@ export function createStructuredAgentSessionHostRestore( failures.report(refusal) return 'failed' } - // What a restore whose own reconcile failed left latched: its open chats' recovery. - for (const { sessionId, lease } of deps.store.listRecords()) { - if (lease.handoffStage === 'recovering' && startup.sessions.has(sessionId)) { - await resolveRecovery(sessionId, recoveryWrites) - } + // The restore's own decision, wherever its reconcile or its write met a lock. + for (const sessionId of latchedTargets()) { + await resolveRecovery(sessionId, recoveryWrites) } failures.clear() return 'settled' @@ -148,6 +158,8 @@ export function createStructuredAgentSessionHostRestore( () => true, (error: unknown) => { failures.report(error) + // The retry's round decides it instead (`reconcileOwed`). + reconciliation.signal(sessionId) return false } ), @@ -160,7 +172,11 @@ export function createStructuredAgentSessionHostRestore( startupSettled = scanStructuredAgentSessionsAtStartup(deps.store, reconciliation) }, startupSettled: () => startupSettled, - restoreReadableSessions: (sessionIds) => gate.run(() => restorer.restore(sessionIds)), + restoreReadableSessions: (sessionIds) => { + restoresAll ||= sessionIds === undefined + sessionIds?.forEach((sessionId) => restoreTargets.add(sessionId)) + return gate.run(() => restorer.restore(sessionIds)) + }, reconciliation } } From ceb7fd314b0779f01cda025a73ad833f8dc9861a Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:28:59 -0700 Subject: [PATCH 15/22] fix(native-chat): queued cards keep main's Steer; the relaunch bound tests the property - reverts the card's Send label on desktop and phone to main's; pressing Steer on an idle chat sends, as before - the relaunch-under-lock test asserts no foreground transaction runs while locked, with a 100 ms bound --- .../MobileNativeChatQueuedMessages.test.tsx | 27 +++++++++++----- .../MobileNativeChatQueuedMessages.tsx | 4 +-- ...le-structured-queued-message-cards.test.ts | 13 -------- .../mobile-structured-queued-message-cards.ts | 5 --- .../use-mobile-structured-agent-session.ts | 2 -- ...bile-structured-queued-message-controls.ts | 5 --- ...ured-agent-session-background-lock.test.ts | 31 ++++++++++++++++--- ...uctured-agent-session-queued-cards.test.ts | 15 +++------ .../structured-agent-session-queued-cards.ts | 16 ++-------- ...tructured-agent-session-queued-messages.ts | 9 ++---- 10 files changed, 57 insertions(+), 70 deletions(-) diff --git a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx index 22dd447d0e99..b781a230a94a 100644 --- a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx +++ b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx @@ -4,7 +4,10 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import type { AgentSessionQueuePause } from '../../../src/shared/agent-session-wire' import type { ActionSheetAction } from '../components/ActionSheetModal' import { MobileNativeChatQueuedMessages } from './MobileNativeChatQueuedMessages' -import type { MobileQueuedMessageCard } from './mobile-structured-queued-message-cards' +import { + mobileQueuedMessageCards, + type MobileQueuedMessageCard +} from './mobile-structured-queued-message-cards' vi.mock('react-native', () => ({ Pressable: 'Pressable', @@ -209,13 +212,21 @@ describe('MobileNativeChatQueuedMessages', () => { expect(nodeTypes(rows[1]!)).toContain('CornerDownRight') }) - it('reads Send, not Steer, on a waiting card while the host says nothing runs', async () => { - const mounted = await mount({ - cards: [card({ messageId: 'waiting', agentIdle: true })], - onSend: vi.fn(async () => true) - }) - expect(texts(mounted).filter((text) => text === 'Send' || text === 'Steer')).toEqual(['Send']) - expect(mounted.root.findByProps({ accessibilityLabel: 'Send this message' })).toBeTruthy() + it('a waiting card still reads Steer while nothing runs, as when its agent died and its cleanup waits', async () => { + const [waiting] = mobileQueuedMessageCards( + [ + { + messageId: 'waiting', + position: 1, + state: 'waiting', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'next' }] } + } + ], + [], + { pendingPrompt: false, agentWorking: false } + ) + const mounted = await mount({ cards: [waiting!], onSend: vi.fn(async () => true) }) + expect(texts(mounted).filter((text) => text === 'Send' || text === 'Steer')).toEqual(['Steer']) }) it('a command card never steers: Send only while the agent is idle, and no menu', async () => { diff --git a/mobile/src/session/MobileNativeChatQueuedMessages.tsx b/mobile/src/session/MobileNativeChatQueuedMessages.tsx index d32c8815dc49..80476207c04d 100644 --- a/mobile/src/session/MobileNativeChatQueuedMessages.tsx +++ b/mobile/src/session/MobileNativeChatQueuedMessages.tsx @@ -109,8 +109,8 @@ export function MobileNativeChatQueuedMessages({ const busy = busyIds.has(card.messageId) const returned = card.state === 'returned' // "Steer" submits beside the running turn, the paused queue's cards too; a card whose - // own send failed, or a returned one, is sent again, as is any while nothing runs. - const steers = !returned && !card.paused && !card.command && !card.agentIdle + // own send failed, or a returned one, is sent again. + const steers = !returned && !card.paused && !card.command return ( { ) }) - it("marks every card idle only on the host's own word that nothing runs", () => { - const idle = mobileQueuedMessageCards([draft({ messageId: 'waiting' })], [], { - pendingPrompt: false, - hostIdle: true - }) - expect(idle.map((card) => card.agentIdle)).toEqual([true]) - // An older host says nothing: its cards read as before. - const older = mobileQueuedMessageCards([draft({ messageId: 'waiting' })], [], { - pendingPrompt: false - }) - expect(older.map((card) => card.agentIdle)).toEqual([undefined]) - }) - it('renders nothing without a published list', () => { expect(mobileQueuedMessageCards(null, [], { pendingPrompt: false })).toEqual([]) expect(mobileQueuedMessageCards([], [], { pendingPrompt: false })).toEqual([]) diff --git a/mobile/src/session/mobile-structured-queued-message-cards.ts b/mobile/src/session/mobile-structured-queued-message-cards.ts index 50904833ca15..42883b4ed4df 100644 --- a/mobile/src/session/mobile-structured-queued-message-cards.ts +++ b/mobile/src/session/mobile-structured-queued-message-cards.ts @@ -38,8 +38,6 @@ export type MobileQueuedMessageCard = { command?: true /** A command card while the agent works: it offers no send until the agent is idle. */ waitsForAgent?: true - /** No turn runs, by the host's own answer, so there is nothing to steer into. */ - agentIdle?: true } function queuedMessageBodyText(body: AgentSessionQueuedMessage['body']): string { @@ -117,8 +115,6 @@ export function mobileQueuedMessageCards( pendingPrompt: boolean queuePaused?: boolean agentWorking?: boolean - /** The host said nothing runs; an older host never says, and its cards read as before. */ - hostIdle?: boolean agentName?: string statedFailures?: readonly AgentSessionFailureFact[] } @@ -166,7 +162,6 @@ export function mobileQueuedMessageCards( (paused && draft.pausedReason === QUEUED_MESSAGE_PAUSED_SEND_FAILED), caption, attribution: agentMessageAttribution('From', readAgentMessageSource(draft.body.from)), - ...(facts.hostIdle ? { agentIdle: true as const } : {}), ...(draft.body.command !== undefined ? { command: true as const, diff --git a/mobile/src/session/use-mobile-structured-agent-session.ts b/mobile/src/session/use-mobile-structured-agent-session.ts index fa519eebd201..1b1aab57c176 100644 --- a/mobile/src/session/use-mobile-structured-agent-session.ts +++ b/mobile/src/session/use-mobile-structured-agent-session.ts @@ -237,8 +237,6 @@ export function useMobileStructuredAgentSession(args: { submissions: state.submissions, pendingPrompt: approvalPrompt !== null || questionPrompt !== null, agentWorking: isWorking, - // The host's own answer, as desktop reads it: an older host gives none, and its cards steer. - hostIdle: state.working === false && (state.nextQueuedMessageId ?? null) === null, mutate, appendComposerText, onSendError, diff --git a/mobile/src/session/use-mobile-structured-queued-message-controls.ts b/mobile/src/session/use-mobile-structured-queued-message-controls.ts index 1858cca5b637..1a73756841f1 100644 --- a/mobile/src/session/use-mobile-structured-queued-message-controls.ts +++ b/mobile/src/session/use-mobile-structured-queued-message-controls.ts @@ -58,8 +58,6 @@ export function useMobileStructuredQueuedMessageControls(args: { pendingPrompt: boolean /** The chat shows the agent working: a command card offers no send then. */ agentWorking?: boolean - /** The host said nothing runs, so no card steers; absent from an older host. */ - hostIdle?: boolean mutate: MobileStructuredAgentMutate /** The active pane's live composer, Edit's copy target; absent = Edit refuses. False when * nothing was copied. */ @@ -80,13 +78,11 @@ export function useMobileStructuredQueuedMessageControls(args: { submissions } = args const agentWorking = args.agentWorking === true - const hostIdle = args.hostIdle === true const cards = useMemo( () => mobileQueuedMessageCards(queuedMessages, submissions, { pendingPrompt, agentWorking, - hostIdle, agentName: args.agentName, statedFailures: queuedMessages?.some((draft) => draft.state === 'returned') ? agentSessionVisibleFailureFacts(args.journalItems ?? []) @@ -95,7 +91,6 @@ export function useMobileStructuredQueuedMessageControls(args: { }), [ agentWorking, - hostIdle, args.agentName, args.journalItems, pendingPrompt, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts index ce0c754a437f..42faabced952 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-lock.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { QUEUED_MESSAGE_PAUSED_SEND_FAILED } from '../../../shared/agent-session-wire' import { agentJournalItemKey } from '../../../shared/agent-session-journal-item-key' import { JournalQueuedMessages } from '../agent-session-journal/journal-queued-messages' +import { JournalHostDatabase } from '../agent-session-journal/journal-host-database' import { AgentSessionJournal } from '../agent-session-journal/journal-store' import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' import { @@ -108,13 +109,33 @@ describe('under another connection’s write lock', () => { await current.workingSend() await leaveUnfinishedWork(current, { prompt: true }) + // Every write that runs while the lock is held: none may be one that waits it out. + const waiting: unknown[] = [] + let held = false + const transaction = JournalHostDatabase.prototype.transaction + vi.spyOn(JournalHostDatabase.prototype, 'transaction').mockImplementation(function ( + this: JournalHostDatabase, + run, + options + ) { + if (held && !options?.background) { + waiting.push(new Error('a write that waits for the lock')) + } + return transaction.call(this, run, options) + }) + // Another process holds the lock from before this one starts: the restart reconcile, the // restore's reconcile and recovery, and the startup scan all meet it. - expect( - await longestStall(() => - current.crashRestartHostProcess(() => holdWriteLock(current.root, LOCK_MS)) - ) - ).toBeLessThan(60) + const stall = await longestStall(() => + current.crashRestartHostProcess(async () => { + const { released } = await holdWriteLock(current.root, LOCK_MS) + held = true + void released.then(() => (held = false)) + }) + ) + expect(waiting).toEqual([]) + // The bar QA holds the app to; the property above is what keeps it there under load. + expect(stall).toBeLessThan(100) // Once it lifts, the retry reconciles, resolves the recovery the restore could not, and settles // the turn as an unlocked startup does: the rig's probe proves nothing, so unverifiable. await vi.waitFor(() => expect(turnState(current)).toBe('unverifiable'), { timeout: 8_000 }) diff --git a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts index 3053d95ee586..833e8d954d97 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.test.ts @@ -171,19 +171,14 @@ describe('queued message cards', () => { expect(cards.map((card) => queuedMessageCardSteers(card))).toEqual([true, true]) }) - it('a card offers Send, not Steer, while the host says no turn runs; an older host keeps Steer', () => { - // An agent that died while its settlement waits: the host names no turn running. - const idle = projectQueuedMessageCards([draft('waiting', 1)], [], { + it('a waiting card still offers Steer while nothing runs, as when its agent died and its cleanup waits', () => { + const cards = projectQueuedMessageCards([draft('waiting', 1)], [], { hasPendingPrompt: false, - agentWorking: false, - hostIdle: true + agentWorking: false }) - expect(idle.map((card) => [card.hold, queuedMessageCardSteers(card)])).toEqual([ - ['turn', false] + expect(cards.map((card) => [card.hold, queuedMessageCardSteers(card)])).toEqual([ + ['turn', true] ]) - // An older host gives no answer of its own: its cards steer as they always did. - const older = projectQueuedMessageCards([draft('waiting', 1)], [], IDLE) - expect(older.map((card) => queuedMessageCardSteers(card))).toEqual([true]) }) it("the header names the queue's pause while it holds a card, and none over cards Resume would not send", () => { diff --git a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts index 8d24463a33fe..4192b552d791 100644 --- a/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts +++ b/src/renderer/src/components/native-chat/structured-agent-session-queued-cards.ts @@ -39,8 +39,6 @@ export type QueuedMessageCard = { command?: true /** A command card while the agent works: it offers no send until the agent is idle. */ waitsForAgent?: true - /** No turn runs, by the host's own answer, so there is nothing to steer into. */ - agentIdle?: true pausedReason?: string returnedReason?: string | null /** The typed fact the returned card's submission settled with; read like its `rejection`. */ @@ -63,13 +61,7 @@ function queuedMessageCardText(body: AgentSessionQueuedMessage['body']): string export function projectQueuedMessageCards( queuedMessages: readonly AgentSessionQueuedMessage[] | null | undefined, submissions: readonly AgentJournalSubmission[], - session: { - hasPendingPrompt: boolean - queuePaused?: boolean - agentWorking?: boolean - /** The host said nothing runs; an older host never says, and its cards read as before. */ - hostIdle?: boolean - } + session: { hasPendingPrompt: boolean; queuePaused?: boolean; agentWorking?: boolean } ): QueuedMessageCard[] { const handedOff = handedOffQueuedMessageIds( submissions.filter((submission) => submission.dispatchState !== 'rejected') @@ -105,7 +97,6 @@ export function projectQueuedMessageCards( ...(session.agentWorking ? { waitsForAgent: true as const } : {}) } : {}), - ...(session.hostIdle ? { agentIdle: true as const } : {}), ...(message.pausedReason !== undefined ? { pausedReason: message.pausedReason } : {}), ...(message.returnedReason !== undefined ? { returnedReason: message.returnedReason } : {}), ...(message.returnedRejection !== undefined @@ -137,10 +128,9 @@ export function queuedMessagesQueuePause( } /** Steer names the mid-turn jump, also while the whole queue is paused; a card held on its own or - * returned is not waiting on the turn, nor is any card while no turn runs, so its action is - * plainly Send. A command never steers. */ + * returned is not waiting on the turn, so its action is plainly Send. A command never steers. */ export function queuedMessageCardSteers(card: QueuedMessageCard): boolean { - return card.hold !== 'paused' && card.hold !== 'returned' && !card.command && !card.agentIdle + return card.hold !== 'paused' && card.hold !== 'returned' && !card.command } /** The card Cmd/Ctrl+Enter steers: the newest one, unless it is a command, which never steers. */ diff --git a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts index 6d638dea9d37..b56d5b0eb916 100644 --- a/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts +++ b/src/renderer/src/components/native-chat/use-structured-agent-session-queued-messages.ts @@ -81,8 +81,6 @@ export function useStructuredAgentSessionQueuedMessages(args: { hasPendingPrompt: boolean /** A turn is running, whoever started it, or the queue is about to send its next card. */ isWorking: boolean - /** The host's own answer (`working`); absent from an older host. */ - hostWork?: { working?: boolean } /** This pane's sends without a host record yet show as sending cards. */ sending?: readonly StructuredAgentSessionPendingSend[] composerScopeKey: string | undefined @@ -90,8 +88,6 @@ export function useStructuredAgentSessionQueuedMessages(args: { }): StructuredAgentSessionQueuedMessagesController { const { composerScopeKey, enabled, hasPendingPrompt, mutate, queuedMessages, submissions } = args const { isWorking, queuePause } = args - // Only the host's own answer: an older host's cards steer as they always did. - const hostIdle = args.hostWork?.working === false && !isWorking const sending = args.sending ?? NO_SENDS const cards = useMemo( @@ -100,8 +96,7 @@ export function useStructuredAgentSessionQueuedMessages(args: { hasPendingPrompt, // A command card offers no send while the agent works. agentWorking: isWorking, - queuePaused: queuePause !== null, - hostIdle + queuePaused: queuePause !== null }), ...sendingQueuedMessageCards( pendingQueueSendsOnTheirWay( @@ -112,7 +107,7 @@ export function useStructuredAgentSessionQueuedMessages(args: { ) ) ], - [hasPendingPrompt, hostIdle, isWorking, sending, queuePause, queuedMessages, submissions] + [hasPendingPrompt, isWorking, sending, queuePause, queuedMessages, submissions] ) const cardsRef = useRef(cards) useEffect(() => { From 13b09e2e95c467175ea029c718c4e5c913971228 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:56:43 -0700 Subject: [PATCH 16/22] refactor(native-chat): one owed set, one wake, one budget for background cleanup - every source of owed work enters through one signal; one wake runs a round, from the backoff timer or a successful commit; the give-up is reset only by the first wake after it - a stopped round keeps the journals it loaded instead of replaying them next round - recovery the restore could not finish stays with the chats latched when it ran, and a latch no probe can decide, or a read-only store, owes nothing --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + ...ed-agent-session-background-writes.test.ts | 3 +- ...ssion-leftover-settlement-triggers.test.ts | 10 +- ...ructured-agent-session-mutation-context.ts | 2 +- ...tructured-agent-session-queued-messages.ts | 6 +- ...ured-agent-session-reconciliation-retry.ts | 171 +++++++-------- ...ured-agent-session-reconciliation-visit.ts | 15 +- ...uctured-agent-session-retry-budget.test.ts | 2 +- ...ructured-agent-session-retry-scope.test.ts | 203 ++++++++++++++++++ ...tructured-agent-session-retry-wake.test.ts | 151 +++++++++++++ .../structured-agent-session-reveal.ts | 48 +++-- 11 files changed, 480 insertions(+), 132 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index a06b1dfa6bed..785aec2f01cf 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -127,6 +127,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-dead-turn-frames.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restore-recovery-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-restart-reconcile-retry.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-ended-prompt-cancel.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-roster-provenance.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-legacy-handoff-record.test.ts', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts index 570fe8db9beb..657a7c1a62ea 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -56,12 +56,11 @@ describe("another connection's write lock", () => { const { reconciliation } = current.host.collaboratorsForTests() const { released } = await holdWriteLock(current.root, 1_500) - const started = performance.now() + // The exit's own refused settlement left a backoff: its timer runs the round. const stall = await longestStall(async () => { reconciliation.signal(SESSION) await reconciliation.attempted(SESSION) }) - expect(performance.now() - started).toBeLessThan(150) expect(stall).toBeLessThan(60) expect(retryOwes(reconciliation, SESSION)).toBe(true) expect(turnState(current)).toBe('running') diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts index 8893748c0f48..4a97303d40ee 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts @@ -210,7 +210,7 @@ describe('the retry', () => { } } - it('backs off from 1 s, doubling; a new signal starts a fresh episode at once; settles once the fault clears', async () => { + it('backs off from 1 s, doubling; a new signal adds no round to the backoff; settles once the fault clears', async () => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) try { rig = await createQueuedMessageTestRig({ restartable: true }) @@ -236,15 +236,15 @@ describe('the retry', () => { await settleTurns() expect(attempts).toHaveBeenCalledTimes(2) - // A new signal is a fresh episode: a round at once, and the budget back at its first step. + // A new signal while it backs off adds no round: the timer's next step runs it. reconciliation.signal(SESSION) await settleTurns() - expect(attempts).toHaveBeenCalledTimes(3) + expect(attempts).toHaveBeenCalledTimes(2) expect(turnState(current)).toBe('running') database.db.exec('DROP TRIGGER reject_recovered') - await vi.advanceTimersByTimeAsync(990) + await vi.advanceTimersByTimeAsync(1_990) await settleTurns() - expect(attempts).toHaveBeenCalledTimes(3) + expect(attempts).toHaveBeenCalledTimes(2) await vi.advanceTimersByTimeAsync(10) await settleTurns() expect(retryOwes(reconciliation, SESSION)).toBe(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 1e3bed77f926..206925c10ba5 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -41,7 +41,7 @@ export type StructuredAgentSessionMutationContext = { /** A message was accepted: the session's delivery loop hands it over. */ wakeDelivery: (sessionId: string) => void /** The host's retry of background bookkeeping (`StructuredAgentSessionRetry`). */ - retry: Pick + retry: Pick /** Stops the session's provider child, keeping its conversation; inside the caller's serialize. * Each caller names why (`ending`). */ stopAgent: (sessionId: string, ending: StructuredAgentSessionStopEnding) => Promise diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 25ceb7c50e7a..05291f9b3d3d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -240,7 +240,7 @@ export type QueuedMessageDrainDeps = { /** The consumed submission is ordinary #22821 work from here on. */ wakeDelivery: (sessionId: string) => void /** The host's retry (`StructuredAgentSessionRetry`), read lazily: it is built after the drain. */ - retry: () => Pick + retry: () => Pick /** Re-sends what clients read (`AgentSessionSubscribers.publish`): an abandon writes nothing. */ publish: (sessionId: string, journal: AgentSessionJournal) => void logger: StructuredAgentSessionLogger @@ -254,6 +254,8 @@ export type QueuedMessageDrainDeps = { * re-derives everything and consumes at most one draft — the consumed submission * then owes work, which gates the next. */ +const LOCKED = { contended: true } as const + export class StructuredAgentSessionQueuedMessageDrain { private readonly scheduled = new Set() private disposed = false @@ -322,7 +324,7 @@ export class StructuredAgentSessionQueuedMessageDrain { return this.step(sessionId, false) }) // Another connection holds the database: the retry sends it in its next round, nothing shown. - .then((outcome) => outcome === 'contended' && this.deps.retry().sendContended(sessionId)) + .then((outcome) => outcome === 'contended' && this.deps.retry().signal(sessionId, LOCKED)) .catch((error: unknown) => { this.scheduled.delete(sessionId) this.warn(sessionId, error) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts index d8490fef5bf4..c07346e570ae 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -1,26 +1,14 @@ -// The host's one retry of background bookkeeping, scoped to its database connection: what ended -// generations left, what an earlier host process left, and the queue's automatic sends that met -// another connection's lock. One set of owed chats, one round at a time, one timer and one budget. -// -// Signals: every committed revocation or supersession of a generation (the record store's -// `onGenerationEnded`), an exit this host observed (whether or not its release landed), a row an -// ended generation committed late, a queue send that met contention, and startup. Each one -// re-derives what readers see at once (the operational revision, the published view, the queued- -// card drain) and marks the chat owed. A round visits every owed chat: closed ones load side by -// side in the few background slots, a reader's first, and each writes in its turn, one chat at a -// time with a yield between (`inTurn`): -// (a) the store-wide restart reconcile and the restore's latched recovery, once, while either is -// owed (`reconcileOwed`); then per chat, in -// its lane, (b) release repair, (c) settlement and what an earlier -// process left (`runStructuredAgentSessionReconciliationPass`), (d) the queue's next send. Every -// write is background (`JournalWriteOptions`): another connection's lock fails it at once and ends -// the round, since contention is connection-wide. -// -// A failed round backs off from 1 s to 2 s; about 3 minutes of them in a row end the episode -// (`RECONCILIATION_GIVE_UP_MS`): the timer stops, the set is kept, and a queue send still owed -// shows as not sent (`abandonSend`). A new signal, a reader's open, or a commit on the connection -// after contention starts a fresh episode. A lease latched in recovery is decided in (a) only for a -// chat the restart restore made readable; any other one's proofs wait in the set for its decision. +// The host's one retry of background bookkeeping on its database connection: what ended +// generations and an earlier host process left, and queue sends that met another connection's lock. +// One owed set (`signal`; a reader's open puts an owed chat first). A round runs (a) the store-wide +// step while owed (`reconcileOwed`), then per owed chat, one at a time in its lane, (b) release +// repair, (c) settlement (`runStructuredAgentSessionReconciliationPass`), (d) the queue's next send. +// Every write is background: a lock fails it at once and stops the round. +// One wake (`wake`): marking a chat owed or any commit runs a round now unless one runs or the timer +// waits; a commit after a lock refused the last round ends that wait. The timer is the backoff +// after a refused round (1 s to 2 s), or its first step after a busy lane. One budget: about 3 +// minutes of refused rounds give up (`RECONCILIATION_GIVE_UP_MS`), keeping the set and showing owed +// sends as not sent; the next wake after that is the only reset. import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' import { setImmediate as yieldToEvents } from 'node:timers/promises' @@ -52,7 +40,7 @@ export type StructuredAgentSessionReconciliationSignal = { exit?: StructuredAgentSessionExitSettlement /** Readers re-baseline at the moved fence (`publishGenerationEnded`). */ restate?: boolean - /** The exit's own settlement met another connection's lock. */ + /** The exit's own writes, or the queue's send, met another connection's lock. */ contended?: boolean } @@ -63,16 +51,16 @@ export class StructuredAgentSessionRetry { private readonly firstOpened = new Map() private readonly slots = new StructuredAgentSessionReconciliationSlots(() => this.disposed) private readonly unsubscribe: () => void + /** The one timer ('soon': a round queued now). */ private timer: ReturnType | 'soon' | null = null private running = false - /** A chat became due while a round ran: another follows at once. */ + /** Woken while a round ran: another follows at once. */ private again = false - /** Consecutive failed rounds in this episode, and the backoffs they waited out; at the budget - * (`RECONCILIATION_GIVE_UP_MS`) the episode ends. */ + /** Refused rounds in a row, and the backoffs they waited out; at the budget the episode ends. + * `locked`: the last one met another connection's lock, which a commit here proves gone. */ private failures = 0 private failingFor = 0 - /** The last round failed on another connection's lock, which a commit here proves gone. */ - private contended = false + private locked = false private disposed = false /** The round's chain of chat turns (`inTurn`), and whether contention stopped the round. */ private turn: Promise = Promise.resolve() @@ -86,13 +74,7 @@ export class StructuredAgentSessionRetry { firstOpened: this.firstOpened, settle: (sessionId, chat) => this.settle(sessionId, chat), warn: (sessionId, error) => this.warn(sessionId, error), - inTurn: (run) => this.inTurn(run), - // Once the lane's own bookkeeping has cleared too, so the next visit finds it free. - afterLane: (sessionId) => - void context - .serialize(sessionId, async () => undefined) - .then(() => yieldToEvents()) - .then(() => this.roundSoon()) + inTurn: (run) => this.inTurn(run) } this.unsubscribe = context.deps.store.onGenerationEnded((ended) => this.signal(ended.sessionId, { evidence: ended.evidence }) @@ -112,47 +94,32 @@ export class StructuredAgentSessionRetry { chat.debts.exit = signal.exit } if (chat && signal.contended) { - // The chat's queued send would meet the same lock: it waits for the round, which sends it. + // Its queued send met, or would meet, the same lock: the round sends it. chat.send = true } this.context.publishGenerationEnded(sessionId, signal.restate ? { restate: true } : {}) - this.freshEpisode() + this.wake() } - /** A committed row: one below the lease's fence is an ended generation's late write, and any - * commit proves the connection free again after contention. */ + /** A committed row: one below the lease's fence is an ended generation's late write; any one + * wakes the retry. */ observeCommit = (sessionId: string, lowestFence: number | null): void => { const fence = this.context.deps.store.getRecord(sessionId)?.lease.runtimeFence if (lowestFence !== null && fence !== undefined && lowestFence < fence) { this.signal(sessionId) - } else if (this.contended) { - this.freshEpisode() + } else if ([...this.owed.values()].some((chat) => chat.due)) { + this.wake(this.locked) } } - /** A reader opened the chat: its handle marks the process boundary if it is the first, and what - * it owes goes ahead of the scan, in a fresh episode unless a backoff runs. */ + /** A reader opened the chat: its handle marks the process boundary if it is the first, and an + * owed chat goes ahead of the scan. */ opened = (sessionId: string, journal: Pick): void => { noteStructuredAgentSessionOpened(this.firstOpened, sessionId, journal) const chat = this.owed.get(sessionId) if (chat) { this.slots.prioritize(chat) - // Backing off after a failed round, it goes first in the next, which an open does not - // hurry: a store that keeps refusing costs a write per round, never one per chat opened. - if (this.failures === 0 || this.failingFor >= RECONCILIATION_GIVE_UP_MS) { - this.freshEpisode() - } - } - } - - /** The queue's automatic send met another connection's lock: the next round sends it again. - * Owed, not a failure: only a round counts against the budget, however many sends collided. */ - sendContended = (sessionId: string): void => { - if (!this.disposed) { - this.owe(sessionId).send = true - this.contended = true - this.context.publishGenerationEnded(sessionId) - this.roundSoon() + this.wake() } } @@ -188,32 +155,32 @@ export class StructuredAgentSessionRetry { } chat.due = true chat.dirty = true - this.again ||= this.running return chat } - /** A new reason to try: the budget resets and a round runs now. */ - private freshEpisode(): void { + /** The one wake: a round now, unless one runs (it runs again) or the timer is pending, which + * `early` (a commit after a lock refused the last round) ends. After a give-up it starts a fresh + * episode, the budget's only reset. */ + private wake(early = false): void { if (this.disposed) { return } - this.failures = 0 - this.failingFor = 0 - this.contended = false - if (!this.running && this.timer !== 'soon') { - this.stopTimer() - this.roundSoon() - } - } - - /** A round now unless one runs, is due or waits out a backoff; the budget stays as it is. */ - private roundSoon(): void { if (this.running) { this.again = true - } else if (!this.disposed && this.timer === null) { - this.timer = 'soon' - queueMicrotask(() => void this.round()) + return + } + if (early) { + this.stopTimer() + } + if (this.timer !== null) { + return } + if (this.failingFor >= RECONCILIATION_GIVE_UP_MS) { + this.failures = 0 + this.failingFor = 0 + } + this.timer = 'soon' + queueMicrotask(() => void this.round()) } private async round(): Promise { @@ -225,10 +192,12 @@ export class StructuredAgentSessionRetry { this.again = false this.stopped = false let failed = false - let contended = false + let locked = false + let busy = false const end = (visit: Visit) => { failed ||= visit === 'failed' || visit === 'contended' - contended ||= visit === 'contended' + locked ||= visit === 'contended' + busy ||= visit === 'busy' this.stopped ||= visit === 'contended' this.again ||= visit === 'again' } @@ -251,39 +220,37 @@ export class StructuredAgentSessionRetry { }) ) this.running = false - this.roundEnded(failed, contended) + this.roundEnded(failed, locked, busy) // After the round's outcome is set, so a waiter reads its backoff, never a round still running. for (const [, chat] of due) { chat.attempted.splice(0).forEach((resolve) => resolve()) } } - private roundEnded(failed: boolean, contended: boolean): void { - if (this.disposed || this.running) { + private roundEnded(failed: boolean, locked: boolean, busy: boolean): void { + if (this.disposed) { return } + this.locked = locked if (!failed) { this.failures = 0 this.failingFor = 0 - this.contended = false - if (this.again && this.timer === null) { - this.freshEpisode() + if (this.again) { + this.wake() + } else if (busy) { + // A person's operation held a lane: its chat waits the timer's first step, not a budget. + this.arm(reconciliationBackoffDelay(1)) } return } this.failures += 1 - this.contended = contended - this.stopTimer() if (this.failingFor < RECONCILIATION_GIVE_UP_MS) { const delay = reconciliationBackoffDelay(this.failures) this.failingFor += delay - const timer = setTimeout(() => void this.round(), delay) - // A backoff alone never keeps the process alive. - timer.unref?.() - this.timer = timer + this.arm(delay) return } - // The episode ends; the set stays for the next signal, open or commit. + // The episode ends; the set stays for the next wake. this.context.deps.logger.warn("gave up settling a gone agent's leftover work for now", { scope: 'reconciliation', error: new Error(`${this.failures} rounds failed`) @@ -295,18 +262,24 @@ export class StructuredAgentSessionRetry { } } - /** An open chat goes straight to its lane; a closed one is replayed in a background slot. */ + private arm(delay: number): void { + this.stopTimer() + const timer = setTimeout(() => void this.round(), delay) + // A backoff alone never keeps the process alive. + timer.unref?.() + this.timer = timer + } + + /** An open chat, or one already loaded, goes straight to its lane; a closed one is replayed in a + * background slot, never once the round stopped. */ private async visitInSlot(sessionId: string, chat: OwedChat): Promise { if (this.context.sessions.has(sessionId) || chat.loaded) { return this.context.track(this.visit(sessionId, chat)) } // The replay yields a macrotask first (`loadStructuredAgentSessionForReconciliation`). - const result = await this.slots.run(chat, async () => { - if (this.disposed) { - return null - } - return this.context.track(this.visit(sessionId, chat)) - }) + const result = await this.slots.run(chat, async () => + this.disposed || this.stopped ? null : this.context.track(this.visit(sessionId, chat)) + ) return result?.value ?? null } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts index d66066cdd96a..43942dd4f17c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-visit.ts @@ -37,7 +37,7 @@ export type StructuredAgentSessionRetryContext = StructuredAgentSessionReconcili } /** `again`: it wrote, or was signalled meanwhile, so the next round verifies nothing is left. - * `busy`: a person's operation held its lane, so it waits, owed, for that lane to free. */ + * `busy`: a person's operation held its lane, so it waits, owed, for a later round. */ export type StructuredAgentSessionVisit = | 'settled' | 'again' @@ -56,7 +56,8 @@ export type StructuredAgentSessionOwedChat = StructuredAgentSessionReconciliatio /** Signalled while its visit ran: the next round visits it again. */ dirty: boolean attempted: (() => void)[] - /** The visit's own read of the chat while nobody has it open; closed when the chat settles. */ + /** The visit's own read of the chat while nobody has it open, kept across rounds; closed when + * the chat settles, a reader opens it, or another handle wrote past it. */ loaded?: StructuredAgentSessionHostSession } @@ -71,8 +72,6 @@ export type StructuredAgentSessionVisitHost = { warn: (sessionId: string, error: unknown) => void /** The round's one-at-a-time turn for a chat's writes; null once the round stopped. */ inTurn: (run: () => Promise) => Promise - /** A round once the chat's busy lane frees. */ - afterLane: (sessionId: string) => void } type Chat = StructuredAgentSessionOwedChat @@ -106,11 +105,8 @@ export async function visitStructuredAgentSessionOwedChat( return host.settle(sessionId, chat) } } + // Its read stays loaded should the round stop before its turn: the next one writes, not replays. const visit = await host.inTurn(() => write(host, sessionId, chat)) - if (visit === null) { - // The round stopped before this chat's turn: its read is not kept open until the next. - dropStructuredAgentSessionLoaded(chat) - } return visit === 'settled' ? host.settle(sessionId, chat) : visit } catch (error) { host.warn(sessionId, error) @@ -120,7 +116,7 @@ export async function visitStructuredAgentSessionOwedChat( /** The chat's writes, in its lane: the pass, then (d) once nothing else of it is due, the send it * owes, with the card named again. Never waits for a lane inside the round's turn: a person's - * operation holding it leaves the chat owed for when it frees. */ + * operation holding it leaves the chat owed for a later round. */ async function write( host: StructuredAgentSessionVisitHost, sessionId: string, @@ -128,7 +124,6 @@ async function write( ): Promise { const { context } = host if (context.laneBusy(sessionId)) { - host.afterLane(sessionId) return 'busy' } return context.serialize(sessionId, async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts index d039fc31f471..3b4990e40ecb 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-budget.test.ts @@ -60,7 +60,7 @@ it('ten chats whose automatic sends meet one lock cost one round, and send once // Each chat's own drain met the lock at the same moment. for (const id of ids) { - retry.sendContended(id) + retry.signal(id, { contended: true }) } await settleTurns() // One round, ended by the first send that met the lock: one failure, nobody given up on. diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts new file mode 100644 index 000000000000..e3810b04896a --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-scope.test.ts @@ -0,0 +1,203 @@ +// What a round of the host's retry touches: a chat's replay is read once however many rounds a +// lock refuses, a latch that comes after the restart restore waits for its own chat, and nothing +// the store-wide step cannot decide holds the other chats back or is probed round after round. + +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { AgentSessionOwnerProbe } from '../../../shared/agent-session-lease-adjudication' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { + openTestAgentSessionRecordStore, + seedTestAgentSessionRecordStore +} from '../../runtime/agent-session-record-store-test-harness' +import { + closeTestJournalHostDatabases, + openTestJournalHostDatabase +} from '../agent-session-journal/journal-host-database-test-support' +import type { StructuredAgentSessionHost } from './structured-agent-session-host' +import * as reconciliationLoad from './structured-agent-session-reconciliation-load' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' +import { + openScanHost, + scanIdle, + scanRecord, + seedScanJournal +} from './structured-agent-session-startup-scan.test-fixture' + +const CHAT = 'chat-aaaaaaa1' +const OTHER = 'chat-aaaaaaa2' + +let root: string +let store: AgentSessionRecordStore +let host: StructuredAgentSessionHost | undefined + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'orca-retry-scope-')) +}) + +afterEach(async () => { + await host?.flushAllStreamedEvents() + host = undefined + vi.restoreAllMocks() + closeTestJournalHostDatabases() + await rm(root, { recursive: true, force: true }) +}) + +const busy = () => + Object.assign(new Error('database is locked'), { errcode: 5, code: 'SQLITE_BUSY' }) +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** Records and journals for `ids`; `alive` names chats whose agent outlived the earlier process. */ +async function seed(ids: readonly string[], alive: readonly string[] = []) { + await seedTestAgentSessionRecordStore(root, { + records: ids.map((id) => scanRecord(id, !alive.includes(id))) + }) + for (const id of ids) { + await seedScanJournal(root, id) + } + store = await openTestAgentSessionRecordStore(root) + vi.spyOn(console, 'warn').mockImplementation(() => undefined) +} + +/** Every pass on `contended` chats meets a lock while `held()`; the rest run as they are. */ +function lockPasses(held: () => boolean, contended: (sessionId: string) => boolean) { + const pass = reconciliationPass.runStructuredAgentSessionReconciliationPass + return vi + .spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass') + .mockImplementation(async (...args) => + held() && contended(args[1]) + ? { failed: [busy()], wrote: false, recovering: false } + : pass(...args) + ) +} + +function surviving() { + let alive = true + const probeOwner = vi.fn(async (): Promise => + alive ? { outcome: 'identity-matched', matchedOn: ['spawn-token'] } : { outcome: 'pid-absent' } + ) + const stopOwnerProcess = vi.fn(() => { + alive = false + }) + return { probeOwner, stopOwnerProcess } +} + +const latch = (lease: Partial) => (record: AgentSessionRecord) => ({ + ...record, + lease: { ...record.lease, handoffStage: 'recovering' as const, ...lease } +}) + +it('replays each of 20 closed chats once while a lock refuses round after round', async () => { + const chats = Array.from({ length: 20 }, (_, i) => `chat-r${String(i).padStart(6, '0')}`) + await seed(chats) + let held = true + const passes = lockPasses( + () => held, + () => true + ) + const loads = vi.spyOn(reconciliationLoad, 'loadStructuredAgentSessionForReconciliation') + host = openScanHost(root, store) + await host.reconcileRestartLeases() + + // Rounds at 0 s, 1 s, 3 s and 5 s, each stopped by the lock. + await pause(5_500) + expect(passes.mock.calls.length).toBeGreaterThanOrEqual(3) + held = false + await scanIdle(host, chats) + + expect(loads.mock.calls.length).toBeLessThanOrEqual(20) +}, 30_000) + +it("never signals a restored chat's agent for a latch that came after the restore", async () => { + await seed([CHAT, OTHER]) + const { probeOwner, stopOwnerProcess } = surviving() + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + await host.reconcileRestartLeases() + await host.restoreReadableSessions([CHAT]) + await scanIdle(host, [CHAT, OTHER]) + const probes = probeOwner.mock.calls.length + + // Mid-session its lease latches, its owner alive; another chat's rounds run. + const { ownerProcess, reservedSpawnToken } = scanRecord(CHAT, false).lease + await store.transitionHandoff( + CHAT, + latch({ ownerProcess, reservedSpawnToken, claimStatus: 'reserved' }) + ) + let held = true + lockPasses( + () => held, + (sessionId) => sessionId === OTHER + ) + host.collaboratorsForTests().reconciliation.signal(OTHER) + await pause(3_500) + held = false + await scanIdle(host, [OTHER]) + + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(probeOwner).toHaveBeenCalledTimes(probes) + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') +}, 30_000) + +it('on a read-only store, attempts no store-wide step and still visits the other chats', async () => { + await seed([CHAT, OTHER], [CHAT]) + // A newer Orca wrote the database: a latched lease here stays latched. + Object.defineProperty(openTestJournalHostDatabase(root), 'readOnly', { value: true }) + const { probeOwner, stopOwnerProcess } = surviving() + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + await host.reconcileRestartLeases() + await host.restoreReadableSessions([CHAT]) + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') + const reconcile = vi.spyOn(store, 'reconcileOnRestart') + const probes = probeOwner.mock.calls.length + const stops = stopOwnerProcess.mock.calls.length + + host.collaboratorsForTests().reconciliation.signal(OTHER) + await scanIdle(host, [CHAT, OTHER]) + await pause(3_500) + + expect(reconcile).not.toHaveBeenCalled() + expect(probeOwner).toHaveBeenCalledTimes(probes) + expect(stopOwnerProcess).toHaveBeenCalledTimes(stops) +}, 30_000) + +it.each([ + ['a conflicted claim', { claimStatus: 'conflicted' as const }], + [ + 'an owner on another host', + { + ownerProcess: { hostId: 'other', pid: 12_000, processStartTimeMs: 0, spawnToken: 'spawn' } + } + ] +])( + 'never re-probes %s round after round', + async (_name, lease) => { + await seed([CHAT, OTHER], [CHAT]) + const { probeOwner, stopOwnerProcess } = surviving() + host = openScanHost(root, store, { probeOwner, stopOwnerProcess }) + await host.reconcileRestartLeases() + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') + await store.transitionHandoff(CHAT, latch(lease)) + await host.restoreReadableSessions([CHAT]) + await scanIdle(host, [CHAT]) + const probes = probeOwner.mock.calls.length + + // Another chat's rounds, refused by a lock at 0 s, 1 s and 3 s. + let held = true + lockPasses( + () => held, + (sessionId) => sessionId === OTHER + ) + host.collaboratorsForTests().reconciliation.signal(OTHER) + await pause(3_500) + held = false + await scanIdle(host, [OTHER]) + + expect(probeOwner).toHaveBeenCalledTimes(probes) + expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(store.getRecord(CHAT)?.lease.handoffStage).toBe('recovering') + }, + 30_000 +) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts new file mode 100644 index 000000000000..384425b6bac3 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts @@ -0,0 +1,151 @@ +// The retry's one wake and one budget: a commit after a lock refused a round runs the next one at +// once; reads during a backoff add no round and leave the budget alone; after a give-up only a +// newly owed chat or a commit starts a fresh episode. + +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as reconciliationPass from './structured-agent-session-reconciliation-pass' +import { RECONCILIATION_GIVE_UP_MS } from './structured-agent-session-reconciliation-backoff' +import { + StructuredAgentSessionRetry, + type StructuredAgentSessionRetryContext +} from './structured-agent-session-reconciliation-retry' +import { retryOwes } from './structured-agent-session-retry.test-fixture' + +const CHAT = 'chat-1' +const journal = { openedAt: () => ({ epoch: 'e', sequence: 0 }) } +const locked = () => Object.assign(new Error('database is locked'), { errcode: 5 }) + +let lockHeld = true +let pass: ReturnType + +beforeEach(() => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + lockHeld = true + pass = vi + .spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass') + .mockImplementation(async () => ({ + failed: lockHeld ? [locked()] : [], + wrote: false, + recovering: false + })) +}) + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +async function settleTurns(): Promise { + for (let turn = 0; turn < 30; turn++) { + await new Promise((resolve) => setImmediate(resolve)) + } +} + +function openRetry(): StructuredAgentSessionRetry { + const context = { + deps: { + store: { + onGenerationEnded: () => () => undefined, + getRecord: () => ({ lease: { unreconciled: false, runtimeFence: 1 } }), + listRecords: () => [], + readOnly: false + }, + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() } + }, + sessions: new Map([[CHAT, { journal }]]), + now: () => 0, + serialize: (_id: string, task: () => Promise) => task(), + laneBusy: () => false, + track: (operation: Promise) => operation, + publishGenerationEnded: vi.fn(), + reconcileOwed: () => false, + reconcile: async () => 'settled' as const, + sendQueued: async () => 'done' as const, + abandonSend: vi.fn() + } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the retry reads only the fields above; the host's full deps would add nothing these tests exercise. + const typed = context as unknown as StructuredAgentSessionRetryContext + return new StructuredAgentSessionRetry(typed) +} + +/** Rounds refused by the lock until the episode gives up. */ +async function exhaust(retry: StructuredAgentSessionRetry): Promise { + retry.signal(CHAT) + await settleTurns() + while (retry['failingFor'] < RECONCILIATION_GIVE_UP_MS || retry['timer'] !== null) { + await vi.advanceTimersByTimeAsync(2_000) + await settleTurns() + } +} + +it('a commit after a lock refused a round runs the next one at once, with no timer', async () => { + const retry = openRetry() + retry.signal(CHAT) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(1) + + lockHeld = false + retry.observeCommit('another-chat', null) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(2) + expect(retryOwes(retry, CHAT)).toBe(false) + retry.dispose() +}) + +it('reads during a backoff add no round and leave the budget as it is', async () => { + const retry = openRetry() + retry.signal(CHAT) + await settleTurns() + const waited = retry['failingFor'] + + for (let read = 0; read < 20; read += 1) { + retry.opened(CHAT, journal) + } + await settleTurns() + + expect(pass.mock.calls.length).toBeLessThanOrEqual(2) + expect(retry['failingFor']).toBe(waited) + retry.dispose() +}) + +it('before a give-up, a signal, a commit or a read leaves the budget counting', async () => { + const retry = openRetry() + retry.signal(CHAT) + await settleTurns() + await vi.advanceTimersByTimeAsync(3_000) + await settleTurns() + const waited = retry['failingFor'] + expect(waited).toBeGreaterThan(0) + + retry.signal(CHAT) + retry.observeCommit('another-chat', null) + retry.opened(CHAT, journal) + await settleTurns() + + expect(retry['failingFor']).toBeGreaterThanOrEqual(waited) + retry.dispose() +}) + +it('after a give-up only a newly owed chat or a commit starts a fresh episode', async () => { + const retry = openRetry() + await exhaust(retry) + const given = pass.mock.calls.length + // Nothing of its own: no timer, however long. + await vi.advanceTimersByTimeAsync(600_000) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(given) + + retry.signal(CHAT) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(given + 1) + expect(retry['failingFor']).toBeLessThan(RECONCILIATION_GIVE_UP_MS) + + await exhaust(retry) + const again = pass.mock.calls.length + lockHeld = false + retry.observeCommit('another-chat', null) + await settleTurns() + expect(pass).toHaveBeenCalledTimes(again + 1) + expect(retryOwes(retry, CHAT)).toBe(false) + retry.dispose() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index b06b67a372cb..b3e4c9bbfff7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -105,15 +105,28 @@ export function createStructuredAgentSessionHostRestore( failures ) const recoveryWrites = backgroundLeaseWrites(deps.store) - // The chats the restart restore was asked to make readable, the visible tabs: theirs is the one - // latched recovery startup owes. A chat a reader opened behind them never is: a read stops no one. + // The visible tabs the restart restore was asked for whose leases it found latched or not yet + // reconciled: theirs is the one recovery decision startup owes. One decided, or found unlatched, + // leaves; a latch that comes later waits for its chat's own attach, start or send, as on main. const restoreTargets = new Set() - let restoresAll = false - const latchedTargets = () => - (restoresAll - ? deps.store.listRecords().map((record) => record.sessionId) - : [...restoreTargets] - ).filter((sessionId) => deps.store.getRecord(sessionId)?.lease.handoffStage === 'recovering') + let restoring = false + const latchedTargets = () => { + for (const sessionId of restoreTargets) { + const lease = deps.store.getRecord(sessionId)?.lease + if (!lease?.unreconciled && lease?.handoffStage !== 'recovering') { + restoreTargets.delete(sessionId) + } + } + // Only one a probe can decide: never a conflicted claim, nor an owner on another host. + return [...restoreTargets].filter((sessionId) => { + const lease = deps.store.getRecord(sessionId)?.lease + return ( + lease?.handoffStage === 'recovering' && + lease.claimStatus !== 'conflicted' && + (lease.ownerProcess?.hostId ?? deps.store.hostId) === deps.store.hostId + ) + }) + } const reconciliation = new StructuredAgentSessionRetry({ deps, sessions: startup.sessions, @@ -124,9 +137,11 @@ export function createStructuredAgentSessionHostRestore( track: (operation) => startup.tasks.trackAttach(operation), publishGenerationEnded: (sessionId, options) => startup.clientDelivery.publishGenerationEnded(sessionId, options), + // A read-only store decides nothing here; its chats' visits still run. reconcileOwed: () => - deps.store.listRecords().some((record) => record.lease.unreconciled) || - latchedTargets().length > 0, + !deps.store.readOnly && + (deps.store.listRecords().some((record) => record.lease.unreconciled) || + latchedTargets().length > 0), reconcile: async () => { try { const refusal = await reconcileLeases(null, BACKGROUND_WRITE) @@ -138,6 +153,7 @@ export function createStructuredAgentSessionHostRestore( for (const sessionId of latchedTargets()) { await resolveRecovery(sessionId, recoveryWrites) } + latchedTargets() // a decided target leaves at once failures.clear() return 'settled' } catch (error) { @@ -173,8 +189,16 @@ export function createStructuredAgentSessionHostRestore( }, startupSettled: () => startupSettled, restoreReadableSessions: (sessionIds) => { - restoresAll ||= sessionIds === undefined - sessionIds?.forEach((sessionId) => restoreTargets.add(sessionId)) + // Once, as the restorer restores once. + for (const { sessionId, lease } of restoring ? [] : deps.store.listRecords()) { + if ( + (lease.unreconciled || lease.handoffStage === 'recovering') && + (!sessionIds || sessionIds.includes(sessionId)) + ) { + restoreTargets.add(sessionId) + } + } + restoring = true return gate.run(() => restorer.restore(sessionIds)) }, reconciliation From 8c06757437cc16e2d577098c23bd33735ffe20d2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:00:52 -0700 Subject: [PATCH 17/22] fix(native-chat): an unreconciled owner reads unverifiable; retry budget keeps wall time - A lease still unreconciled or recovering that this process holds no child for reads its turn unverifiable and not working, as main did, with no write. - The retry's give-up is wall time since the episode's first refused round; only a round that made progress ends the episode, and give-up drops loaded reads. - Drop the unused settleStructuredAgentSessionDeadGeneration wrapper. --- ...claude-subagent-permission-request.test.ts | 4 +- .../codex-subagent-roster-retention.test.ts | 40 +++++++++ ...journal-atomic-terminal-settlement.test.ts | 20 ++--- ...ed-agent-session-background-writes.test.ts | 4 +- ...tructured-agent-session-client-delivery.ts | 1 + .../structured-agent-session-current-work.ts | 36 ++++++-- ...session-dead-generation-settlement.test.ts | 84 ++++------------- ...gent-session-dead-generation-settlement.ts | 29 ------ ...uctured-agent-session-host-current-work.ts | 3 +- ...agent-session-late-proof-tool-ends.test.ts | 9 +- ...-session-leftover-settlement-scope.test.ts | 5 +- ...gent-session-observed-exit.test-fixture.ts | 27 ++++++ ...ctured-agent-session-queued-publication.ts | 4 +- ...ured-agent-session-reasoning-sweep.test.ts | 10 +-- ...ed-agent-session-reconciliation-backoff.ts | 3 +- ...ured-agent-session-reconciliation-retry.ts | 54 ++++++----- ...agent-session-recovered-turn-clock.test.ts | 12 ++- ...agent-session-replaced-server-turn.test.ts | 10 +++ ...nt-session-restart-provider-output.test.ts | 11 +-- ...nt-session-restart-stream-duration.test.ts | 9 +- ...tructured-agent-session-retry-wake.test.ts | 90 ++++++++++++++++--- ...agent-session-status-journal-projection.ts | 2 + 22 files changed, 283 insertions(+), 184 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts diff --git a/src/main/claude/claude-subagent-permission-request.test.ts b/src/main/claude/claude-subagent-permission-request.test.ts index 95bfb84cb2b8..43f96eadcaad 100644 --- a/src/main/claude/claude-subagent-permission-request.test.ts +++ b/src/main/claude/claude-subagent-permission-request.test.ts @@ -23,7 +23,7 @@ import type { AgentStatusStructuredSessionSubject } from '../../shared/agent-sta import { AgentHookServer } from '../agent-hooks/server' import type { AgentSessionJournal } from '../native-chat/agent-session-journal/journal-store' import { createTrackedJournalOpener } from '../native-chat/agent-session-journal/journal-host-database-test-support' -import { settleStructuredAgentSessionDeadGeneration } from '../native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from '../native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture' import { StructuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' import { createDeferredStructuredAgentSessionEventSink } from '../native-chat/agent-session-wire/structured-agent-session-event-sink' import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' @@ -633,7 +633,7 @@ describe("a Claude subagent's permission request", () => { let settled: Promise | undefined hooks.onEvent = (event) => { if (event.type === 'ended') { - settled = settleStructuredAgentSessionDeadGeneration({ + settled = settleObservedExitForTest({ journal: run.journal, sessionId: SESSION, fence: FENCE, diff --git a/src/main/codex/codex-subagent-roster-retention.test.ts b/src/main/codex/codex-subagent-roster-retention.test.ts index 92de0fc2aea3..8b8ae481238b 100644 --- a/src/main/codex/codex-subagent-roster-retention.test.ts +++ b/src/main/codex/codex-subagent-roster-retention.test.ts @@ -12,6 +12,9 @@ import { import { testEventSinkLogging } from '../native-chat/agent-session-wire/structured-agent-session-logger-test-support' import { CodexSubagentRoster } from './codex-subagent-roster' import { MAX_CODEX_SUBAGENT_GROUPS } from './codex-structured-journal-limits' +import { structuredAgentSessionCurrentWork } from '../native-chat/agent-session-wire/structured-agent-session-current-work' +import { planStaleStructuredAgentSessionState } from '../native-chat/agent-session-wire/structured-agent-session-stale-state-settlement' +import { scanRecord } from '../native-chat/agent-session-wire/structured-agent-session-startup-scan.test-fixture' const THREAD = 'parent' const ORIGINAL = 'original' @@ -104,6 +107,7 @@ async function session(refuse?: 'append' | 'publish') { ) } return { + journal, roster, turn, announce, @@ -320,3 +324,39 @@ describe('Codex roster retention through the real sink and journal', () => { } ) }) + +// The roster retains a group while any child in it runs; the generation that wrote it can still end. +describe('a retained running child of a generation that ended', () => { + it('reads as not running and not working once a later generation holds the chat', async () => { + const s = await session() + s.spawn('old-a') + await laterGroups(s, true) + const original = await s.original() + expect(states(original)).toEqual([{ id: 'old-a', state: 'working' }]) + const itemId = original?.row.itemId ?? '' + + // Generation 1 wrote it; the lease now names generation 13, which is live. + const record = scanRecord('session-codex-roster-retention', false) + const work = structuredAgentSessionCurrentWork(s.journal, { record }) + expect(work.isCurrentItem(itemId)).toBe(false) + expect(work.working()).toBe(false) + + const plan = planStaleStructuredAgentSessionState({ + journal: s.journal, + sessionId: record.sessionId, + fence: 13, + acquisitionGeneration: null, + deathEvidence: null, + below: 13, + entriesBelow: 13 + }) + const revised = plan.mutations.find( + (mutation) => mutation.kind === 'item' && mutation.itemId === itemId + ) + const body = revised?.kind === 'item' ? revised.body : undefined + const groups = body?.kind === 'message' ? body.blocks.filter(isSubagentGroupBlock) : [] + expect(groups.flatMap((group) => group.agents.map(({ id, state }) => ({ id, state })))).toEqual( + [{ id: 'old-a', state: 'unverifiable' }] + ) + }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts b/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts index f20b24d60b3c..9321c7529180 100644 --- a/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-atomic-terminal-settlement.test.ts @@ -9,10 +9,8 @@ import { import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' import type { AgentSessionSubscribeEvent } from '../../../shared/agent-session-wire' import { AgentSessionSubscribers } from '../agent-session-wire/structured-agent-session-subscribers' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from '../agent-session-wire/structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from '../agent-session-wire/structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from '../agent-session-wire/structured-agent-session-observed-exit.test-fixture' import { createTrackedJournalOpener, openTestJournalHostDatabase, @@ -96,7 +94,7 @@ async function seedWork(count = 201) { } function settle() { - return settleStructuredAgentSessionDeadGeneration({ + return settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -191,7 +189,7 @@ describe('atomic terminal settlement', () => { return result }) ) - expect(await settle()).toEqual({ ok: true }) + expect(await settle()).toMatchObject({ ok: true }) vi.restoreAllMocks() const settled = journal.snapshot() expect(published).toEqual([settled]) @@ -230,7 +228,7 @@ describe('atomic terminal settlement', () => { 'lifecycle-batch' ]) expect(committed.every((row) => row.recovered)).toBe(true) - expect(await settle()).toEqual({ ok: true }) + expect(await settle()).toMatchObject({ ok: true }) expect(journal.snapshot()).toEqual(settled) expect(rows().slice(diskBefore.length)).toEqual(committed) expect(published).toEqual([settled]) @@ -238,7 +236,7 @@ describe('atomic terminal settlement', () => { await journal.close() journal = await open() expect(journal.snapshot()).toEqual(settled) - expect(await settle()).toEqual({ ok: true }) + expect(await settle()).toMatchObject({ ok: true }) expect(journal.snapshot()).toEqual(settled) } ) @@ -392,12 +390,12 @@ describe('atomic terminal settlement', () => { WHEN NEW.session_id = '${SESSION}' AND json_extract(NEW.row_json, '$.recovered') = 1 AND json_extract(NEW.row_json, '$.kind') = 'item' BEGIN SELECT RAISE(ABORT, 'item rejected after submission'); END`) - expect(await settleStructuredAgentSessionDeadGeneration(input)).toMatchObject({ ok: false }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: false }) expect(journal.snapshot()).toEqual(before) expect(rows()).toEqual(diskBefore) expect(published).toEqual([]) database.db.exec('DROP TRIGGER reject_submission_settlement_item') - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) expect(journal.submission('unanswered')).toMatchObject({ dispatchState: 'rejected', recovered: true, @@ -406,7 +404,7 @@ describe('atomic terminal settlement', () => { const committed = rows().slice(diskBefore.length) expect(committed.filter((row) => row.kind === 'dispatch')).toHaveLength(1) expect(published).toEqual([journal.snapshot()]) - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) expect(rows().slice(diskBefore.length)).toEqual(committed) expect(published).toHaveLength(1) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts index 657a7c1a62ea..c3e443f7ca7f 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-background-writes.test.ts @@ -169,7 +169,7 @@ describe('an episode the retry gives up', () => { 'is re-armed by %s, and the work lands', async (trigger) => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) const current = await settlementOwed() await exhaust(current) expect(turnState(current)).toBe('running') @@ -193,7 +193,7 @@ describe('an episode the retry gives up', () => { describe('a queued send the retry gives up on', () => { it('reads as not sent from memory when its stored hold cannot be written either', async () => { vi.spyOn(console, 'warn').mockImplementation(() => undefined) - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) rig = await createQueuedMessageTestRig({ restartable: true }) const current = rig const working = await current.workingSend() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index 7f369a4934b8..c1a07719ebf9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -203,6 +203,7 @@ export class StructuredAgentSessionClientDelivery { return structuredAgentSessionCurrentWork(journal, { record: this.deps().store.getRecord(sessionId), replaced: this.deps().store.replacedRuntime(sessionId), + ...(session ? { child: session.child } : {}), ...(ended ? { ended } : {}), revision: session?.operationalRevision ?? 0 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index d0bd40155509..37243f698354 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -41,6 +41,8 @@ export type StructuredAgentSessionWorkEvidence = { revision?: number /** The runtimes this one replaced for the chat (`AgentSessionRecordStore.replacedRuntime`). */ replaced?: StructuredAgentSessionReplacedEnd | undefined + /** This host's live child (`StructuredAgentSessionHostSession.child`); absent: not known. */ + child?: { fence: number } | null } /** Every fence at or below `fence` was granted by a runtime this one replaced. */ @@ -60,13 +62,27 @@ export function structuredAgentSessionLiveFence( if (lease.claimStatus === 'released') { return null } - if (structuredAgentSessionReplacedRuntimeEnded(lease, evidence.replaced)) { + if ( + structuredAgentSessionReplacedRuntimeEnded(lease, evidence.replaced) || + structuredAgentSessionOwnerUnverifiable(evidence) + ) { return null } const { ended } = evidence return ended?.rootGone === true && ended.fence === lease.runtimeFence ? null : lease.runtimeFence } +/** An owner not yet reconciled, or latched in recovery, that this process holds no child for: it + * may still be running, so its turn is unverifiable, and nothing it left is current work. */ +function structuredAgentSessionOwnerUnverifiable(evidence: StructuredAgentSessionWorkEvidence) { + const lease = evidence.record?.lease + return ( + (lease?.unreconciled === true || lease?.handoffStage === 'recovering') && + evidence.child !== undefined && + evidence.child?.fence !== lease.runtimeFence + ) +} + /** Whether a runtime this one replaced granted the lease's generation, which is then over. A * conflicted claim's agent is a terminal's, which no runtime's pipes held. */ export function structuredAgentSessionReplacedRuntimeEnded( @@ -110,7 +126,9 @@ export class StructuredAgentSessionCurrentWork { * what it derived from this answer keys it here and on the journal's cursor. */ readonly revision = 0, private readonly ended?: StructuredAgentSessionSeenEnd, - private readonly replaced?: StructuredAgentSessionReplacedEnd + private readonly replaced?: StructuredAgentSessionReplacedEnd, + /** `structuredAgentSessionOwnerUnverifiable`. */ + private readonly ownerUnverifiable = false ) {} /** What a reader that published this answer compares to learn it changed with no row: the live @@ -210,14 +228,17 @@ export class StructuredAgentSessionCurrentWork { } /** The newest turn record as a client is told it: a running one an ended generation opened is - * none, or interrupted when it died on its own (`diedTurn`), so no client reads it as working, - * steers into it or reads it finished. */ + * none, interrupted when it died on its own (`diedTurn`), or unverifiable while its owner is, so + * no client reads it as working, steers into it or reads it finished. */ publishedLatestTurn(latest: AgentSessionLatestTurn | null): AgentSessionLatestTurn | null { if (latest?.turn.state !== 'running' || this.isCurrentItem(latest.itemId)) { return latest } - return this.diedTurn()?.item.itemId === latest.itemId - ? { ...latest, turn: { ...latest.turn, state: 'interrupted' } } + if (this.diedTurn()?.item.itemId === latest.itemId) { + return { ...latest, turn: { ...latest.turn, state: 'interrupted' } } + } + return this.ownerUnverifiable + ? { ...latest, turn: { ...latest.turn, state: 'unverifiable' } } : null } } @@ -231,7 +252,8 @@ export function structuredAgentSessionCurrentWork( structuredAgentSessionLiveFence(evidence), evidence.revision, evidence.ended, - evidence.replaced + evidence.replaced, + structuredAgentSessionOwnerUnverifiable(evidence) ) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts index ba466bf7dcb0..21d59549faa0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.test.ts @@ -4,14 +4,13 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' -import type { AgentJournalRenderItem } from '../../../shared/agent-session-journal-types' import { agentSessionFailureFact, MAX_PROVIDER_DIAGNOSTIC_CHARS, providerDiagnostic } from '../../../shared/agent-session-failure' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { settleStructuredAgentSessionDeadGeneration } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import { captureUnfinishedStructuredAgentSessionWork, unfinishedStructuredAgentSessionWorkWasInterrupted @@ -93,7 +92,7 @@ describe('dead structured-session generation settlement', () => { await seedUnfinishedWork() await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, @@ -102,7 +101,7 @@ describe('dead structured-session generation settlement', () => { verdict: { state: 'unverifiable' }, showUnexpectedExitOutcome: false }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) const snapshot = journal.snapshot() expect(snapshot.submissions).toEqual([ @@ -141,9 +140,9 @@ describe('dead structured-session generation settlement', () => { showUnexpectedExitOutcome: true } - await expect(settleStructuredAgentSessionDeadGeneration(input)).resolves.toEqual({ ok: true }) + await expect(settleObservedExitForTest(input)).resolves.toMatchObject({ ok: true }) const settledCursor = journal.cursor() - await expect(settleStructuredAgentSessionDeadGeneration(input)).resolves.toEqual({ ok: true }) + await expect(settleObservedExitForTest(input)).resolves.toMatchObject({ ok: true }) expect(journal.cursor()).toEqual(settledCursor) expect( @@ -158,7 +157,7 @@ describe('dead structured-session generation settlement', () => { it('cuts short a call the proven death interrupted', async () => { await seedUnfinishedWork() - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -193,7 +192,7 @@ describe('dead structured-session generation settlement', () => { verdict: { state: 'interrupted' as const, completedAt: 1_000 }, showUnexpectedExitOutcome: true } - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) const settled = journal.snapshot() expect(settled.items).toContainEqual(terminal) expect(settled.items.map((item) => item.body)).toContainEqual({ @@ -203,7 +202,7 @@ describe('dead structured-session generation settlement', () => { state: 'failed', endedAs: 'interrupted' }) - expect(await settleStructuredAgentSessionDeadGeneration(input)).toEqual({ ok: true }) + expect(await settleObservedExitForTest(input)).toMatchObject({ ok: true }) expect(journal.snapshot()).toEqual(settled) }) @@ -211,7 +210,7 @@ describe('dead structured-session generation settlement', () => { await seedUnfinishedWork() await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -223,7 +222,7 @@ describe('dead structured-session generation settlement', () => { detail: providerDiagnostic('stack frame '.repeat(4_000), 'log') }) }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) const statuses = journal .snapshot() @@ -238,7 +237,7 @@ describe('dead structured-session generation settlement', () => { it("words Orca's own fault as Orca's, never as the provider stopping", async () => { await seedUnfinishedWork() - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -263,55 +262,6 @@ describe('dead structured-session generation settlement', () => { ]) }) - it('retries an already settled expected close without writing through a closed journal gate', async () => { - const settledItem: AgentJournalRenderItem = { - itemId: 'codex:thread-1:turn-1:0', - revision: 2, - sequence: 2, - observedAt: 1_000, - body: { - kind: 'turn', - turnId: 'turn-1', - state: 'interrupted', - completedAt: 1_000 - } - } - const settledSnapshot = journal.snapshot() - const closedJournal: Pick< - AgentSessionJournal, - | 'snapshot' - | 'submissions' - | 'rejectQueuedSubmissions' - | 'appendPlannedLifecycleBatch' - | 'itemFence' - > = { - itemFence: () => undefined, - snapshot: () => ({ - ...settledSnapshot, - items: [settledItem] - }), - submissions: () => [], - rejectQueuedSubmissions: async () => { - throw new Error('journal_closed') - }, - appendPlannedLifecycleBatch: async () => { - throw new Error('journal_closed') - } - } - - await expect( - settleStructuredAgentSessionDeadGeneration({ - journal: closedJournal, - sessionId: SESSION, - fence: 7, - settlementId: `expected-close:${SESSION}:7:generation-1`, - pendingSubmissionReason: 'provider_closed_before_acknowledgement', - verdict: { state: 'interrupted', completedAt: 1_000 }, - showUnexpectedExitOutcome: false - }) - ).resolves.toEqual({ ok: true }) - }) - it('settles a live unknown submission even when no unfinished item remains', async () => { await journal.appendSubmission({ clientMessageId: 'client-unknown', @@ -327,7 +277,7 @@ describe('dead structured-session generation settlement', () => { }) await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -336,7 +286,7 @@ describe('dead structured-session generation settlement', () => { verdict: { state: 'interrupted', completedAt: 1_000 }, showUnexpectedExitOutcome: false }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) expect(journal.submissions()).toEqual([ expect.objectContaining({ @@ -356,7 +306,7 @@ describe('dead structured-session generation settlement', () => { fence: 7 }) - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -417,7 +367,7 @@ describe('dead structured-session generation settlement', () => { { fence: 7, ...child, turnScope: AGENT_JOURNAL_THREAD_SCOPE } ) - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, @@ -501,7 +451,7 @@ describe('whether a dead generation interrupted anything', () => { await seedIdlePendingApproval() await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 7, @@ -514,7 +464,7 @@ describe('whether a dead generation interrupted anything', () => { 1_000 ) }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) const snapshot = journal.snapshot() expect(snapshot.items.some((item) => item.body.kind === 'status')).toBe(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts index 2c351fb2941f..41890413e13c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-dead-generation-settlement.ts @@ -49,11 +49,6 @@ export { settleStaleStructuredAgentSessionState } from './structured-agent-sessi * same cap. */ export const MAX_UNEXPECTED_EXIT_REASON_CHARS = MAX_PROVIDER_DIAGNOSTIC_CHARS -/** Whether the settlement was written, and what stopped it when it was not. */ -export type StructuredAgentSessionDeadGenerationSettlement = - | { ok: true } - | { ok: false; error: unknown } - export type StructuredAgentSessionDeadGenerationInput = { journal: DeadGenerationJournal sessionId: string @@ -77,30 +72,6 @@ export type StructuredAgentSessionDeadGenerationInput = { unrunRejection?: SubmissionRejectionFact } -/** The exited child's own settlement, planned at the batch's own turn in the write queue. */ -export async function settleStructuredAgentSessionDeadGeneration( - input: StructuredAgentSessionDeadGenerationInput -): Promise { - try { - if ( - !input.showUnexpectedExitOutcome && - !hasUnfinishedStructuredAgentSessionWork(input.journal, input.exit) - ) { - return { ok: true } - } - await input.journal.appendPlannedLifecycleBatch({ - settlementId: `dead-generation:${input.settlementId}`, - fence: input.fence, - recovered: true, - plan: () => planStructuredAgentSessionDeadGeneration(input) - }) - return { ok: true } - } catch (error) { - // Returned rather than logged: each caller logs it under its own scope. - return { ok: false, error } - } -} - /** The exit's rows, read from the journal as it stands. `owns`: which fences are the exited * generation's (and later); the rest are an earlier generation's, settled by its own rule. */ export function planStructuredAgentSessionDeadGeneration( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts index dad5bb06a6af..5e152573cae7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts @@ -18,7 +18,7 @@ export type StructuredAgentSessionCurrentWorkHost = { ): | Pick< StructuredAgentSessionHostSession, - 'journal' | 'lastEndedChild' | 'operationalRevision' + 'journal' | 'lastEndedChild' | 'operationalRevision' | 'child' > | undefined } @@ -34,6 +34,7 @@ export function hostStructuredAgentSessionCurrentWork( ? structuredAgentSessionCurrentWork(session.journal, { record: host.store.getRecord(sessionId), replaced: host.store.replacedRuntime(sessionId), + child: session.child, ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), revision: session.operationalRevision ?? 0 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts index b9093d850994..3ba3e3043892 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-late-proof-tool-ends.test.ts @@ -12,10 +12,8 @@ import type { AgentSessionDeathEvidence } from '../../../shared/agent-session-re import { readAgentJournalTurn } from '../../../shared/agent-session-turn-record' import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from './structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import { codexProviderHandle } from '../../../shared/agent-session-provider-handle-encoding' const SESSION = 'session-late-proof' @@ -153,10 +151,11 @@ describe('a proof written after an unverifiable settle', () => { it('corrects a call an unverifiable restart eviction closed', async () => { await seedTurn('turn-1', 7) - await settleStructuredAgentSessionDeadGeneration({ + await settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, + ownerFence: 7, settlementId: `restart-eviction:${SESSION}:8`, pendingSubmissionReason: 'provider_exited_before_acknowledgement', verdict: { state: 'unverifiable' }, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts index 4d5ab34f4cdb..69ba41d34b97 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -126,7 +126,7 @@ describe('a release nothing proved', () => { }) describe('an owner that may still run', () => { - it('keeps an unreconciled owner’s current work, which still holds, and settles nothing', async () => { + it('reads an unreconciled owner’s work as unverifiable, holding nothing, and settles nothing', async () => { rig = await createQueuedMessageTestRig({ restartable: true, probeOwner: async () => { @@ -143,7 +143,8 @@ describe('an owner that may still run', () => { expect(current.store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual([]) expect(turnState(current)).toBe('running') - expect(holds(current)).toEqual({ working: true, queueHeld: true }) + // No child here holds it, and it may still run: not working, its prompt not actionable. + expect(holds(current)).toEqual({ working: false, queueHeld: false }) }) it('leaves a live owner’s work unchanged at every trigger but its own exit', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts new file mode 100644 index 000000000000..1cedf8da3543 --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-observed-exit.test-fixture.ts @@ -0,0 +1,27 @@ +// An observed exit's settlement as the host's retry writes it (`settleStructuredAgentSessionLeftovers` +// with the exit's own account), for a chat whose lease that exit released. + +import type { AgentSessionJournal } from '../agent-session-journal/journal-store' +import { backgroundSettlementWrites } from './structured-agent-session-background-writes' +import type { StructuredAgentSessionDeadGenerationInput } from './structured-agent-session-dead-generation-settlement' +import { settleStructuredAgentSessionLeftovers } from './structured-agent-session-leftover-settlement' +import { scanRecord } from './structured-agent-session-startup-scan.test-fixture' + +/** `ownerFence`: the exited generation's; by default every fence, as a chat's lone generation. */ +export function settleObservedExitForTest( + input: StructuredAgentSessionDeadGenerationInput & { + journal: AgentSessionJournal + ownerFence?: number + } +) { + const { journal, sessionId, fence, ownerFence = 0, ...exit } = input + const released = scanRecord(sessionId, true) + const record = { ...released, lease: { ...released.lease, runtimeFence: fence } } + return settleStructuredAgentSessionLeftovers({ + store: { getRecord: () => record }, + sessionId, + journal, + writes: backgroundSettlementWrites(journal), + exit: { ...exit, ownerFence } + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index f206a40d1fa7..6c21fb6e4197 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -51,13 +51,15 @@ export function structuredQueueSendGate( ): QueueSendGate { return (journal) => { const record = host.store.getRecord(sessionId) - const ended = host.sessions.get(sessionId)?.lastEndedChild + const session = host.sessions.get(sessionId) + const ended = session?.lastEndedChild const abandoned = drain?.abandonedCard(sessionId) return { record, work: structuredAgentSessionCurrentWork(journal, { record, replaced: host.store.replacedRuntime(sessionId), + ...(session ? { child: session.child } : {}), ...(ended ? { ended } : {}) }), ...(drain ? { drainWaits: (messageId: string) => drain.waits(sessionId, messageId) } : {}), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts index 2ea74bd77499..b29e3dab59ad 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reasoning-sweep.test.ts @@ -12,10 +12,8 @@ import { codexProviderHandle } from '../../../shared/agent-session-provider-hand import { openAgentSessionJournal } from '../agent-session-journal/journal-store-factory' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { openTestJournalHostDatabase } from '../agent-session-journal/journal-host-database-test-support' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from './structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import { captureUnfinishedStructuredAgentSessionWork } from './structured-agent-session-unfinished-work' const SESSION = 'session-reasoning-sweep' @@ -72,7 +70,7 @@ describe('an open reasoning row a dead generation left', () => { it('ends, with no claimed time, when the generation is settled at its exit', async () => { await seedOpenReasoning('running') await expect( - settleStructuredAgentSessionDeadGeneration({ + settleObservedExitForTest({ journal, sessionId: SESSION, fence: 8, @@ -81,7 +79,7 @@ describe('an open reasoning row a dead generation left', () => { verdict: { state: 'interrupted', completedAt: 1_500 }, showUnexpectedExitOutcome: false }) - ).resolves.toEqual({ ok: true }) + ).resolves.toMatchObject({ ok: true }) expect(reasoningBody()).toEqual({ kind: 'message', role: 'reasoning', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts index 89aa02c718d7..da8d0e5986d0 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-backoff.ts @@ -4,8 +4,7 @@ // and nothing announces the lift. export const RECONCILIATION_BACKOFF_MS = { first: 1_000, max: 2_000 } as const -/** The backoffs a retry waits out with every round failing, summed, after which it gives up: a - * round fails at once, so this is the time it kept failing. */ +/** The time since an episode's first refused round after which its retry gives up. */ export const RECONCILIATION_GIVE_UP_MS = 180_000 /** The wait before the next try, after `failures` consecutive failed ones. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts index c07346e570ae..addf76959305 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.ts @@ -7,8 +7,9 @@ // One wake (`wake`): marking a chat owed or any commit runs a round now unless one runs or the timer // waits; a commit after a lock refused the last round ends that wait. The timer is the backoff // after a refused round (1 s to 2 s), or its first step after a busy lane. One budget: about 3 -// minutes of refused rounds give up (`RECONCILIATION_GIVE_UP_MS`), keeping the set and showing owed -// sends as not sent; the next wake after that is the only reset. +// minutes since the episode's first refused round give up (`RECONCILIATION_GIVE_UP_MS`), keeping +// the set, dropping its loaded reads and showing owed sends as not sent; only a round that made +// progress ends the episode, and only the next wake after a give-up resets it. import type { AgentSessionGenerationEnd } from '../../runtime/agent-session-generation-end' import { setImmediate as yieldToEvents } from 'node:timers/promises' @@ -56,10 +57,11 @@ export class StructuredAgentSessionRetry { private running = false /** Woken while a round ran: another follows at once. */ private again = false - /** Refused rounds in a row, and the backoffs they waited out; at the budget the episode ends. - * `locked`: the last one met another connection's lock, which a commit here proves gone. */ + /** Refused rounds in a row, and when the first was (`performance.now()`); at the budget the + * episode ends. `locked`: the last one met another connection's lock, which a commit here proves + * gone. */ private failures = 0 - private failingFor = 0 + private failingSince: number | null = null private locked = false private disposed = false /** The round's chain of chat turns (`inTurn`), and whether contention stopped the round. */ @@ -169,16 +171,17 @@ export class StructuredAgentSessionRetry { this.again = true return } + // No backoff pending once the budget is spent: the episode gave up, and this wake starts anew. + if (this.timer === null && this.failingFor() >= RECONCILIATION_GIVE_UP_MS) { + this.failures = 0 + this.failingSince = null + } if (early) { this.stopTimer() } if (this.timer !== null) { return } - if (this.failingFor >= RECONCILIATION_GIVE_UP_MS) { - this.failures = 0 - this.failingFor = 0 - } this.timer = 'soon' queueMicrotask(() => void this.round()) } @@ -194,7 +197,9 @@ export class StructuredAgentSessionRetry { let failed = false let locked = false let busy = false + let progress = false const end = (visit: Visit) => { + progress ||= visit === 'settled' || visit === 'again' failed ||= visit === 'failed' || visit === 'contended' locked ||= visit === 'contended' busy ||= visit === 'busy' @@ -205,12 +210,12 @@ export class StructuredAgentSessionRetry { // Store-wide: once a round, for every chat, never once per chat. No chat is visited while it // owes anything. const reconciled = await this.context.reconcile() - if (reconciled !== 'settled') { - end(reconciled) - this.stopped = true - } + end(reconciled) + this.stopped = reconciled !== 'settled' } const due = [...this.owed].filter(([, chat]) => chat.due) + // Nothing owed is progress too; a round only busy or parked is not. + progress ||= due.length === 0 await Promise.all( due.map(async ([sessionId, chat]) => { const visit = this.stopped ? null : await this.visitInSlot(sessionId, chat) @@ -220,21 +225,23 @@ export class StructuredAgentSessionRetry { }) ) this.running = false - this.roundEnded(failed, locked, busy) + this.roundEnded(failed, locked, busy, progress) // After the round's outcome is set, so a waiter reads its backoff, never a round still running. for (const [, chat] of due) { chat.attempted.splice(0).forEach((resolve) => resolve()) } } - private roundEnded(failed: boolean, locked: boolean, busy: boolean): void { + private roundEnded(failed: boolean, locked: boolean, busy: boolean, progress: boolean): void { if (this.disposed) { return } this.locked = locked if (!failed) { - this.failures = 0 - this.failingFor = 0 + if (progress) { + this.failures = 0 + this.failingSince = null + } if (this.again) { this.wake() } else if (busy) { @@ -244,10 +251,9 @@ export class StructuredAgentSessionRetry { return } this.failures += 1 - if (this.failingFor < RECONCILIATION_GIVE_UP_MS) { - const delay = reconciliationBackoffDelay(this.failures) - this.failingFor += delay - this.arm(delay) + this.failingSince ??= performance.now() + if (this.failingFor() < RECONCILIATION_GIVE_UP_MS) { + this.arm(reconciliationBackoffDelay(this.failures)) return } // The episode ends; the set stays for the next wake. @@ -256,12 +262,18 @@ export class StructuredAgentSessionRetry { error: new Error(`${this.failures} rounds failed`) }) for (const [sessionId, chat] of this.owed) { + // Owed still, but no hidden chat's replay is held through the wait for the next wake. + dropStructuredAgentSessionLoaded(chat) if (chat.send) { this.context.abandonSend(sessionId) } } } + private failingFor(): number { + return this.failingSince === null ? 0 : performance.now() - this.failingSince + } + private arm(delay: number): void { this.stopTimer() const timer = setTimeout(() => void this.round(), delay) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts index 6dfcb4f898e9..ba5d609b94e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-recovered-turn-clock.test.ts @@ -22,10 +22,8 @@ import { selectStructuredAgentSettledTurns } from '../../../shared/structured-ag import { AgentHookServer, _internals } from '../../agent-hooks/server' import { createTrackedJournalOpener } from '../agent-session-journal/journal-host-database-test-support' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' -import { - settleStaleStructuredAgentSessionState, - settleStructuredAgentSessionDeadGeneration -} from './structured-agent-session-dead-generation-settlement' +import { settleStaleStructuredAgentSessionState } from './structured-agent-session-dead-generation-settlement' +import { settleObservedExitForTest } from './structured-agent-session-observed-exit.test-fixture' import type { StructuredAgentSessionTurnVerdict } from './structured-agent-session-stale-turn-verdict' import { StructuredAgentSessionStatusFeed } from './structured-agent-session-status-feed' import { indexedStatusFeedSession } from './structured-agent-session-status-feed-test-session' @@ -128,8 +126,8 @@ async function sessionWithRunningTurn() { function settleDeadGeneration( journal: AgentSessionJournal, verdict: StructuredAgentSessionTurnVerdict -): ReturnType { - return settleStructuredAgentSessionDeadGeneration({ +): ReturnType { + return settleObservedExitForTest({ journal, sessionId: SESSION, fence: 2, @@ -159,7 +157,7 @@ describe('a turn recovery settled after its host went away', () => { async (_label, verdict, outcome, mark) => { const session = await sessionWithRunningTurn() session.recoverAt(RECOVERED) - expect(await settleDeadGeneration(session.journal, verdict)).toEqual({ ok: true }) + expect(await settleDeadGeneration(session.journal, verdict)).toMatchObject({ ok: true }) session.publish() expect(session.summaries.at(-1)).toMatchObject({ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts index b3f24d9ec103..56b8a0c0d6bc 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-replaced-server-turn.test.ts @@ -306,6 +306,16 @@ describe('a chat whose turn a replaced server held, before anything settles it', expect(rig.adapter.acquire).toHaveBeenCalledTimes(2) }) + it('reads unverifiable and not working while its owner recovers with no child here', async () => { + await survivingOwnerStartup(false) + + const { page } = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + + expect(page.working).toBe(false) + expect(page.latestTurn?.turn).toMatchObject({ turnId: TURN.turnId, state: 'unverifiable' }) + expect(rig.host.currentWork(SESSION)?.working()).toBe(false) + }) + it('settles nothing and stops nothing with no runtime replaced, while its recovery waits', async () => { const { stopOwnerProcess, lease } = await survivingOwnerStartup(false) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts index b627ec1d4add..6c766d4447fa 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-provider-output.test.ts @@ -354,13 +354,14 @@ describe('provider output survives host restart', () => { await rig.host.reconcileRestartLeases() await rig.host.startupSettled() await retryIdle(rig.host.collaboratorsForTests().reconciliation, SESSION) - const snapshot = await rig.host.journalSnapshot(SESSION) - const turn = snapshot.items.map((item) => readAgentJournalTurn(item.body)).find(Boolean) - // No runtime was replaced, so the owner's fate is unverifiable: startup settles nothing while - // its recovery is undecided, and the turn is never ended. + const read = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + // No runtime was replaced and no child here holds it: the turn reads unverifiable and not + // working, derived, while its recovery is undecided; nothing is written. expect(rig.store.getRecord(SESSION)?.lease.handoffStage).toBe('recovering') - expect(turn?.state).toBe('running') + const turn = read.page.latestTurn?.turn + expect(turn?.state).toBe('unverifiable') expect(turn?.completedAt).toBeUndefined() + expect(read.page.working).toBe(false) expect(stopOwnerProcess).not.toHaveBeenCalled() expect(rig.store.getRecord(SESSION)?.lease.deathEvidence).toBeFalsy() }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts index 9b6876b62974..58b84c9a20a1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-restart-stream-duration.test.ts @@ -179,11 +179,10 @@ describe('a provider stream cut short before its next lease renewal', () => { it('retains the output bound when a client opens the journal before owner reconciliation', async () => { await streamBeforeCrash() await restartAfterCrash() - // The open writes nothing: the turn reads as the crash left it until startup settles it. - const beforeProof = await rig.host.journalSnapshot(SESSION) - expect( - beforeProof.items.some((item) => readAgentJournalTurn(item.body)?.state === 'running') - ).toBe(true) + // The open writes nothing; with its owner not yet reconciled the turn reads unverifiable. + const beforeProof = await rig.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(beforeProof.page.latestTurn?.turn.state).toBe('unverifiable') + expect(beforeProof.page.working).toBe(false) await rig.host.reconcileRestartLeases() await rig.host.collaboratorsForTests().serialize(SESSION, async () => {}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts index 384425b6bac3..c7e47ef25abd 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-retry-wake.test.ts @@ -3,6 +3,7 @@ // newly owed chat or a commit starts a fresh episode. import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import * as reconciliationLoad from './structured-agent-session-reconciliation-load' import * as reconciliationPass from './structured-agent-session-reconciliation-pass' import { RECONCILIATION_GIVE_UP_MS } from './structured-agent-session-reconciliation-backoff' import { @@ -19,7 +20,7 @@ let lockHeld = true let pass: ReturnType beforeEach(() => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] }) lockHeld = true pass = vi .spyOn(reconciliationPass, 'runStructuredAgentSessionReconciliationPass') @@ -41,7 +42,10 @@ async function settleTurns(): Promise { } } -function openRetry(): StructuredAgentSessionRetry { +/** `busy`: whether a person's operation holds the chat's lane. `closed`: no reader has it open. */ +function openRetry( + options: { busy?: () => boolean; closed?: true } = {} +): StructuredAgentSessionRetry { const context = { deps: { store: { @@ -52,10 +56,10 @@ function openRetry(): StructuredAgentSessionRetry { }, logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() } }, - sessions: new Map([[CHAT, { journal }]]), + sessions: new Map(options.closed ? [] : [[CHAT, { journal }]]), now: () => 0, serialize: (_id: string, task: () => Promise) => task(), - laneBusy: () => false, + laneBusy: () => options.busy?.() ?? false, track: (operation: Promise) => operation, publishGenerationEnded: vi.fn(), reconcileOwed: () => false, @@ -68,14 +72,15 @@ function openRetry(): StructuredAgentSessionRetry { return new StructuredAgentSessionRetry(typed) } -/** Rounds refused by the lock until the episode gives up. */ +/** Rounds refused by the lock until the episode gives up: no timer is left. */ async function exhaust(retry: StructuredAgentSessionRetry): Promise { retry.signal(CHAT) await settleTurns() - while (retry['failingFor'] < RECONCILIATION_GIVE_UP_MS || retry['timer'] !== null) { + for (let elapsed = 0; elapsed < RECONCILIATION_GIVE_UP_MS + 4_000; elapsed += 2_000) { await vi.advanceTimersByTimeAsync(2_000) await settleTurns() } + expect(retry['timer']).toBeNull() } it('a commit after a lock refused a round runs the next one at once, with no timer', async () => { @@ -96,7 +101,7 @@ it('reads during a backoff add no round and leave the budget as it is', async () const retry = openRetry() retry.signal(CHAT) await settleTurns() - const waited = retry['failingFor'] + const waited = retry['failingFor']() for (let read = 0; read < 20; read += 1) { retry.opened(CHAT, journal) @@ -104,7 +109,7 @@ it('reads during a backoff add no round and leave the budget as it is', async () await settleTurns() expect(pass.mock.calls.length).toBeLessThanOrEqual(2) - expect(retry['failingFor']).toBe(waited) + expect(retry['failingFor']()).toBe(waited) retry.dispose() }) @@ -114,7 +119,7 @@ it('before a give-up, a signal, a commit or a read leaves the budget counting', await settleTurns() await vi.advanceTimersByTimeAsync(3_000) await settleTurns() - const waited = retry['failingFor'] + const waited = retry['failingFor']() expect(waited).toBeGreaterThan(0) retry.signal(CHAT) @@ -122,7 +127,7 @@ it('before a give-up, a signal, a commit or a read leaves the budget counting', retry.opened(CHAT, journal) await settleTurns() - expect(retry['failingFor']).toBeGreaterThanOrEqual(waited) + expect(retry['failingFor']()).toBeGreaterThanOrEqual(waited) retry.dispose() }) @@ -138,7 +143,7 @@ it('after a give-up only a newly owed chat or a commit starts a fresh episode', retry.signal(CHAT) await settleTurns() expect(pass).toHaveBeenCalledTimes(given + 1) - expect(retry['failingFor']).toBeLessThan(RECONCILIATION_GIVE_UP_MS) + expect(retry['failingFor']()).toBeLessThan(RECONCILIATION_GIVE_UP_MS) await exhaust(retry) const again = pass.mock.calls.length @@ -149,3 +154,66 @@ it('after a give-up only a newly owed chat or a commit starts a fresh episode', expect(retryOwes(retry, CHAT)).toBe(false) retry.dispose() }) + +it('keeps the budget through a round whose only chat waited on a busy lane', async () => { + let busy = false + const retry = openRetry({ busy: () => busy }) + retry.signal(CHAT) + await settleTurns() + await vi.advanceTimersByTimeAsync(500) + expect(retry['failures']).toBe(1) + + // The lock lifts, but a person's operation holds the lane: no progress, so no reset. + lockHeld = false + busy = true + retry.observeCommit('another-chat', null) + await settleTurns() + + expect(pass).toHaveBeenCalledTimes(1) + expect(retry['failures']).toBe(1) + retry.dispose() +}) + +it('drops a closed chat it loaded when its episode gives up, and keeps it owed', async () => { + const close = vi.fn(async () => undefined) + const loads = vi + .spyOn(reconciliationLoad, 'loadStructuredAgentSessionForReconciliation') + .mockResolvedValue({ + kind: 'loaded', + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the visit reads only the journal's open cursor and close of a loaded read. + session: { journal: { ...journal, close } } as never + }) + vi.spyOn(reconciliationLoad, 'structuredAgentSessionJournalIsCurrent').mockReturnValue(true) + const retry = openRetry({ closed: true }) + + await exhaust(retry) + + expect(loads).toHaveBeenCalledTimes(1) + expect(close).toHaveBeenCalledOnce() + expect(retry['owed'].get(CHAT)?.loaded).toBeUndefined() + expect(retryOwes(retry, CHAT)).toBe(true) + retry.dispose() +}) + +it('gives up after about 3 minutes of wall time, however often commits wake it early', async () => { + const retry = openRetry() + const warn = vi.mocked(retry['context'].deps.logger.warn) + const gaveUp = () => warn.mock.calls.filter(([message]) => message.startsWith('gave up')).length + retry.signal(CHAT) + await settleTurns() + // A commit every 100 ms: each ends the backoff and runs a refused round at once. + for (let elapsed = 0; elapsed < 170_000; elapsed += 100) { + retry.observeCommit('another-chat', null) + await settleTurns() + await vi.advanceTimersByTimeAsync(100) + } + expect(gaveUp()).toBe(0) + + for (let elapsed = 0; elapsed < 12_000; elapsed += 100) { + retry.observeCommit('another-chat', null) + await settleTurns() + await vi.advanceTimersByTimeAsync(100) + } + expect(gaveUp()).toBe(1) + retry.dispose() +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index d0858417bbbd..6cf0f293a3ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -59,6 +59,7 @@ export class StructuredAgentSessionJournalProjections { seen?: { lastEndedChild?: StructuredAgentSessionWorkEvidence['ended'] operationalRevision?: number + child?: StructuredAgentSessionWorkEvidence['child'] } ): StructuredAgentSessionJournalProjection { const cursor = journal.cursor() @@ -68,6 +69,7 @@ export class StructuredAgentSessionJournalProjections { const work = structuredAgentSessionCurrentWork(journal, { record, replaced: record ? this.replacedRuntime?.(record.sessionId) : undefined, + ...(seen ? { child: seen.child ?? null } : {}), ...(ended ? { ended } : {}), revision: seen?.operationalRevision ?? 0 }) From dda2bfcb0264e1973de86d7b9606583fec270bac Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 05:27:26 -0700 Subject: [PATCH 18/22] fix(native-chat): a queue waits on an owner no adjudication concluded about What a chat shows and whether a card may be delivered are separate answers. The display reads the projection's live generation; delivery reads the owner's adjudication, so an unreconciled or recovering owner holds the queue (and reads busy to orchestration) as on main, while its turn reads unverifiable and not working. The view key carries that state so subscribers republish when recovery concludes. --- .../scripts/vitest-sqlite-runtime-files.mjs | 1 + ...uctured-agent-session-current-work.test.ts | 26 ++++++ .../structured-agent-session-current-work.ts | 6 +- ...-session-leftover-settlement-scope.test.ts | 7 +- ...tructured-agent-session-queued-messages.ts | 8 +- ...ructured-agent-session-queued-mutations.ts | 8 +- .../structured-agent-session-reveal.ts | 6 +- ...-session-unadjudicated-owner-queue.test.ts | 79 +++++++++++++++++++ .../structured-mailbox-pointer-host.test.ts | 19 ++++- .../structured-mailbox-pointer-host.ts | 9 ++- .../agent-session-lease-adjudication.ts | 10 +++ 11 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts diff --git a/config/scripts/vitest-sqlite-runtime-files.mjs b/config/scripts/vitest-sqlite-runtime-files.mjs index 7d992f31d112..55c542a2f4c8 100644 --- a/config/scripts/vitest-sqlite-runtime-files.mjs +++ b/config/scripts/vitest-sqlite-runtime-files.mjs @@ -116,6 +116,7 @@ export const SQLITE_RUNTIME_INCLUDE = [ 'src/main/native-chat/agent-session-wire/structured-agent-session-late-settlement.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-lease-renewer.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts', + 'src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-triggers.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-reconciliation-retry.test.ts', 'src/main/native-chat/agent-session-wire/structured-agent-session-startup-scan.test.ts', diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts index 6f2f3c25b472..03dd59da8010 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.test.ts @@ -37,6 +37,8 @@ import { turnState } from './structured-agent-session-leftover-settlement.test-fixture' import { retryIdle } from './structured-agent-session-retry.test-fixture' +import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { scanRecord } from './structured-agent-session-startup-scan.test-fixture' let rig: QueuedMessageTestRig | undefined @@ -301,3 +303,27 @@ describe('a live owner', () => { expect(view.listed()?.status).not.toBe('idle') }) }) + +describe('an owner whose adjudication concludes with no row written', () => { + it('changes the published answer when it goes from unverifiable to released', () => { + const journal = { + runningTurn: () => null, + itemFence: () => undefined, + visitItems: () => undefined, + submissions: () => [], + wroteBeforeOpen: () => false + } + const record = scanRecord(SESSION, false) + const unverifiable = structuredAgentSessionCurrentWork(journal, { + record: { lease: { ...record.lease, unreconciled: true } }, + child: null + }) + const released = structuredAgentSessionCurrentWork(journal, { + record: scanRecord(SESSION, true), + child: null + }) + + expect(unverifiable.liveFence).toBe(released.liveFence) + expect(unverifiable.viewKey()).not.toBe(released.viewKey()) + }) +}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index 37243f698354..7bf0728e8d4a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -21,6 +21,7 @@ import { type AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' +import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionLatestTurn } from '../../../shared/agent-session-wire' import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' @@ -77,7 +78,8 @@ export function structuredAgentSessionLiveFence( function structuredAgentSessionOwnerUnverifiable(evidence: StructuredAgentSessionWorkEvidence) { const lease = evidence.record?.lease return ( - (lease?.unreconciled === true || lease?.handoffStage === 'recovering') && + lease !== undefined && + agentSessionOwnerUnadjudicated(evidence.record) && evidence.child !== undefined && evidence.child?.fence !== lease.runtimeFence ) @@ -135,7 +137,7 @@ export class StructuredAgentSessionCurrentWork { * generation, since every answer here is the journal read through it. Not the revision, which * moves on every generation-end signal: an end that changed nothing is published once. */ viewKey(): string { - return `${this.liveFence}` + return `${this.liveFence}${this.ownerUnverifiable ? ':unverifiable' : ''}` } /** Whether the live generation's execution produced this item. */ diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts index 69ba41d34b97..d3d35635b6ed 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -126,7 +126,7 @@ describe('a release nothing proved', () => { }) describe('an owner that may still run', () => { - it('reads an unreconciled owner’s work as unverifiable, holding nothing, and settles nothing', async () => { + it('reads an unreconciled owner’s work as unverifiable, holds its queue, and settles nothing', async () => { rig = await createQueuedMessageTestRig({ restartable: true, probeOwner: async () => { @@ -143,8 +143,9 @@ describe('an owner that may still run', () => { expect(current.store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual([]) expect(turnState(current)).toBe('running') - // No child here holds it, and it may still run: not working, its prompt not actionable. - expect(holds(current)).toEqual({ working: false, queueHeld: false }) + // No child here holds it, and it may still run: shown not working, its prompt not actionable, + // while its queue waits for an adjudication, as on main. + expect(holds(current)).toEqual({ working: false, queueHeld: true }) }) it('leaves a live owner’s work unchanged at every trigger but its own exit', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index c1ab3ace8311..05b37d982173 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -18,6 +18,7 @@ import { queuedSendAnswer } from './structured-agent-session-queued-send-answer' import { structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { queuedMessagesPublishedBytesRefusal } from './structured-agent-session-queued-published-bytes' import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { contextStructuredAgentSessionCurrentWork, @@ -72,7 +73,8 @@ function oldestActionableQueuedMessage( * drain step: any hold returns early; whatever clears it publishes or * commits, which re-derives. * Send-now: overrides only `working` (plus FIFO order and the stored hold), - * never for a command card; `blocked` and `prompt` refuse readably. + * never for a command card while a turn runs; `blocked` and `prompt` refuse + * readably. * * `blocked` is whatever refuses any send (an uncertain rewind, a cleared source); * the rest are waits. A /compact is a queued message and then a turn, @@ -103,7 +105,9 @@ export function structuredQueueHold(input: StructuredQueueGateInput): Structured if (input.work.hasActionablePrompt()) { return 'prompt' } - return input.work.working() ? 'working' : null + // Delivery reads the owner's adjudication, not what is shown: an owner none has concluded about + // may still run, so its queue waits as a live one's does, though nothing it left reads working. + return agentSessionOwnerUnadjudicated(input.record) || input.work.working() ? 'working' : null } /** The card the drain sends next, or null while anything holds the queue: the drain's own pick diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts index 8a0709c7991f..ec3f1f028a03 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-mutations.ts @@ -130,10 +130,8 @@ export function sendQueuedStructuredAgentMessage( // The one queue gate; Send-now's override set is exactly `working` (plus // FIFO order and the stored hold, which the consume below clears). const record = context.deps.store.getRecord(ctx.sessionId) - const hold = structuredQueueHold({ - record, - work: contextStructuredAgentSessionCurrentWork(ctx) - }) + const work = contextStructuredAgentSessionCurrentWork(ctx) + const hold = structuredQueueHold({ record, work }) if (hold === 'blocked') { return structuredAgentSessionSendBlock(record) ?? invalid('This conversation cannot send.') } @@ -156,7 +154,7 @@ export function sendQueuedStructuredAgentMessage( } // A command never steers: handed over mid-turn it would only be refused. Clients offer its // Send only while the agent is idle; this answers an older one that offers it mid-turn. - if (hold === 'working' && row.body.command) { + if (row.body.command && work.working()) { return invalid("A command can't be sent while the agent is working.") } const submissionId = operationId diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index b3e4c9bbfff7..b750bb88a605 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -10,6 +10,7 @@ // a send does. And a journal it cannot open is not a refusal — the chat shows that failure with a // Retry, so the tab is worth publishing either way. +import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { journalOpenRefusal } from '../agent-session-journal/journal-open-failure' @@ -112,8 +113,7 @@ export function createStructuredAgentSessionHostRestore( let restoring = false const latchedTargets = () => { for (const sessionId of restoreTargets) { - const lease = deps.store.getRecord(sessionId)?.lease - if (!lease?.unreconciled && lease?.handoffStage !== 'recovering') { + if (!agentSessionOwnerUnadjudicated(deps.store.getRecord(sessionId))) { restoreTargets.delete(sessionId) } } @@ -192,7 +192,7 @@ export function createStructuredAgentSessionHostRestore( // Once, as the restorer restores once. for (const { sessionId, lease } of restoring ? [] : deps.store.listRecords()) { if ( - (lease.unreconciled || lease.handoffStage === 'recovering') && + agentSessionOwnerUnadjudicated({ lease }) && (!sessionIds || sessionIds.includes(sessionId)) ) { restoreTargets.add(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts new file mode 100644 index 000000000000..85c05a1e7cdb --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts @@ -0,0 +1,79 @@ +// A queued card over an owner no adjudication has concluded about (unreconciled after a reload, or +// latched in recovery), on a chat no tab shows: it may still run, so the card waits, as on main. No +// automatic send starts an agent in its place, stops it, or hands the card back; once a probe +// proves it gone, the card drains. + +import { afterEach, expect, it, vi } from 'vitest' +import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + createQueuedMessageTestRig, + type QueuedMessageTestRig +} from './structured-agent-session-queued-message-rig.test-fixture' +import { + leaveUnfinishedWork, + sendText +} from './structured-agent-session-leftover-settlement.test-fixture' + +let rig: QueuedMessageTestRig | undefined + +afterEach(async () => { + await rig?.dispose() + rig = undefined + vi.restoreAllMocks() +}) + +it.each(['unreconciled', 'recovering'] as const)( + 'holds the queued card while the owner is %s, and sends it once the owner is proven gone', + async (mode) => { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + let ownerGone = false + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => { + if (ownerGone) { + return { outcome: 'pid-absent' } + } + if (mode === 'unreconciled') { + throw new Error('the probe could not run') + } + return { outcome: 'identity-matched', matchedOn: ['process-start-time'] } + } + }) + const current = rig + await current.workingSend() + await leaveUnfinishedWork(current) + // No visible-tab restore: nothing at startup decides the owner. + await current.crashReloadHostProcess(() => undefined) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + const stopOwnerProcess = vi.fn() + current.host.deps.stopOwnerProcess = stopOwnerProcess + const lease = current.store.getRecord(SESSION)?.lease + expect(mode === 'unreconciled' ? lease?.unreconciled : lease?.handoffStage).toBe( + mode === 'unreconciled' ? true : 'recovering' + ) + const starts = current.starts.length + const dispatches = current.dispatch.mock.calls.length + + const sent = await sendText(current, 'while it may still run').result + await new Promise((resolve) => setTimeout(resolve, 3_000)) + + expect(sent).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + expect(current.starts.length).toBe(starts) + expect(current.dispatch.mock.calls.length).toBe(dispatches) + expect(stopOwnerProcess).not.toHaveBeenCalled() + const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + expect(page.ok && page.page.queuedMessages?.map((card) => card.state)).toEqual(['waiting']) + expect(page.ok && page.page.queuedMessages?.[0]?.returnedReason).toBeUndefined() + // Shown as main shows it: unverifiable, not working. + expect(page.ok && page.page.working).toBe(false) + + ownerGone = true + await current.host.restoreReadableSessions([SESSION]) + await vi.waitFor(() => expect(current.dispatch.mock.calls.length).toBeGreaterThan(dispatches), { + timeout: 10_000 + }) + expect(JSON.stringify(current.dispatch.mock.calls.at(-1))).toContain('while it may still run') + expect(stopOwnerProcess).not.toHaveBeenCalled() + }, + 30_000 +) diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index 05683fcc505c..c686a82a1a67 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -75,7 +75,24 @@ describe('structured mailbox pointer host', () => { const items = [runningTurn(), ...transcript(500)] hostRef.current = { journalSnapshot: () => ({ items, submissions: [] }), - currentWork: () => null + currentWork: () => null, + deps: { store: { getRecord: () => null } } + } + expect(await readStructuredSessionGateFacts('s1')).toEqual({ + turnRunning: true, + awaitingHuman: false + }) + }) + + it.each([ + { unreconciled: true, handoffStage: null }, + { unreconciled: false, handoffStage: 'recovering' } + ])('never reads an owner no adjudication has concluded about as idle (%o)', async (lease) => { + // Nothing it left reads as current work, yet it may still run: `@idle` must not pick it. + hostRef.current = { + journalSnapshot: () => ({ items: [runningTurn()], submissions: [] }), + currentWork: () => ({ scope: { isCurrentItem: () => false, owesSend: () => false } }), + deps: { store: { getRecord: () => ({ lease }) } } } expect(await readStructuredSessionGateFacts('s1')).toEqual({ turnRunning: true, diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index 92b1e0594823..4efd0e5baaf8 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -18,6 +18,7 @@ import { type StructuredSessionGateFacts } from './structured-session-pointer-delivery' import { sendAgentTurn } from './send-agent-turn' +import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' /** Names the dispatch a nudge belongs to on the operation row it writes. */ export function structuredPointerCallerKey(dispatchId: string): string { @@ -51,8 +52,12 @@ export async function readStructuredSessionGateFacts( return null } // The host's projection of current work: an agent that ended is idle, whatever its rows say. - const work = getStructuredAgentSessionHost()?.currentWork(sessionId) - return structuredSessionGateFacts(snapshot.items, work?.scope) + const host = getStructuredAgentSessionHost() + const facts = structuredSessionGateFacts(snapshot.items, host?.currentWork(sessionId)?.scope) + // An owner no adjudication has concluded about may still run: never idle, as its queue holds. + return agentSessionOwnerUnadjudicated(host?.deps.store.getRecord(sessionId)) + ? { ...facts, turnRunning: true } + : facts } /** What each recorded send settled as. */ diff --git a/src/shared/agent-session-lease-adjudication.ts b/src/shared/agent-session-lease-adjudication.ts index 3365159b7a0b..0f924b11ae60 100644 --- a/src/shared/agent-session-lease-adjudication.ts +++ b/src/shared/agent-session-lease-adjudication.ts @@ -122,6 +122,16 @@ export function agentSessionLeaseOwnerVerdict(lease: AgentSessionLease): AgentSe : 'unverifiable' } +/** No adjudication has concluded about the recorded owner yet: unreconciled since the restart, or + * latched in recovery. It may still run, so nothing starts in its place (`evaluateAgentSession- + * Acquisition` refuses both) and nothing is delivered to the chat until one concludes. */ +export function agentSessionOwnerUnadjudicated( + record: { lease: Pick } | null | undefined +): boolean { + const lease = record?.lease + return lease !== undefined && (lease.unreconciled || lease.handoffStage === 'recovering') +} + /** Nothing holds this lease: released, no handoff in flight, and reconciled since the last restart. */ export function agentSessionLeaseIsReleased(lease: AgentSessionLease): boolean { return !lease.unreconciled && lease.claimStatus === 'released' && lease.handoffStage === null From cde3a53d4be0f9aaad635d5c4a7d61241c36ce9c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:06:41 -0700 Subject: [PATCH 19/22] fix(native-chat): every host reader reads one current work, with this host's child Admission, Send-now, steer, Stop, prompt answers and command admission built current work without this host's child, so an owner no adjudication concluded about read working there and idle on the display and in the drain. Every host reader now builds it in one place. The queue hold and the orchestration idle gate read only that answer, as on main; whether a send may acquire stays with start, which reconciles and resolves recovery first. --- .../structured-agent-session-attach-flow.ts | 4 +- ...tructured-agent-session-client-delivery.ts | 11 +- ...red-agent-session-conversation-lifetime.ts | 5 +- .../structured-agent-session-current-work.ts | 6 +- .../structured-agent-session-delivery-loop.ts | 16 +- ...uctured-agent-session-host-current-work.ts | 31 ++- .../structured-agent-session-host-delivery.ts | 2 + ...tructured-agent-session-host-stop-event.ts | 28 +-- ...tructured-agent-session-idle-sweep.test.ts | 2 +- .../structured-agent-session-idle-sweep.ts | 20 +- ...-session-leftover-settlement-scope.test.ts | 7 +- ...ctured-agent-session-mutation-admission.ts | 6 +- ...ructured-agent-session-mutation-context.ts | 7 +- ...red-agent-session-mutation-turn-context.ts | 10 +- ...tructured-agent-session-queued-messages.ts | 5 +- ...ctured-agent-session-queued-publication.ts | 20 +- .../structured-agent-session-reveal.ts | 6 +- .../structured-agent-session-rewind.ts | 4 +- ...agent-session-status-journal-projection.ts | 2 +- .../structured-agent-session-turns.ts | 2 +- ...-session-unadjudicated-owner-queue.test.ts | 217 ++++++++++++++---- ...ructured-conversation-command-admission.ts | 2 +- .../structured-conversation-command.ts | 4 +- .../structured-mailbox-pointer-host.test.ts | 19 +- .../structured-mailbox-pointer-host.ts | 9 +- .../agent-session-lease-adjudication.ts | 10 - 26 files changed, 276 insertions(+), 179 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts index 4e2f7a9a07f4..a437a1db496a 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-attach-flow.ts @@ -36,7 +36,7 @@ import type { StructuredAgentRegistry } from './structured-agent-registry' import type { StructuredAgentSessionEventSink } from './structured-agent-session-event-sink' import { resolveAgentSessionReplayOutcome } from './structured-agent-session-replay-outcome' import { readAgentSessionHydrationPage } from './agent-session-history-page' -import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { heldStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { acquireOwner } from './structured-agent-session-acquisition' import { importAdoptedTranscript, @@ -289,7 +289,7 @@ export async function performAttach( page: readAgentSessionHydrationPage( attached.journal, fence, - structuredAgentSessionCurrentWork(attached.journal, { record }) + heldStructuredAgentSessionCurrentWork(store, sessionId, attached.journal, undefined) ), unconfirmedClientMessageIds: attached.unconfirmedClientMessageIds, ...(tabId ? { tabId } : {}) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts index c1a07719ebf9..1d99f1379f71 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-client-delivery.ts @@ -16,7 +16,7 @@ import type { } from './structured-agent-session-host-types' import { AGENT_SESSION_NOT_ATTACHED } from './structured-agent-session-mutation-admission' import { StructuredAgentSessionSendSettlement } from './structured-agent-session-send-settlement' -import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { heldStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { structuredAgentSessionConversationFence } from './structured-agent-session-provider-child' import type { StructuredAgentSessionStatusSubscriber } from './structured-agent-session-status-feed' import { createStructuredAgentSessionHostStatusFeed } from './structured-agent-session-host-status-feed' @@ -199,14 +199,7 @@ export class StructuredAgentSessionClientDelivery { /** The current-work projection over this journal (`structuredAgentSessionCurrentWork`). */ private currentWork(sessionId: string, journal: AgentSessionJournal) { const session = this.sessions.get(sessionId) - const ended = session?.lastEndedChild - return structuredAgentSessionCurrentWork(journal, { - record: this.deps().store.getRecord(sessionId), - replaced: this.deps().store.replacedRuntime(sessionId), - ...(session ? { child: session.child } : {}), - ...(ended ? { ended } : {}), - revision: session?.operationalRevision ?? 0 - }) + return heldStructuredAgentSessionCurrentWork(this.deps().store, sessionId, journal, session) } private settlementReading(sessionId: string, journal: AgentSessionJournal) { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts index 4211929805cc..bba4aeec2ca2 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-conversation-lifetime.ts @@ -84,7 +84,10 @@ export function createStructuredAgentSessionConversationLifetime(host: { isDisposed: () => disposed, deliveryActive: host.deliveryActive, childWork: host.readChildWork, - getRecord: (sessionId) => deps().store.getRecord(sessionId), + store: { + getRecord: (sessionId) => deps().store.getRecord(sessionId), + replacedRuntime: (sessionId) => deps().store.replacedRuntime(sessionId) + }, hasOpenDispatch: (sessionId) => { const record = deps().store.getRecord(sessionId) return record !== null && deps().hasOpenDispatch?.(record) === true diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts index 7bf0728e8d4a..2cee82ca63f9 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-current-work.ts @@ -21,7 +21,6 @@ import { type AgentJournalTurnScope } from '../../../shared/agent-session-journal-types' import { isQueuedAgentJournalSubmission } from '../../../shared/agent-session-queued-submission' -import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionLatestTurn } from '../../../shared/agent-session-wire' import type { StructuredAgentSessionWorkScope } from '../../../shared/structured-agent-session-main-agent-working' @@ -78,8 +77,7 @@ export function structuredAgentSessionLiveFence( function structuredAgentSessionOwnerUnverifiable(evidence: StructuredAgentSessionWorkEvidence) { const lease = evidence.record?.lease return ( - lease !== undefined && - agentSessionOwnerUnadjudicated(evidence.record) && + (lease?.unreconciled === true || lease?.handoffStage === 'recovering') && evidence.child !== undefined && evidence.child?.fence !== lease.runtimeFence ) @@ -264,7 +262,7 @@ export function structuredAgentSessionCurrentWork( export function contextStructuredAgentSessionCurrentWork(ctx: { journal: StructuredAgentSessionCurrentWorkJournal fence: number - currentWork?: () => StructuredAgentSessionCurrentWork + currentWork?: () => StructuredAgentSessionCurrentWork | null }): StructuredAgentSessionCurrentWork { return ctx.currentWork?.() ?? new StructuredAgentSessionCurrentWork(ctx.journal, ctx.fence) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts index 85cd97d0d6eb..25307a2e5d31 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-delivery-loop.ts @@ -39,7 +39,7 @@ import { handOverSubmission } from './structured-agent-session-turns' import { structuredAgentSessionNextHandover } from './structured-agent-session-opening-send' import { structuredAgentSessionCommandRunning } from './structured-agent-session-command-turn' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { structuredAgentSessionEndedChildFailure } from './structured-agent-session-ended-child-failure' export type StructuredAgentSessionDeliveryLoopDeps = { @@ -66,6 +66,8 @@ export type StructuredAgentSessionDeliveryLoopDeps = { failureTextContext: (sessionId: string) => AgentSessionFailureWordsContext logger: StructuredAgentSessionLogger record: (sessionId: string) => AgentSessionRecord | null + /** The host's projection of the chat's current work (`hostStructuredAgentSessionCurrentWork`). */ + currentWork: (sessionId: string) => StructuredAgentSessionCurrentWork | null readChildWork: (sessionId: string) => readonly AgentChildWorkView[] | undefined /** A person's Stop is still ending the session's work: the status feed's own reading. */ stopping: (sessionId: string) => boolean @@ -182,16 +184,8 @@ export class StructuredAgentSessionDeliveryLoop { const oldest = oldestQueuedSubmission(session) // A running command takes no input while its child carries it; its end is a commit, which // wakes the loop again. With no child it is a gone generation's, which the start below settles. - if ( - !oldest || - (session.child && - structuredAgentSessionCommandRunning( - structuredAgentSessionCurrentWork(session.journal, { - record: this.deps.record(sessionId), - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) - }) - )) - ) { + const work = this.deps.currentWork(sessionId) + if (!oldest || (session.child && work && structuredAgentSessionCommandRunning(work))) { return this.stop(sessionId) } const failedStart = startThatFailedWhileQueued(session, oldest) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts index 5e152573cae7..5c5ddaccea16 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-current-work.ts @@ -5,7 +5,8 @@ import type { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' import { structuredAgentSessionCurrentWork, - type StructuredAgentSessionCurrentWork + type StructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkJournal } from './structured-agent-session-current-work' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' @@ -31,12 +32,26 @@ export function hostStructuredAgentSessionCurrentWork( ): StructuredAgentSessionCurrentWork | null { const session = host.sessions.get(sessionId) return session - ? structuredAgentSessionCurrentWork(session.journal, { - record: host.store.getRecord(sessionId), - replaced: host.store.replacedRuntime(sessionId), - child: session.child, - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}), - revision: session.operationalRevision ?? 0 - }) + ? heldStructuredAgentSessionCurrentWork(host.store, sessionId, session.journal, session) : null } + +/** The one current work every host reader decides and displays by: this host's child is always + * evidence (none held reads as none), so an action never reads work the display does not show. */ +export function heldStructuredAgentSessionCurrentWork( + store: StructuredAgentSessionCurrentWorkHost['store'], + sessionId: string, + journal: StructuredAgentSessionCurrentWorkJournal, + session: + | Pick + | undefined +): StructuredAgentSessionCurrentWork { + const ended = session?.lastEndedChild + return structuredAgentSessionCurrentWork(journal, { + record: store.getRecord(sessionId), + replaced: store.replacedRuntime(sessionId), + child: session?.child ?? null, + ...(ended ? { ended } : {}), + revision: session?.operationalRevision ?? 0 + }) +} diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts index 5eac9309bcd3..524a370b4a5b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-delivery.ts @@ -105,6 +105,8 @@ export function createStructuredAgentSessionConversationDelivery(input: { ), logger: deps.logger, record: (sessionId) => deps.store.getRecord(sessionId), + currentWork: (sessionId) => + hostStructuredAgentSessionCurrentWork({ store: deps.store, sessions }, sessionId), readChildWork: input.clientDelivery.readChildWork, stopping: input.clientDelivery.readStopping, now: () => deps.now?.() ?? Date.now() diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts index 35198cf64fef..308aa76035e6 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-host-stop-event.ts @@ -1,10 +1,8 @@ // The Stop event a host stop writes (`JournalStopEvent`): whether it ends work its event must // record, and the write itself, issued before the kill. -import { - structuredAgentSessionCurrentWork, - type StructuredAgentSessionCurrentWork -} from './structured-agent-session-current-work' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { heldStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { withTimeout } from '../../../shared/promise-timeout-fallback' import type { StructuredAgentSessionStopCause } from './structured-agent-session-adapter' import type { StructuredAgentSessionLifetimeContext } from './structured-agent-session-host-lifetime' @@ -66,11 +64,12 @@ export async function stopEndsWork( 'slow' as const ) // The child is live: the work its stop ends is its generation's. - const work = structuredAgentSessionCurrentWork(journal, { - record: context.deps.store.getRecord(sessionId), - replaced: context.deps.store.replacedRuntime(sessionId), - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) - }) + const work = heldStructuredAgentSessionCurrentWork( + context.deps.store, + sessionId, + journal, + session + ) const working = (drain === 'slow' && journal.stopMarks.latestAcceptedSendUnopened()) || work.working() return working && (ending.cause === 'user-close' || !defersToPersonsStop(session, work)) @@ -109,11 +108,12 @@ export function recordStopEvent( return Promise.resolve(null) } const fence = structuredAgentSessionConversationFence(context.deps.store, sessionId) - const turnId = structuredAgentSessionCurrentWork(session.journal, { - record: context.deps.store.getRecord(sessionId), - replaced: context.deps.store.replacedRuntime(sessionId), - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) - }).activeTurnId() + const turnId = heldStructuredAgentSessionCurrentWork( + context.deps.store, + sessionId, + session.journal, + session + ).activeTurnId() return session.journal .appendStopEvent({ reason: ending.cause, ...(turnId !== null ? { turnId } : {}) }, fence) .then( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts index 93d3a3c4fc8a..f3adb18b2a2d 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.test.ts @@ -320,7 +320,7 @@ describe('the idle sweep with no child running (P2-22 ii)', () => { now: () => IDLE_MS + 1, isDisposed: () => false, deliveryActive: () => false, - getRecord: () => null, + store: { getRecord: () => null, replacedRuntime: () => undefined }, childWork: () => undefined, hasOpenDispatch: () => false, providerHoldsDispatch: () => false, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts index 2e9616ad5a92..2821be3b2f7b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-idle-sweep.ts @@ -12,12 +12,14 @@ import { agentChildWorkLiveness } from '../../../shared/agent-status-child-work- import type { AgentChildWorkView } from '../../../shared/agent-status-child-work-view' import type { StructuredAgentSessionHostSession } from './structured-agent-session-host-types' import type { StructuredAgentSessionLogger } from './structured-agent-session-logger' -import type { AgentSessionRecord } from '../../../shared/agent-session-record' import { - structuredAgentSessionCurrentWork, structuredAgentSessionHandsOver, type StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' +import { + heldStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkHost +} from './structured-agent-session-host-current-work' export const STRUCTURED_AGENT_SESSION_IDLE_SWEEP_INTERVAL_MS = 5 * 60_000 export const STRUCTURED_AGENT_SESSION_IDLE_MS = 30 * 60_000 @@ -37,8 +39,8 @@ export type StructuredAgentSessionIdleSweepDeps = { hasOpenDispatch: (sessionId: string) => boolean /** The provider holds a send it took and has not answered; see `holdsDispatch`. */ providerHoldsDispatch: (sessionId: string) => boolean - /** The lease the current-work projection reads (`structuredAgentSessionCurrentWork`). */ - getRecord: (sessionId: string) => AgentSessionRecord | null + /** The lease store the current-work projection reads (`heldStructuredAgentSessionCurrentWork`). */ + store: StructuredAgentSessionCurrentWorkHost['store'] /** Each of these runs inside the session's serialize and never takes it again. */ stopAgent: (sessionId: string) => Promise stopStartingAgent: (sessionId: string) => Promise @@ -170,10 +172,12 @@ export class StructuredAgentSessionIdleSweep { this.queuedOrDelivering(sessionId, session) || structuredAgentSessionChildHasOpenWork({ currentWork: () => - structuredAgentSessionCurrentWork(session.journal, { - record: this.deps.getRecord(sessionId), - ...(session.lastEndedChild ? { ended: session.lastEndedChild } : {}) - }), + heldStructuredAgentSessionCurrentWork( + this.deps.store, + sessionId, + session.journal, + session + ), childWork: () => this.deps.childWork(sessionId), hasOpenDispatch: () => this.deps.hasOpenDispatch(sessionId), providerHoldsDispatch: () => this.deps.providerHoldsDispatch(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts index d3d35635b6ed..69ba41d34b97 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-leftover-settlement-scope.test.ts @@ -126,7 +126,7 @@ describe('a release nothing proved', () => { }) describe('an owner that may still run', () => { - it('reads an unreconciled owner’s work as unverifiable, holds its queue, and settles nothing', async () => { + it('reads an unreconciled owner’s work as unverifiable, holding nothing, and settles nothing', async () => { rig = await createQueuedMessageTestRig({ restartable: true, probeOwner: async () => { @@ -143,9 +143,8 @@ describe('an owner that may still run', () => { expect(current.store.getRecord(SESSION)?.lease.claimStatus).not.toBe('released') expect(recoveredRows(openTestJournalHostDatabase(current.root).db)).toEqual([]) expect(turnState(current)).toBe('running') - // No child here holds it, and it may still run: shown not working, its prompt not actionable, - // while its queue waits for an adjudication, as on main. - expect(holds(current)).toEqual({ working: false, queueHeld: true }) + // No child here holds it, and it may still run: not working, its prompt not actionable. + expect(holds(current)).toEqual({ working: false, queueHeld: false }) }) it('leaves a live owner’s work unchanged at every trigger but its own exit', async () => { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts index aa56a68c2b9d..7e8e8aa35182 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-admission.ts @@ -40,7 +40,7 @@ import { import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' import type { MutationPlan } from './structured-agent-session-mutation-plans' -import type { StructuredAgentSessionEndedChild } from './structured-agent-session-host-types' +import type { StructuredAgentSessionCurrentWork } from './structured-agent-session-current-work' import { runSettledAgentSessionMutation } from './structured-agent-session-operation-settlement' import { agentSessionOperationOutcomeUnknown, @@ -75,8 +75,8 @@ export type AgentSessionMutationRequest = { plan: MutationPlan /** Journal of the attached session, read after `prepareSession`; absent when this host holds none. */ journal: () => AgentSessionJournal | undefined - /** The last child this host saw end, which the current-work projection reads. */ - endedChild?: () => StructuredAgentSessionEndedChild | undefined + /** The host's projection of the chat's current work (`hostStructuredAgentSessionCurrentWork`). */ + currentWork?: () => StructuredAgentSessionCurrentWork | null /** Between the ledger's answer and the lease check, for a call that may first have to make the * session ready for itself. Answers with the refusal that ends the call, if any. */ prepareSession?: ( diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts index 487f5417761f..e91d81e03c7b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-context.ts @@ -16,6 +16,7 @@ import type { MutationPlan } from './structured-agent-session-mutation-plans' import type { StructuredAgentSessionStopEnding } from './structured-agent-session-host-lifetime' import type { StructuredAgentSessionAcquireAborts } from './structured-agent-session-acquire-aborts' import type { StructuredAgentSessionOptionRevisions } from './structured-agent-session-option-revisions' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import type { StructuredAgentSessionCaller, StructuredAgentSessionHostDeps, @@ -75,7 +76,11 @@ export function mutateStructuredAgentSession( envelope, plan, journal: () => context.sessions.get(envelope.sessionId)?.journal, - endedChild: () => context.sessions.get(envelope.sessionId)?.lastEndedChild, + currentWork: () => + hostStructuredAgentSessionCurrentWork( + { store: context.deps.store, sessions: context.sessions }, + envelope.sessionId + ), prepareSession, publish: (journal) => context.publish(envelope.sessionId, journal), wakeDelivery: (sessionId) => context.wakeDelivery(sessionId), diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts index 86824a116a0e..40ecfb34c6c1 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-mutation-turn-context.ts @@ -4,7 +4,6 @@ import type { AgentSessionRecord } from '../../../shared/agent-session-record' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import type { AgentSessionMutationRequest } from './structured-agent-session-mutation-admission' import type { AgentSessionTurnContext } from './structured-agent-session-turns' -import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' export function mutationTurnContext( request: AgentSessionMutationRequest, @@ -18,14 +17,7 @@ export function mutationTurnContext( sessionId, journal, fence, - currentWork: () => { - const ended = request.endedChild?.() - return structuredAgentSessionCurrentWork(journal, { - record: request.store.getRecord(sessionId), - replaced: request.store.replacedRuntime(sessionId), - ...(ended ? { ended } : {}) - }) - }, + ...(request.currentWork ? { currentWork: request.currentWork } : {}), adapter: request.adapter, agents: request.agents, agent: record.provider, diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts index 05b37d982173..cc1c623c0d24 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-messages.ts @@ -18,7 +18,6 @@ import { queuedSendAnswer } from './structured-agent-session-queued-send-answer' import { structuredAgentSessionSendBlock } from './structured-agent-session-send-preparation' import { queuedMessagesPublishedBytesRefusal } from './structured-agent-session-queued-published-bytes' import type { AgentSessionRecord } from '../../../shared/agent-session-record' -import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionJournal } from '../agent-session-journal/journal-store' import { contextStructuredAgentSessionCurrentWork, @@ -105,9 +104,7 @@ export function structuredQueueHold(input: StructuredQueueGateInput): Structured if (input.work.hasActionablePrompt()) { return 'prompt' } - // Delivery reads the owner's adjudication, not what is shown: an owner none has concluded about - // may still run, so its queue waits as a live one's does, though nothing it left reads working. - return agentSessionOwnerUnadjudicated(input.record) || input.work.working() ? 'working' : null + return input.work.working() ? 'working' : null } /** The card the drain sends next, or null while anything holds the queue: the drain's own pick diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts index 6c21fb6e4197..9cf9a0cf9317 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-queued-publication.ts @@ -19,8 +19,10 @@ import { nextStructuredQueuedMessage, type StructuredQueueGateInput } from './structured-agent-session-queued-messages' -import { structuredAgentSessionCurrentWork } from './structured-agent-session-current-work' -import type { StructuredAgentSessionCurrentWorkHost } from './structured-agent-session-host-current-work' +import { + heldStructuredAgentSessionCurrentWork, + type StructuredAgentSessionCurrentWorkHost +} from './structured-agent-session-host-current-work' export type QueuePublication = { queuedMessages: AgentSessionQueuedMessage[] @@ -51,17 +53,15 @@ export function structuredQueueSendGate( ): QueueSendGate { return (journal) => { const record = host.store.getRecord(sessionId) - const session = host.sessions.get(sessionId) - const ended = session?.lastEndedChild const abandoned = drain?.abandonedCard(sessionId) return { record, - work: structuredAgentSessionCurrentWork(journal, { - record, - replaced: host.store.replacedRuntime(sessionId), - ...(session ? { child: session.child } : {}), - ...(ended ? { ended } : {}) - }), + work: heldStructuredAgentSessionCurrentWork( + host.store, + sessionId, + journal, + host.sessions.get(sessionId) + ), ...(drain ? { drainWaits: (messageId: string) => drain.waits(sessionId, messageId) } : {}), ...(abandoned ? { abandoned } : {}) } diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts index b750bb88a605..b3e4c9bbfff7 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-reveal.ts @@ -10,7 +10,6 @@ // a send does. And a journal it cannot open is not a refusal — the chat shows that failure with a // Retry, so the tab is worth publishing either way. -import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' import type { AgentSessionWireRefusal } from '../../../shared/agent-session-wire' import { agentSessionRefusalError } from '../../../shared/agent-session-wire-refusals' import { journalOpenRefusal } from '../agent-session-journal/journal-open-failure' @@ -113,7 +112,8 @@ export function createStructuredAgentSessionHostRestore( let restoring = false const latchedTargets = () => { for (const sessionId of restoreTargets) { - if (!agentSessionOwnerUnadjudicated(deps.store.getRecord(sessionId))) { + const lease = deps.store.getRecord(sessionId)?.lease + if (!lease?.unreconciled && lease?.handoffStage !== 'recovering') { restoreTargets.delete(sessionId) } } @@ -192,7 +192,7 @@ export function createStructuredAgentSessionHostRestore( // Once, as the restorer restores once. for (const { sessionId, lease } of restoring ? [] : deps.store.listRecords()) { if ( - agentSessionOwnerUnadjudicated({ lease }) && + (lease.unreconciled || lease.handoffStage === 'recovering') && (!sessionIds || sessionIds.includes(sessionId)) ) { restoreTargets.add(sessionId) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts index 7886149e0356..4c48b155598c 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-rewind.ts @@ -18,6 +18,7 @@ import { openWithAgent } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionAttachContext } from './structured-agent-session-attach-context' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' import { admitAndRunAgentSessionMutation } from './structured-agent-session-mutation-admission' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { conversationCommandBlocked } from './structured-conversation-command-admission' import { rewindRefusal } from './structured-rewind-refusal' import { renameRewindTurnOpener } from './structured-rewind-journal-body' @@ -60,7 +61,8 @@ export async function rewindStructuredAgentSession( : { ok: true } }), journal: () => context.sessions.get(sessionId)?.journal, - endedChild: () => context.sessions.get(sessionId)?.lastEndedChild, + currentWork: () => + hostStructuredAgentSessionCurrentWork({ store, sessions: context.sessions }, sessionId), publish: (journal) => context.publish(sessionId, journal), now: context.now, plan: { diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts index 6cf0f293a3ed..23347ef6948b 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-status-journal-projection.ts @@ -69,7 +69,7 @@ export class StructuredAgentSessionJournalProjections { const work = structuredAgentSessionCurrentWork(journal, { record, replaced: record ? this.replacedRuntime?.(record.sessionId) : undefined, - ...(seen ? { child: seen.child ?? null } : {}), + child: seen?.child ?? null, ...(ended ? { ended } : {}), revision: seen?.operationalRevision ?? 0 }) diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts index 919773ae54c0..423049ad7f86 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-turns.ts @@ -59,7 +59,7 @@ export type AgentSessionTurnContext = { fence: number /** The host's projection of what work is current (`structuredAgentSessionCurrentWork`), read * fresh per call: a child's end can move it mid-operation. */ - currentWork?: () => StructuredAgentSessionCurrentWork + currentWork?: () => StructuredAgentSessionCurrentWork | null adapter: StructuredAgentSessionAdapter /** What each agent declares; the session's own answer is `agents.capabilities(agent)`. */ agents: StructuredAgentRegistry diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts index 85c05a1e7cdb..94bbf5a82260 100644 --- a/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-unadjudicated-owner-queue.test.ts @@ -1,79 +1,202 @@ -// A queued card over an owner no adjudication has concluded about (unreconciled after a reload, or -// latched in recovery), on a chat no tab shows: it may still run, so the card waits, as on main. No -// automatic send starts an agent in its place, stops it, or hands the card back; once a probe -// proves it gone, the card drains. +// After a restart, an owner no adjudication has concluded about (unreconciled: its probe failed; +// recovering: proven alive, same runtime) on a chat no tab shows. Every host reader reads its +// leftover turn as the chat shows it: unverifiable, not working. The earlier process's card waits +// under the restart's pause; a person's own send goes to a start, which decides the owner. import { afterEach, expect, it, vi } from 'vitest' -import { HOST_TEST_SESSION as SESSION } from './structured-agent-session-host-test-data' +import { + HOST_TEST_SESSION as SESSION, + hostTestOperationId +} from './structured-agent-session-host-test-data' import { createQueuedMessageTestRig, + QUEUED_RIG_CALLER as CALLER, type QueuedMessageTestRig } from './structured-agent-session-queued-message-rig.test-fixture' import { + currentJournal, leaveUnfinishedWork, sendText } from './structured-agent-session-leftover-settlement.test-fixture' +import { structuredQueuePauses } from './structured-agent-session-queued-pause' +import { setStructuredAgentSessionHost } from './structured-agent-session-registry' +import { readStructuredSessionGateFacts } from '../../runtime/orchestration/structured-mailbox-pointer-host' +import { structuredWorkerAgentStatus } from '../../runtime/orchestration/structured-worker-group-addressing' + +const MODES = ['unreconciled', 'recovering'] as const let rig: QueuedMessageTestRig | undefined afterEach(async () => { + setStructuredAgentSessionHost(null) await rig?.dispose() rig = undefined vi.restoreAllMocks() }) -it.each(['unreconciled', 'recovering'] as const)( - 'holds the queued card while the owner is %s, and sends it once the owner is proven gone', - async (mode) => { - vi.spyOn(console, 'warn').mockImplementation(() => undefined) - let ownerGone = false - rig = await createQueuedMessageTestRig({ - restartable: true, - probeOwner: async () => { - if (ownerGone) { - return { outcome: 'pid-absent' } - } - if (mode === 'unreconciled') { - throw new Error('the probe could not run') - } - return { outcome: 'identity-matched', matchedOn: ['process-start-time'] } +/** The chat reopened in a new process with its owner undecided; `earlier` is a card the earlier + * process queued while it worked. Answers the owner-stop spy and the probe's call count. */ +async function reopened( + mode: (typeof MODES)[number], + options: { leftoverTurn: boolean; earlier?: 'message' | 'compact' } +) { + vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const probes = { calls: 0 } + rig = await createQueuedMessageTestRig({ + restartable: true, + probeOwner: async () => { + probes.calls += 1 + if (mode === 'unreconciled') { + throw new Error('the probe could not run') } - }) - const current = rig - await current.workingSend() + return { outcome: 'identity-matched', matchedOn: ['process-start-time'] } + } + }) + const current = rig + await current.workingSend() + let earlierCard: string | undefined + if (options.earlier === 'message') { + const queued = await sendText(current, 'queued before the crash').result + earlierCard = queued.ok && 'queued' in queued.value ? queued.value.queued.messageId : undefined + } else if (options.earlier === 'compact') { + earlierCard = await queueCompact(current) + } + if (options.leftoverTurn) { await leaveUnfinishedWork(current) - // No visible-tab restore: nothing at startup decides the owner. - await current.crashReloadHostProcess(() => undefined) - await current.host.history({ sessionId: SESSION, direction: 'tail' }) - const stopOwnerProcess = vi.fn() - current.host.deps.stopOwnerProcess = stopOwnerProcess - const lease = current.store.getRecord(SESSION)?.lease - expect(mode === 'unreconciled' ? lease?.unreconciled : lease?.handoffStage).toBe( - mode === 'unreconciled' ? true : 'recovering' - ) + } + // No tab restore: nothing at startup decides the owner. + await current.crashReloadHostProcess(() => undefined) + await current.host.history({ sessionId: SESSION, direction: 'tail' }) + const stopOwnerProcess = vi.fn() + current.host.deps.stopOwnerProcess = stopOwnerProcess + const lease = current.store.getRecord(SESSION)?.lease + expect(mode === 'unreconciled' ? lease?.unreconciled : lease?.handoffStage).toBe( + mode === 'unreconciled' ? true : 'recovering' + ) + return { current, stopOwnerProcess, probes, earlierCard } +} + +async function queueCompact(current: QueuedMessageTestRig): Promise { + const fields = { command: 'compact' as const, delivery: 'queue-if-active' as const } + const id = hostTestOperationId() + const result = await current.host.conversationCommand(CALLER, { + envelope: current.envelope(fields, 'agentSession.conversationCommand', id), + ...fields, + userSend: true + }) + return result.ok && result.value.queued ? result.value.queued.messageId : undefined +} + +async function page(current: QueuedMessageTestRig) { + const read = await current.host.history({ sessionId: SESSION, direction: 'tail' }) + if (!read.ok) { + throw new Error('history refused') + } + return read.page +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 3_000)) + +it.each(MODES)( + "holds the earlier process's card under the restart's pause while the owner is %s: nothing sent or stopped", + async (mode) => { + const { current, stopOwnerProcess, earlierCard } = await reopened(mode, { + leftoverTurn: true, + earlier: 'message' + }) + expect(earlierCard).toBeDefined() const starts = current.starts.length const dispatches = current.dispatch.mock.calls.length - const sent = await sendText(current, 'while it may still run').result - await new Promise((resolve) => setTimeout(resolve, 3_000)) + await settle() - expect(sent).toMatchObject({ ok: true, value: { queued: expect.anything() } }) + expect(structuredQueuePauses(currentJournal(current)).map((pause) => pause.reason)).toContain( + 'restarted' + ) + const held = await page(current) + expect(held.working).toBe(false) + expect(held.nextQueuedMessageId ?? null).toBeNull() + expect(held.queuedMessages?.map((card) => [card.messageId, card.state])).toEqual([ + [earlierCard, 'waiting'] + ]) expect(current.starts.length).toBe(starts) expect(current.dispatch.mock.calls.length).toBe(dispatches) expect(stopOwnerProcess).not.toHaveBeenCalled() - const page = await current.host.history({ sessionId: SESSION, direction: 'tail' }) - expect(page.ok && page.page.queuedMessages?.map((card) => card.state)).toEqual(['waiting']) - expect(page.ok && page.page.queuedMessages?.[0]?.returnedReason).toBeUndefined() - // Shown as main shows it: unverifiable, not working. - expect(page.ok && page.page.working).toBe(false) - - ownerGone = true - await current.host.restoreReadableSessions([SESSION]) - await vi.waitFor(() => expect(current.dispatch.mock.calls.length).toBeGreaterThan(dispatches), { - timeout: 10_000 + + // The person's own send is no card the queue could send for them later: it goes to a start. + const sent = await sendText(current, 'while it may still run').result + expect(sent).toMatchObject({ ok: true, value: { submission: expect.anything() } }) + await settle() + const after = await page(current) + expect(after.queuedMessages?.map((card) => [card.messageId, card.state])).toEqual([ + [earlierCard, 'waiting'] + ]) + expect(after.queuedMessages?.[0]?.returnedReason).toBeUndefined() + expect(await current.handoff(earlierCard ?? '')).toBeUndefined() + }, + 30_000 +) + +it.each(MODES.flatMap((mode) => [true, false].map((leftoverTurn) => ({ mode, leftoverTurn }))))( + "sends a person's queue-if-active message to a start that decides the owner ($mode, leftover turn: $leftoverTurn)", + async ({ mode, leftoverTurn }) => { + const { current, stopOwnerProcess, probes } = await reopened(mode, { leftoverTurn }) + expect((await page(current)).working).toBe(false) + const probed = probes.calls + + const sent = await sendText(current, 'while it may still run').result + + expect(sent).toMatchObject({ ok: true, value: { submission: expect.anything() } }) + expect(sent.ok && 'queued' in sent.value).toBe(false) + await vi.waitFor(() => expect(probes.calls).toBeGreaterThan(probed), { timeout: 10_000 }) + expect((await page(current)).queuedMessages ?? []).toEqual([]) + // A recovery the start resolves ends the owner it proved alive, as on main; no proof, no stop. + if (mode === 'recovering') { + await vi.waitFor(() => expect(stopOwnerProcess).toHaveBeenCalledOnce(), { timeout: 10_000 }) + } else { + expect(stopOwnerProcess).not.toHaveBeenCalled() + } + }, + 30_000 +) + +it.each(MODES)( + "sends a /compact card on Send while the chat shows the %s owner's turn as not working", + async (mode) => { + const { current, earlierCard } = await reopened(mode, { + leftoverTurn: true, + earlier: 'compact' }) - expect(JSON.stringify(current.dispatch.mock.calls.at(-1))).toContain('while it may still run') - expect(stopOwnerProcess).not.toHaveBeenCalled() + expect(earlierCard).toBeDefined() + expect((await page(current)).working).toBe(false) + + const sent = await current.sendNow(earlierCard ?? '') + + expect(sent).toMatchObject({ + ok: true, + value: { submission: { queuedMessageId: earlierCard } } + }) + }, + 30_000 +) + +it.each(MODES)( + "reads the %s owner's leftover turn for @idle as the chat shows it", + async (mode) => { + const { current } = await reopened(mode, { leftoverTurn: true }) + setStructuredAgentSessionHost(current.host) + const record = current.store.getRecord(SESSION) + if (!record) { + throw new Error('expected the record') + } + const { working } = await page(current) + + const facts = await readStructuredSessionGateFacts(SESSION) + const status = await structuredWorkerAgentStatus({ sessionId: SESSION, record, lineage: [] }) + + expect(working).toBe(false) + expect(facts?.turnRunning).toBe(working) + expect(status).toBe('idle') }, 30_000 ) diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts index 0881fe6ba3b2..b40265c16b4b 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command-admission.ts @@ -28,7 +28,7 @@ export type ConversationCommandAdmissionContext = { snapshot(): Pick, 'items'> } /** The host's projection of current work; without one, the generation at `fence` is live. */ - currentWork?: () => StructuredAgentSessionCurrentWork + currentWork?: () => StructuredAgentSessionCurrentWork | null adapter: Pick } diff --git a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts index 9d36e1b43d03..606d2093c93b 100644 --- a/src/main/native-chat/agent-session-wire/structured-conversation-command.ts +++ b/src/main/native-chat/agent-session-wire/structured-conversation-command.ts @@ -11,6 +11,7 @@ import { agentSessionOperationKey } from '../../../shared/agent-session-operatio import type { StructuredAgentSessionMutationContext } from './structured-agent-session-host-mutations' import { sendPreparation } from './structured-agent-session-send-preparation' import type { StructuredAgentSessionCaller } from './structured-agent-session-host-types' +import { hostStructuredAgentSessionCurrentWork } from './structured-agent-session-host-current-work' import { conversationCommandBlocked } from './structured-conversation-command-admission' import type { AgentSessionFailureFact } from '../../../shared/agent-session-failure' import { @@ -69,7 +70,8 @@ export function runStructuredConversationCommand( // Starts the agent only to settle a rewind in doubt, as a send does; a /clear itself starts nothing. prepareSession: sendPreparation(context, envelope, { refusesInRun: true }), journal: () => context.sessions.get(sessionId)?.journal, - endedChild: () => context.sessions.get(sessionId)?.lastEndedChild, + currentWork: () => + hostStructuredAgentSessionCurrentWork({ store, sessions: context.sessions }, sessionId), publish: (journal) => context.publish(sessionId, journal), now: context.now, plan: { diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts index c686a82a1a67..05683fcc505c 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.test.ts @@ -75,24 +75,7 @@ describe('structured mailbox pointer host', () => { const items = [runningTurn(), ...transcript(500)] hostRef.current = { journalSnapshot: () => ({ items, submissions: [] }), - currentWork: () => null, - deps: { store: { getRecord: () => null } } - } - expect(await readStructuredSessionGateFacts('s1')).toEqual({ - turnRunning: true, - awaitingHuman: false - }) - }) - - it.each([ - { unreconciled: true, handoffStage: null }, - { unreconciled: false, handoffStage: 'recovering' } - ])('never reads an owner no adjudication has concluded about as idle (%o)', async (lease) => { - // Nothing it left reads as current work, yet it may still run: `@idle` must not pick it. - hostRef.current = { - journalSnapshot: () => ({ items: [runningTurn()], submissions: [] }), - currentWork: () => ({ scope: { isCurrentItem: () => false, owesSend: () => false } }), - deps: { store: { getRecord: () => ({ lease }) } } + currentWork: () => null } expect(await readStructuredSessionGateFacts('s1')).toEqual({ turnRunning: true, diff --git a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts index 4efd0e5baaf8..92b1e0594823 100644 --- a/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/structured-mailbox-pointer-host.ts @@ -18,7 +18,6 @@ import { type StructuredSessionGateFacts } from './structured-session-pointer-delivery' import { sendAgentTurn } from './send-agent-turn' -import { agentSessionOwnerUnadjudicated } from '../../../shared/agent-session-lease-adjudication' /** Names the dispatch a nudge belongs to on the operation row it writes. */ export function structuredPointerCallerKey(dispatchId: string): string { @@ -52,12 +51,8 @@ export async function readStructuredSessionGateFacts( return null } // The host's projection of current work: an agent that ended is idle, whatever its rows say. - const host = getStructuredAgentSessionHost() - const facts = structuredSessionGateFacts(snapshot.items, host?.currentWork(sessionId)?.scope) - // An owner no adjudication has concluded about may still run: never idle, as its queue holds. - return agentSessionOwnerUnadjudicated(host?.deps.store.getRecord(sessionId)) - ? { ...facts, turnRunning: true } - : facts + const work = getStructuredAgentSessionHost()?.currentWork(sessionId) + return structuredSessionGateFacts(snapshot.items, work?.scope) } /** What each recorded send settled as. */ diff --git a/src/shared/agent-session-lease-adjudication.ts b/src/shared/agent-session-lease-adjudication.ts index 0f924b11ae60..3365159b7a0b 100644 --- a/src/shared/agent-session-lease-adjudication.ts +++ b/src/shared/agent-session-lease-adjudication.ts @@ -122,16 +122,6 @@ export function agentSessionLeaseOwnerVerdict(lease: AgentSessionLease): AgentSe : 'unverifiable' } -/** No adjudication has concluded about the recorded owner yet: unreconciled since the restart, or - * latched in recovery. It may still run, so nothing starts in its place (`evaluateAgentSession- - * Acquisition` refuses both) and nothing is delivered to the chat until one concludes. */ -export function agentSessionOwnerUnadjudicated( - record: { lease: Pick } | null | undefined -): boolean { - const lease = record?.lease - return lease !== undefined && (lease.unreconciled || lease.handoffStage === 'recovering') -} - /** Nothing holds this lease: released, no handoff in flight, and reconciled since the last restart. */ export function agentSessionLeaseIsReleased(lease: AgentSessionLease): boolean { return !lease.unreconciled && lease.claimStatus === 'released' && lease.handoffStage === null From 2947840eaa120b558790e9a99e5ad6ae4cc6d8f6 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 06:06:43 -0700 Subject: [PATCH 20/22] test(mobile): queued Steer pin uses main's card options --- mobile/src/session/MobileNativeChatQueuedMessages.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx index b781a230a94a..24f78f532ba2 100644 --- a/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx +++ b/mobile/src/session/MobileNativeChatQueuedMessages.test.tsx @@ -223,7 +223,7 @@ describe('MobileNativeChatQueuedMessages', () => { } ], [], - { pendingPrompt: false, agentWorking: false } + { agentWorking: false } ) const mounted = await mount({ cards: [waiting!], onSend: vi.fn(async () => true) }) expect(texts(mounted).filter((text) => text === 'Send' || text === 'Steer')).toEqual(['Steer']) From c26186a778d95b9c8404d1b209d54cc2cf5ae0c4 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:35:50 -0700 Subject: [PATCH 21/22] test(native-chat): status-feed journal fixtures answer runningTurn --- src/main/agent-hooks/first-work-branch-rename.test.ts | 10 ++++++---- .../claude/claude-background-wake-retention.test.ts | 2 ++ .../structured-agent-session-rpc.test-fixture.ts | 5 +++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/agent-hooks/first-work-branch-rename.test.ts b/src/main/agent-hooks/first-work-branch-rename.test.ts index 13106a8e91bf..51501b9f6600 100644 --- a/src/main/agent-hooks/first-work-branch-rename.test.ts +++ b/src/main/agent-hooks/first-work-branch-rename.test.ts @@ -112,14 +112,15 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { // A real journal's sequence only ever advances, so the feed's projection // cache must miss on every publish here: this test is about the rename. let sequence = 0 - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot() and the provenance reads of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot(), the provenance reads and runningTurn() of a journal. const journal = { snapshot: () => ({ items, submissions: [] }), stopMarks: { latest: () => null, revision: () => 0 }, lastActivityAt: () => 1, cursor: () => ({ epoch: 1, sequence: (sequence += 1) }), itemFence: () => undefined, - wroteBeforeOpen: () => false + wroteBeforeOpen: () => false, + runningTurn: () => null } as unknown as AgentSessionJournal const pending: Promise[] = [] const observe = vi.fn((summary, options) => { @@ -203,7 +204,7 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { const { deps, setDisplayName, setRenameError } = makeDeps({ getRepo: () => ({ id: REPO_ID, kind: 'folder', path: '/workspace/platform' }) as Repo }) - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot() and the provenance reads of a journal. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot(), the provenance reads and runningTurn() of a journal. const journal = { lastActivityAt: () => 1, stopMarks: { latest: () => null, revision: () => 0 }, @@ -222,7 +223,8 @@ describe('maybeAutoRenameBranchOnFirstWork', () => { submissions: [] }), itemFence: () => undefined, - wroteBeforeOpen: () => false + wroteBeforeOpen: () => false, + runningTurn: () => null } as unknown as AgentSessionJournal const location = { executionHostId: 'local' as const, diff --git a/src/main/claude/claude-background-wake-retention.test.ts b/src/main/claude/claude-background-wake-retention.test.ts index 7e3679d2e328..d65d4a9a4451 100644 --- a/src/main/claude/claude-background-wake-retention.test.ts +++ b/src/main/claude/claude-background-wake-retention.test.ts @@ -43,6 +43,8 @@ async function wiredSession() { // One generation produced everything here, and every send was accepted by this process. itemFence: () => undefined, wroteBeforeOpen: () => false, + // The live generation's own turns are never a died turn, so this is never consulted for one. + runningTurn: () => null, snapshot: () => ({ items: [...run.journalItems.values()] .sort((a, b) => a.sequence - b.sequence) diff --git a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts index 9d0bc4eb4663..ce9b35c8f3c3 100644 --- a/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts +++ b/src/main/runtime/rpc/methods/structured-agent-session-rpc.test-fixture.ts @@ -101,13 +101,14 @@ function statusFeed(): StructuredAgentSessionStatusFeed { [ STATUS_SESSION, { - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot() and itemFence() of a journal with no sends. + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the status feed reads only cursor(), lastActivityAt(), stopMarks, snapshot(), itemFence() and runningTurn() of a journal with no sends. journal: { cursor: () => ({ epoch: 'epoch-status', sequence: 2 }), lastActivityAt: () => 2, stopMarks: { latest: () => null, revision: () => 0 }, snapshot: () => ({ items: STATUS_ITEMS, submissions: [] }), - itemFence: () => undefined + itemFence: () => undefined, + runningTurn: () => null } as unknown as AgentSessionJournal, params: { location: { From 3b19224a296ce989a1ecbf86b7eedb3a0df79d50 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:48:22 -0700 Subject: [PATCH 22/22] style(native-chat): keep the message list under its line cap with main's newest additions --- .../src/components/native-chat/NativeChatMessageList.tsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx index c6dbfa325e7e..10b3e7414e31 100644 --- a/src/renderer/src/components/native-chat/NativeChatMessageList.tsx +++ b/src/renderer/src/components/native-chat/NativeChatMessageList.tsx @@ -68,7 +68,7 @@ import { structuredAgentLabel } from '@/lib/structured-agent-session-launch-labe * `unshown`: it cannot (the prompt is only in the agent's terminal). */ export type NativeChatAwaitingInput = 'shown' | 'unshown' -type NativeChatNavigationRequest = +type NavigationRequest = | { kind: 'diff'; target: NativeChatDiffReveal } | { kind: 'rail'; messageId: string; requestId: number } @@ -125,9 +125,7 @@ export function NativeChatMessageList({ stopping?: boolean runtimeContext?: RuntimeFileOperationArgs | null }): React.JSX.Element { - const [navigationRequest, setNavigationRequest] = useState( - null - ) + const [navigationRequest, setNavigationRequest] = useState(null) const navigationSequence = useRef(0) const revealedDiff = navigationRequest?.kind === 'diff' ? navigationRequest.target : null const railJump = navigationRequest?.kind === 'rail' ? navigationRequest : null