From 0b638203f06d6265c71cdeb6b52d165d364badb9 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 00:47:56 +0800 Subject: [PATCH 01/15] fix(loro.js): replay history on the first checkout after a snapshot import Snapshot-hydrated state has no tombstones or op history, so incremental transitions returned wrong historical state, and a replay could hydrate lazy latest-state entries on top of replayed ops ("duplicate sequence id"). Rationale: context/loro-js-performance.md (snapshot-hydrated state). Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-snapshot-checkout.md | 5 ++ context/loro-js-performance.md | 13 +++- loro-js/src/runtime/document.ts | 36 ++++++++++ loro-js/tests/snapshot-checkout.test.ts | 93 +++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 .changeset/loro-js-snapshot-checkout.md create mode 100644 loro-js/tests/snapshot-checkout.test.ts diff --git a/.changeset/loro-js-snapshot-checkout.md b/.changeset/loro-js-snapshot-checkout.md new file mode 100644 index 000000000..00e44a649 --- /dev/null +++ b/.changeset/loro-js-snapshot-checkout.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +Fix checkout and `diff` on a `loro.js` document imported from a snapshot. Checking out an earlier version could return the latest value of a child container that had not been read yet, restore nothing for elements deleted before the snapshot, skip a delete when moving forward again, or throw `duplicate sequence id`. The first checkout after a snapshot import now replays history once; later checkouts keep using incremental transitions. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 896482166..84ee145da 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -1,6 +1,6 @@ # loro-js Performance Architecture -Verified against code 2026-07-22. +Verified against code 2026-09-28. The pure TypeScript runtime lives in `loro-js/src/runtime`. Its performance target is the asymptotic behavior of the Rust runtime, while accepting a larger @@ -127,6 +127,17 @@ JavaScript constant factor. staging document before installing them. Import subscribers retain eager state hydration because their import event must describe every changed container. +- State hydrated from a latest-state snapshot (eager or lazy) has no + tombstones, winner history, or move history, so incremental version + transitions cannot retreat it. `LoroDoc.#stateFromSnapshot` makes + `#canTransitionRecords` refuse, and the first checkout or `diff` after such an + import replays history once through `#rebuildFromHistory` (about 34 ms for a + 64k-operation Text on an Apple M5 Pro; later checkouts stay around 0.3–0.7 + ms). The replay first discards the lazy latest-state SSTable + (`#discardDeferredSnapshotState`); otherwise replayed containers would later + hydrate their latest encoded state on top of the replayed operations. A + delete transition is also refused unless the deletion index recorded that + delete operation, since a replay to an earlier version never applied it. When an element's deleted flag, tree parent/position, or map visibility changes, mutate it through its owning index helper. Direct mutation leaves subtree or diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 9b5decf69..754a80edd 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -255,6 +255,10 @@ export class LoroDoc = Record(); #deferredSnapshotHistory: DeferredSnapshotHistory | undefined; #deferredSnapshotState: DeferredSnapshotState | undefined; + // Latest-state snapshot entries carry no tombstones, winner history, or move + // history, so incremental version transitions cannot retreat them. The first + // checkout replays history instead; see context/loro-js-performance.md. + #stateFromSnapshot = false; #hydratedSnapshotContainers = new Set(); #hydratingSnapshotContainers = new Set(); #snapshotContainerDepths = new Map(); @@ -787,6 +791,7 @@ export class LoroDoc = Record = Record = Record(); if (movableMoveMode === "replay") { for (const { change } of records) { @@ -3823,6 +3844,17 @@ export class LoroDoc = Record = Record = Record + versions.map((version) => { + doc.checkout(version); + return doc.toJSON(); + }); + +describe("checkout after a lazy snapshot import", () => { + test("matches the full-history document at every version", () => { + const { doc, versions } = buildHistory(); + const expected = valuesAt(doc, versions); + doc.checkoutToLatest(); + const bytes = doc.export({ mode: "snapshot" }); + + // Checking out before anything is read exercises both the version transition + // and the full-replay paths on unhydrated children. + for (const order of [versions, [...versions].reverse()]) { + const loaded = new LoroDoc(); + loaded.import(bytes); + const expectedInOrder = order.map((version) => expected[versions.indexOf(version)]); + expect(valuesAt(loaded, order)).toEqual(expectedInOrder); + loaded.checkoutToLatest(); + expect(loaded.toJSON()).toEqual(doc.toJSON()); + } + }); + + test("reads a child first touched after checkout at its historical state", () => { + const { doc, versions } = buildHistory(); + const loaded = new LoroDoc(); + loaded.import(doc.export({ mode: "snapshot" })); + loaded.checkout(versions[0]!); + const root = loaded.getMap("root"); + expect((root.get("map") as LoroMap).toJSON()).toEqual({ a: 1 }); + expect((root.get("list") as LoroList).toJSON()).toEqual([1]); + expect((root.get("text") as LoroText).toString()).toBe("abc"); + }); + + test("keeps exported snapshots consistent after checkout round trips", () => { + const { doc, versions } = buildHistory(); + const loaded = new LoroDoc(); + loaded.import(doc.export({ mode: "snapshot" })); + loaded.checkout(versions[0]!); + loaded.checkoutToLatest(); + const again = new LoroDoc(); + again.import(loaded.export({ mode: "snapshot" })); + expect(again.toJSON()).toEqual(doc.toJSON()); + again.checkout(versions[1]!); + doc.checkout(versions[1]!); + expect(again.toJSON()).toEqual(doc.toJSON()); + }); + + test("checks out a shallow snapshot through its retained history", () => { + const { doc, versions } = buildHistory(); + const loaded = new LoroDoc(); + loaded.import(doc.export({ mode: "shallow-snapshot", frontiers: versions[0]! })); + const expected = valuesAt(doc, versions); + expect(valuesAt(loaded, versions)).toEqual(expected); + }); +}); From 0f0bad0afed525735c5ef214e01e15e85b204c14 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 02:57:41 +0800 Subject: [PATCH 02/15] fix(loro.js): rebuild only the snapshot containers a checkout touches The first checkout after a snapshot import replayed the whole document, up to three times for the first `diff`, and replaced correct snapshot state with loro.js replays. Replaying Rust-created rich text lost styles and moved text, because loro.js positions do not count Rust style anchors. A transition now hydrates and rebuilds only the Text/List containers it touches, once, and keeps a snapshot state that the replay does not reproduce for every version with the same operations on that container. Checkout events of unread nested containers are correct, and a full snapshot export keeps unread containers after history is loaded. See context/loro-js-performance.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-snapshot-checkout.md | 2 +- context/loro-js-performance.md | 46 +- loro-js/src/runtime/containers.ts | 97 ++- loro-js/src/runtime/document.ts | 791 ++++++++++++++++-- loro-js/tests/fixtures/rust/styled-text.blob | Bin 0 -> 404 bytes .../fixtures/rust/styled-text.expected.json | 154 ++++ loro-js/tests/snapshot-checkout.test.ts | 189 ++++- 7 files changed, 1197 insertions(+), 82 deletions(-) create mode 100644 loro-js/tests/fixtures/rust/styled-text.blob create mode 100644 loro-js/tests/fixtures/rust/styled-text.expected.json diff --git a/.changeset/loro-js-snapshot-checkout.md b/.changeset/loro-js-snapshot-checkout.md index 00e44a649..dfd53d8bd 100644 --- a/.changeset/loro-js-snapshot-checkout.md +++ b/.changeset/loro-js-snapshot-checkout.md @@ -2,4 +2,4 @@ "loro.js": patch --- -Fix checkout and `diff` on a `loro.js` document imported from a snapshot. Checking out an earlier version could return the latest value of a child container that had not been read yet, restore nothing for elements deleted before the snapshot, skip a delete when moving forward again, or throw `duplicate sequence id`. The first checkout after a snapshot import now replays history once; later checkouts keep using incremental transitions. +Fix checkout and `diff` on a `loro.js` document imported from a snapshot. Checking out an earlier version could return the latest value of a child container that had not been read yet, restore nothing for elements deleted before the snapshot, skip a delete when moving forward again, or throw `duplicate sequence id`. A transition now rebuilds only the Text and List containers it touches from their own history, once. Checkout events for containers that had not been read are now correct, and a snapshot exported after history was loaded no longer drops containers that had not been read. Rust-created rich text keeps its snapshot styles and text at the snapshot version. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 84ee145da..c001db273 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -127,17 +127,41 @@ JavaScript constant factor. staging document before installing them. Import subscribers retain eager state hydration because their import event must describe every changed container. -- State hydrated from a latest-state snapshot (eager or lazy) has no - tombstones, winner history, or move history, so incremental version - transitions cannot retreat it. `LoroDoc.#stateFromSnapshot` makes - `#canTransitionRecords` refuse, and the first checkout or `diff` after such an - import replays history once through `#rebuildFromHistory` (about 34 ms for a - 64k-operation Text on an Apple M5 Pro; later checkouts stay around 0.3–0.7 - ms). The replay first discards the lazy latest-state SSTable - (`#discardDeferredSnapshotState`); otherwise replayed containers would later - hydrate their latest encoded state on top of the replayed operations. A - delete transition is also refused unless the deletion index recorded that - delete operation, since a replay to an earlier version never applied it. +- Text, List, and MovableList state hydrated from a snapshot (eager, lazy, or a + shallow root) has no tombstones, deletion index, or style, value, and move + history for the snapshot's operations. `LoroDoc.#snapshotSequences` records + each such container with its snapshot version. Before `checkout`, + `checkoutToLatest`, or `diff` transitions, + `#prepareSnapshotTransition` looks only at the containers the transition + touches: it hydrates lazily encoded ones (an untouched lazy container still + holds its latest state, which is its state at the current version) and, when + the transition crosses a snapshot operation, rebuilds that one container from + its own operations (`#completeSnapshotSequence`, from its shallow root entry + in a shallow document). Operations applied after hydration are indexed like + any others, and Map, Tree, and Counter state needs no rebuild. The per-container + record index is built once per history revision. Consecutive text inserts that + continue each other replay as one span (`coalescedTextInsert`). Unrelated + containers are never replayed and the lazy SSTable is kept, so snapshot export + copies every untouched entry and rewrites only the touched ones. A delete + transition is still refused unless the deletion index recorded that delete, + for example one imported while detached. +- loro.js does not count Rust rich-text style anchors in Text operation + positions, so a replay of Rust-created styled text can differ from its + snapshot state (for example, an insert right after a mark's end anchor). The + completion therefore compares the replay with the snapshot state (visible ids, + plus the delta when styled). When they differ, the replayed state serves + transitions and the snapshot state is reinstated at every version with the + same operations on that container (`#settleSnapshotSequences`), so the + snapshot version and exports stay exact. A full `#rebuildFromHistory`, still + used when a transition cannot be incremental, stashes installed snapshot states + the same way. +- First checkout after importing a 262,144-operation single-peer Text snapshot: + about 60–70 ms on a loaded Apple M5 Pro, versus about 150 ms for the earlier + whole-document replay; the replay of that one container dominates. A doc with + 32,768 child Maps that retreats one of them needs no replay: 63 ms versus + 170 ms, with the same 234 MiB peak RSS as before the fix. A shallow document + whose Map or Tree winner at the target is a trimmed root-time write replays + from the shallow root state. When an element's deleted flag, tree parent/position, or map visibility changes, mutate it through its owning index helper. Direct mutation leaves subtree or diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 9b3979b91..a543a0a00 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -74,6 +74,34 @@ export interface SequenceMoveMeta { export type CausalVersion = ReadonlyMap; +/** + * Detached internal state of a List, MovableList, or Text, returned by + * `_swapState`. The document swaps a snapshot-hydrated state out while it + * rebuilds the container from history, and can install it again later. + */ +export interface SequenceContainerState { + readonly sequence: SequenceIndex | SequenceIndex; +} + +interface ListState extends SequenceContainerState { + readonly sequence: SequenceIndex; + readonly detachedCounter: number; +} + +interface MovableListState extends ListState { + readonly valueHistoryComplete: boolean; + readonly moveHistoryComplete: boolean; +} + +interface TextState extends SequenceContainerState { + readonly sequence: SequenceIndex; + readonly detachedCounter: number; + readonly detachedStyleCounter: number; + readonly attributeHistoryComplete: boolean; + readonly styleIndex: TextStyleIndex; + readonly styleVersion: CausalVersion | undefined; +} + interface ParentLink { readonly container: LoroContainer; readonly binding?: @@ -363,7 +391,7 @@ export class LoroMap< } export class LoroList extends LoroContainer { - readonly _sequence = new SequenceIndex(); + _sequence = new SequenceIndex(); _detachedCounter = 0; get _elements(): SequenceElement[] { @@ -579,6 +607,21 @@ export class LoroList extends LoroContainer { this._detachedCounter = 0; } + /** Installs `state`, or an empty state, and returns the replaced state. */ + _swapState(state?: SequenceContainerState): SequenceContainerState { + const previous: ListState = { + sequence: this._sequence, + detachedCounter: this._detachedCounter, + }; + const next = state as ListState | undefined; + this._sequence = next?.sequence ?? new SequenceIndex(); + this._detachedCounter = next?.detachedCounter ?? 0; + this._sequence.forEachPhysicalRaw((element) => { + if (element.value instanceof LoroContainer) this._bindChildren([element]); + }); + return previous; + } + _bindChildren(elements: readonly SequenceElement[]): void { for (const element of elements) { if (element.value instanceof LoroContainer) { @@ -1362,17 +1405,11 @@ export interface TextStyleMeta { } export class LoroText extends LoroContainer { - readonly _sequence = new SequenceIndex( - (element) => ({ - utf16: element.value.length, - utf8: utf8CodePointLength(element.value), - }), - (element) => countLineBreaks(element.value), - ); + _sequence = createTextSequence(); _detachedCounter = 0; _detachedStyleCounter = 0; _attributeHistoryComplete = true; - readonly _styleIndex = new TextStyleIndex(); + _styleIndex = new TextStyleIndex(); _styleVersion: CausalVersion | undefined; readonly #attributeValuesCache = new WeakMap< ReadonlyMap, @@ -1995,6 +2032,36 @@ export class LoroText extends LoroContainer { this._styleIndex.reset(); this._styleVersion = undefined; } + + /** Installs `state`, or an empty state, and returns the replaced state. */ + _swapState(state?: SequenceContainerState): SequenceContainerState { + const previous: TextState = { + sequence: this._sequence, + detachedCounter: this._detachedCounter, + detachedStyleCounter: this._detachedStyleCounter, + attributeHistoryComplete: this._attributeHistoryComplete, + styleIndex: this._styleIndex, + styleVersion: this._styleVersion, + }; + const next = state as TextState | undefined; + this._sequence = next?.sequence ?? createTextSequence(); + this._detachedCounter = next?.detachedCounter ?? 0; + this._detachedStyleCounter = next?.detachedStyleCounter ?? 0; + this._attributeHistoryComplete = next?.attributeHistoryComplete ?? true; + this._styleIndex = next?.styleIndex ?? new TextStyleIndex(); + this._styleVersion = next?.styleVersion; + return previous; + } +} + +function createTextSequence(): SequenceIndex { + return new SequenceIndex( + (element) => ({ + utf16: element.value.length, + utf8: utf8CodePointLength(element.value), + }), + (element) => countLineBreaks(element.value), + ); } export class LoroMovableList extends LoroList { @@ -2159,6 +2226,18 @@ export class LoroMovableList extends LoroList { this._valueHistoryComplete = true; this._moveHistoryComplete = true; } + + override _swapState(state?: SequenceContainerState): SequenceContainerState { + const previous: MovableListState = { + ...(super._swapState(state) as ListState), + valueHistoryComplete: this._valueHistoryComplete, + moveHistoryComplete: this._moveHistoryComplete, + }; + const next = state as MovableListState | undefined; + this._valueHistoryComplete = next?.valueHistoryComplete ?? true; + this._moveHistoryComplete = next?.moveHistoryComplete ?? true; + return previous; + } } export class LoroCounter extends LoroContainer { diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 324a4adac..fd41ec00e 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -65,6 +65,7 @@ import { type CausalVersion, type LastWriter, type RuntimeValue, + type SequenceContainerState, type SequenceElement, type SequenceMoveMeta, type SequenceValueMeta, @@ -171,6 +172,8 @@ interface DeferredSnapshotHistory { interface DeferredSnapshotState { readonly store: Extract; + // The version of the encoded state. + readonly version: VersionVector; } interface IndexedHistoryOperation { @@ -192,6 +195,43 @@ interface EventRecording { type MovableMoveTransitionMode = "anchors" | "replay"; +type SnapshotSequenceState = + // The snapshot state of `version` is installed. It has no history for the + // operations of `version`; operations applied since then are indexed. + | { readonly kind: "hydrated"; readonly version: VersionVector } + // A replayed state is installed. `snapshot` is the container's state at + // every version whose operations on it match `version`. `fingerprint` is set + // until a replay has been compared with it. + | { + readonly kind: "replayed"; + readonly snapshot: SequenceContainerState; + readonly version: VersionVector; + readonly fingerprint: SequenceFingerprint | undefined; + } + // The snapshot state is installed again; `replayed` is the replayed state for + // the same operations as `version`, valid until an operation is applied to + // the container. + | { + readonly kind: "pinned"; + readonly replayed: SequenceContainerState; + readonly version: VersionVector; + }; + +interface SnapshotTransition { + readonly beforeValues: Map; + readonly pins: readonly string[]; +} + +interface SequenceFingerprint { + readonly runs: readonly SequenceIdRun[]; + readonly values: readonly unknown[] | undefined; +} + +interface ContainerHistoryIndex { + readonly revision: number; + readonly records: Map; +} + interface PendingChange extends EventRecording { readonly id: CodecId; readonly dependencies: CodecId[]; @@ -256,10 +296,18 @@ export class LoroDoc = Record(); #deferredSnapshotHistory: DeferredSnapshotHistory | undefined; #deferredSnapshotState: DeferredSnapshotState | undefined; - // Latest-state snapshot entries carry no tombstones, winner history, or move - // history, so incremental version transitions cannot retreat them. The first - // checkout replays history instead; see context/loro-js-performance.md. - #stateFromSnapshot = false; + // Text, List, and MovableList containers whose state came from a snapshot. + // It has no tombstones or style, value, and move history, so a version + // transition first rebuilds each touched one from its own history; see + // #prepareSnapshotTransition and context/loro-js-performance.md. + #snapshotSequences = new Map(); + // Incremented whenever retained history changes; invalidates + // #containerHistoryIndex. + #historyRevision = 0; + #containerHistoryIndex: ContainerHistoryIndex | undefined; + // Per container and peer, the sorted disjoint [start, end) counter ranges of + // its operations, flattened. Built on first use, then kept up to date. + #containerOperationSpans: Map> | undefined; #hydratedSnapshotContainers = new Set(); #hydratingSnapshotContainers = new Set(); #snapshotContainerDepths = new Map(); @@ -290,6 +338,9 @@ export class LoroDoc = Record | undefined; #textStyles = new Map([ ["bold", "after"], ["italic", "after"], @@ -745,12 +796,14 @@ export class LoroDoc = Record = Record = Record = Record = Record 0) { const recording = this.#hasEventSubscribers() @@ -924,6 +977,7 @@ export class LoroDoc = Record ({ ...id })); this.#shallowRootStore = rootStore; + this.#shallowRootEntries = undefined; } for (const decoded of ordered) { @@ -941,14 +995,19 @@ export class LoroDoc = Record = Record = Record = Record>(); let mapKeysAtTo = new Map>(); if (useIncrementalTransition || retreatRecords.length === 0) { + const snapshotTransition = this.#prepareSnapshotTransition( + [...retreatRecords, ...forwardRecords], + fromVersion, + toVersion, + true, + ); const recording: EventRecording = { beforeValues: new Map(), eventStates: new Map(), @@ -1123,12 +1213,17 @@ export class LoroDoc = Record = Record = Record = Record = Record ({ ...id })); fork.#shallowRootStore = this.#shallowRootStore; + fork.#shallowRootEntries = this.#shallowRootEntries; fork.#integrateHistory(this.#recordsAtVersion(version), undefined); fork.#rebuildFromHistory(); return fork; @@ -1725,18 +1835,24 @@ export class LoroDoc = Record(); let preparedDiffs = new Map(); this.#checkoutVersion = targetVersion; this.#detached = true; - const changedRecords = [...forwardRecords, ...retreatRecords]; const movableMoveMode = movableMoveTransitionMode( retreatRecords, forwardRecords, this.#movableMovePeers, ); - if (this.#canTransitionRecords(changedRecords, movableMoveMode)) { + if (this.#canTransitionRecords(changedRecords, movableMoveMode, targetVersion)) { const recording = this.#hasEventSubscribers() ? { beforeValues: new Map(), eventStates: new Map() } : undefined; @@ -1767,7 +1883,12 @@ export class LoroDoc = Record = Record(); let preparedDiffs = new Map(); this.#checkoutVersion = undefined; this.#detached = false; - if (wasDetached) { - const changedRecords = [...forwardRecords, ...retreatRecords]; + if (snapshotTransition !== undefined) { const movableMoveMode = movableMoveTransitionMode( retreatRecords, forwardRecords, this.#movableMovePeers, ); - if (this.#canTransitionRecords(changedRecords, movableMoveMode)) { + if (this.#canTransitionRecords(changedRecords, movableMoveMode, latestVersion)) { const recording = this.#hasEventSubscribers() ? { beforeValues: new Map(), eventStates: new Map() } : undefined; @@ -1842,7 +1971,12 @@ export class LoroDoc = Record = Record = Record = Record + left === right || this.#containerKey(left) === this.#containerKey(right), + )); + } const finishEvent = recording === undefined ? undefined @@ -3681,37 +3853,42 @@ export class LoroDoc = Record(); if (movableMoveMode === "replay") { for (const { change } of records) { @@ -3858,12 +4035,35 @@ export class LoroDoc = Record = Record(); + const pins: string[] = []; + const lazy = this.#deferredSnapshotState; + if (lazy === undefined && this.#snapshotSequences.size === 0) { + return { beforeValues, pins }; + } + const touched = new Map(); + // Snapshot-hydrated containers whose snapshot operations the transition + // crosses. Operations applied after hydration are indexed like any others. + const crossesSnapshot = new Set(); + for (const { change } of records) { + for (const operation of change.operations) { + const key = this.#containerKey(operation.container); + if (!touched.has(key)) touched.set(key, operation.container); + const entry = this.#snapshotSequences.get(key); + if ( + entry?.kind === "hydrated" && + operation.counter < (entry.version.get(change.id.peer) ?? 0) + ) { + crossesSnapshot.add(key); + } + } + } + for (const [key, id] of touched) { + if (lazy !== undefined) { + if ( + this.#containers.has(key) || + getLazyStateSnapshotContainer(lazy.store, id) !== undefined + ) { + this.#getOrCreateContainer(id); + } + this.#dirtySnapshotContainers.add(key); + } + const entry = this.#snapshotSequences.get(key); + if (entry === undefined) continue; + const container = this.#containers.get(key) as LoroList | LoroText; + if (entry.kind === "hydrated") { + if (!crossesSnapshot.has(key)) continue; + const snapshot = this.#completeSnapshotSequence(container, key, current); + if (snapshot !== undefined && captureValues) { + beforeValues.set(key, sequenceStateEventValue(container, snapshot)); + } + } else if (entry.kind === "pinned") { + if (captureValues) beforeValues.set(key, containerEventValue(container)); + this.#snapshotSequences.set(key, { + kind: "replayed", + fingerprint: undefined, + snapshot: container._swapState(entry.replayed), + version: current.clone(), + }); + } + const replayed = this.#snapshotSequences.get(key); + if ( + replayed?.kind === "replayed" && + !this.#containerChangedBetween(key, replayed.version, target) + ) { + pins.push(key); + if (captureValues && !beforeValues.has(key)) { + beforeValues.set(key, containerEventValue(container)); + } + } + } + return { beforeValues, pins }; + } + + /** + * Rebuilds a snapshot-hydrated sequence container from its own operations up + * to `version`, starting from its shallow root state in a shallow document. + * loro.js does not model Rust rich-text style anchors in operation positions, + * so a replay can differ from the snapshot state. In that case the replayed + * state stays installed for version transitions, and the snapshot state is + * returned and kept for every version with the same operations on this + * container (see context/loro-js-performance.md). + */ + #completeSnapshotSequence( + container: LoroList | LoroText, + key: string, + version: VersionVector, + ): SequenceContainerState | undefined { + const expected = sequenceFingerprint(container); + const snapshot = container._swapState(); + try { + const root = + this.#shallowRootStore === undefined + ? undefined + : this.#shallowRootEntryIndex().get(key); + if (root !== undefined) this.#hydrateContainerState(container, root.wrapper.state); + for (const record of this.#containerHistoryRecords(key)) { + const { change } = record; + const length = changeLength(change); + const start = Math.max( + 0, + (this.#shallowRootVersion.get(change.id.peer) ?? 0) - change.id.counter, + ); + const end = Math.min( + length, + (version.get(change.id.peer) ?? 0) - change.id.counter, + ); + if (start >= end) continue; + this.#applyChange( + start === 0 && end === length ? change : sliceChange(change, start, end), + record.keys, + undefined, + { key, container }, + ); + } + } catch (error) { + container._swapState(snapshot); + throw error; + } + if (sameSequenceFingerprint(expected, sequenceFingerprint(container))) { + this.#snapshotSequences.delete(key); + return undefined; + } + this.#snapshotSequences.set(key, { + kind: "replayed", + fingerprint: undefined, + snapshot, + version: version.clone(), + }); + return snapshot; + } + + /** + * Reinstates stashed snapshot states once the document reaches a version + * with the same operations on their containers. A stash that was not yet + * compared with a replay is dropped when the replay reproduces it. + */ + #settleSnapshotSequences(target: VersionVector, keys?: readonly string[]): void { + for (const key of keys ?? [...this.#snapshotSequences.keys()]) { + const entry = this.#snapshotSequences.get(key); + if ( + entry?.kind !== "replayed" || + this.#containerChangedBetween(key, entry.version, target) + ) { + continue; + } + const container = this.#containers.get(key) as LoroList | LoroText | undefined; + if (container === undefined) { + this.#snapshotSequences.delete(key); + continue; + } + if ( + entry.fingerprint !== undefined && + sameSequenceFingerprint(entry.fingerprint, sequenceFingerprint(container)) + ) { + this.#snapshotSequences.delete(key); + continue; + } + this.#snapshotSequences.set(key, { + kind: "pinned", + replayed: container._swapState(entry.snapshot), + version: entry.version, + }); + this.#dirtySnapshotContainers.add(key); + } + } + + /** Whether the container has an operation in exactly one of the versions. */ + #containerChangedBetween( + key: string, + left: VersionVector, + right: VersionVector, + ): boolean { + if (left.compare(right) === 0) return false; + this.#materializeDeferredHistory(); + if (this.#containerOperationSpans === undefined) { + this.#containerOperationSpans = new Map(); + for (const record of this.#historyOrder.values()) this.#addOperationSpans(record); + } + const spans = this.#containerOperationSpans.get(key); + if (spans === undefined) return false; + for (const { peer } of [ + ...left._codecEntriesUnsorted(), + ...right._codecEntriesUnsorted(), + ]) { + const leftCounter = left.get(peer) ?? 0; + const rightCounter = right.get(peer) ?? 0; + if ( + spansIntersect( + spans.get(peer), + Math.min(leftCounter, rightCounter), + Math.max(leftCounter, rightCounter), + ) + ) { + return true; + } + } + return false; + } + + /** + * Retained history records that touch the container, in replay order. The + * index is built on first use after each history change. + */ + #containerHistoryRecords(key: string): readonly HistoryRecord[] { + const records = this.#sortedHistory(); + let index = this.#containerHistoryIndex; + if (index === undefined || index.revision !== this.#historyRevision) { + const byContainer = new Map(); + for (const record of records) { + for (const operation of record.change.operations) { + const containerKey = this.#containerKey(operation.container); + let containerRecords = byContainer.get(containerKey); + if (containerRecords === undefined) { + containerRecords = []; + byContainer.set(containerKey, containerRecords); + } + if (containerRecords.at(-1) !== record) containerRecords.push(record); + } + } + index = { revision: this.#historyRevision, records: byContainer }; + this.#containerHistoryIndex = index; + } + return index.records.get(key) ?? []; + } + + #addOperationSpans(record: HistoryRecord): void { + const spans = this.#containerOperationSpans!; + const peer = record.change.id.peer; + for (const operation of record.change.operations) { + const key = this.#containerKey(operation.container); + let byPeer = spans.get(key); + if (byPeer === undefined) { + byPeer = new Map(); + spans.set(key, byPeer); + } + let ranges = byPeer.get(peer); + if (ranges === undefined) { + ranges = []; + byPeer.set(peer, ranges); + } + addSpan(ranges, operation.counter, operation.counter + operation.length); + } + } + + #shallowRootEntryIndex(): Map { + return (this.#shallowRootEntries ??= stateStoreEntriesByKey( + this.#shallowRootStore ?? { kind: "absent" }, + )); } #setHistoryRecord(key: string, record: HistoryRecord, appended: HistoryRecord): void { @@ -4662,6 +5148,7 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record = Record { + return new Map( + store.kind === "sstable" + ? store.containers.map((entry) => [formatContainerId(entry.id), entry] as const) + : [], + ); +} + +/** + * Operations in a run of text inserts on one container, each continuing the + * previous one at the next counter and position, have the same Fugue placement + * as one multi-character insert; Loro's Rust runtime stores such a run as one + * operation. Applying the run as one span avoids a sequence-index update per + * character when replaying history. + */ +function coalescedTextInsert( + operations: readonly DecodedOperation[], + index: number, + sameContainer: (left: CodecContainerId, right: CodecContainerId) => boolean, +): { readonly operation: DecodedOperation; readonly next: number } { + const first = operations[index]!; + if (first.content.type !== "text-insert") return { operation: first, next: index + 1 }; + let next = index + 1; + let counter = first.counter + first.length; + let position = first.content.position + first.length; + const values = [first.content.value]; + for (; next < operations.length; next += 1) { + const operation = operations[next]!; + if ( + operation.content.type !== "text-insert" || + operation.counter !== counter || + operation.content.position !== position || + !sameContainer(first.container, operation.container) + ) { + break; + } + values.push(operation.content.value); + counter += operation.length; + position += operation.length; + } + if (next === index + 1) return { operation: first, next }; + return { + operation: { + container: first.container, + counter: first.counter, + length: counter - first.counter, + content: { + type: "text-insert", + position: first.content.position, + value: values.join(""), + }, + }, + next, + }; +} + +/** + * Visible element ids of a sequence, plus the Text delta when it has styles and + * MovableList values (other element values are fixed by their ids). + */ +function sequenceFingerprint(container: LoroList | LoroText): SequenceFingerprint { + const sequence = container._sequence; + const runs: SequenceIdRun[] = []; + for (const run of sequence.visibleIdRuns(0, sequence.visibleLength)) { + const last = runs.at(-1); + if ( + last !== undefined && + last.start.peer === run.start.peer && + last.start.counter + last.length === run.start.counter + ) { + runs[runs.length - 1] = { start: last.start, length: last.length + run.length }; + } else { + runs.push(run); + } + } + let values: readonly unknown[] | undefined; + if (container instanceof LoroText) { + values = container._styleIndex.isEmpty ? undefined : container.toDelta(); + } else if (container instanceof LoroMovableList) { + values = container._visibleElements().map((element) => element.value); + } + return { runs, values }; +} + +function sameSequenceFingerprint( + left: SequenceFingerprint, + right: SequenceFingerprint, +): boolean { + if ( + left.runs.length !== right.runs.length || + left.runs.some( + (run, index) => + run.length !== right.runs[index]!.length || + !idsEqual(run.start, right.runs[index]!.start), + ) + ) { + return false; + } + if (left.values === undefined || right.values === undefined) { + // Same ids mean the same characters; an unstyled Text only matches a delta + // without attributes. + const values = left.values ?? right.values; + return ( + values === undefined || + values.every((item) => (item as { attributes?: unknown }).attributes === undefined) + ); + } + return eventValuesEqual(left.values, right.values); +} + +function sequenceStateEventValue( + container: LoroList | LoroText, + state: SequenceContainerState, +): unknown { + const installed = container._swapState(state); + try { + return containerEventValue(container); + } finally { + container._swapState(installed); + } +} + +/** Adds [start, end) to flattened sorted disjoint ranges, merging neighbors. */ +function addSpan(ranges: number[], start: number, end: number): void { + if (ranges.length === 0 || ranges.at(-1)! < start) { + ranges.push(start, end); + return; + } + if (ranges.at(-2)! <= start) { + ranges[ranges.length - 1] = Math.max(ranges.at(-1)!, end); + return; + } + let first = 0; + let last = ranges.length / 2; + while (first < last) { + const middle = (first + last) >>> 1; + if (ranges[middle * 2 + 1]! < start) first = middle + 1; + else last = middle; + } + let mergedStart = start; + let mergedEnd = end; + let removeEnd = first; + while (removeEnd < ranges.length / 2 && ranges[removeEnd * 2]! <= mergedEnd) { + mergedStart = Math.min(mergedStart, ranges[removeEnd * 2]!); + mergedEnd = Math.max(mergedEnd, ranges[removeEnd * 2 + 1]!); + removeEnd += 1; + } + ranges.splice(first * 2, (removeEnd - first) * 2, mergedStart, mergedEnd); +} + +/** Whether flattened sorted [start, end) ranges intersect [low, high). */ +function spansIntersect( + spans: readonly number[] | undefined, + low: number, + high: number, +): boolean { + if (spans === undefined || low >= high) return false; + let first = 0; + let last = spans.length / 2; + while (first < last) { + const middle = (first + last) >>> 1; + if (spans[middle * 2 + 1]! <= low) first = middle + 1; + else last = middle; + } + return first < spans.length / 2 && spans[first * 2]! < high; +} + function mergeStateSnapshotStores( root: StateSnapshotStore, overlay: StateSnapshotStore, + rootEntries = stateStoreEntriesByKey(root), ): StateSnapshotStore { if (root.kind !== "sstable") return overlay; if (overlay.kind !== "sstable") { return { kind: "sstable", frontiers: undefined, containers: root.containers }; } - const containers = new Map( - root.containers.map((entry) => [formatContainerId(entry.id), entry]), - ); + const containers = new Map(rootEntries); for (const entry of overlay.containers) { containers.set(formatContainerId(entry.id), entry); } diff --git a/loro-js/tests/fixtures/rust/styled-text.blob b/loro-js/tests/fixtures/rust/styled-text.blob new file mode 100644 index 0000000000000000000000000000000000000000..67412c643d6b4720f977b74ab4b762d046cbb51b GIT binary patch literal 404 zcmYk2u}T9$5QhJMc4sfSy95h`AU;5>EPMj3g@9Gs1TO|WJV` { expect(valuesAt(loaded, versions)).toEqual(expected); }); }); + +/** + * `styled-text.blob` is a Rust snapshot (peer 1, `bold` expands after, `keep` + * expands both ways). Versions, one commit each: + * 0. `both`: insert "a", mark it `keep`. + * 1. `both`: delete "a", leaving an empty both-expand pair. + * 2. `both`: insert "b" at 0; Rust places it inside the pair, so it is kept. + * 3. `after`: insert "hello". + * 4. `after`: mark 1..3 `bold`. + * 5. `after`: insert "X" at 4, after the end anchor of the mark. + * 6. `after`: delete 0..1. + * The expected file holds the Rust `toDelta()` of both texts at each version. + */ +interface StyledTextVersion { + readonly frontiers: Frontiers; + readonly both: Delta[]; + readonly after: Delta[]; +} + +const styledText = (): { bytes: Uint8Array; versions: StyledTextVersion[] } => ({ + bytes: new Uint8Array( + readFileSync(new URL("./fixtures/rust/styled-text.blob", import.meta.url)), + ), + versions: JSON.parse( + readFileSync( + new URL("./fixtures/rust/styled-text.expected.json", import.meta.url), + "utf8", + ), + ) as StyledTextVersion[], +}); + +// loro.js interprets Text operation positions without Rust style anchors, so +// replaying version 5 places "X" after "o". Every other state matches Rust. +const REPLAY_POSITION_GAP = 5; +const REPLAYED_AT_GAP: Delta[] = [ + { insert: "h" }, + { insert: "el", attributes: { bold: true } }, + { insert: "loX" }, +]; + +describe("checkout after a Rust rich-text snapshot import", () => { + test("keeps snapshot styles and text that a history replay cannot reproduce", () => { + const { bytes, versions } = styledText(); + const latest = versions.at(-1)!; + for (const order of [ + [0, 1, 2, 3, 4, 5, 6], + [6, 5, 4, 3, 2, 1, 0], + [1, 6, 2, 5, 0, 3, 6, 4], + ]) { + const doc = new LoroDoc(); + doc.import(bytes); + const mirror = new LoroDoc(); + mirror.configTextStyle({ keep: { expand: "both" }, bold: { expand: "after" } }); + mirror.getText("both").applyDelta(doc.getText("both").toDelta()); + mirror.getText("after").applyDelta(doc.getText("after").toDelta()); + doc.subscribe((batch: LoroEventBatch) => { + mirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + }); + const check = (expected: StyledTextVersion, index: number): void => { + const both = doc.getText("both").toDelta(); + const after = doc.getText("after").toDelta(); + expect(both).toEqual(expected.both); + expect(after).toEqual( + index === REPLAY_POSITION_GAP ? REPLAYED_AT_GAP : expected.after, + ); + expect(mirror.getText("both").toDelta()).toEqual(both); + expect(mirror.getText("after").toDelta()).toEqual(after); + }; + for (const index of order) { + doc.checkout(versions[index]!.frontiers); + check(versions[index]!, index); + } + doc.checkoutToLatest(); + check(latest, versions.length - 1); + + const again = new LoroDoc(); + again.import(doc.export({ mode: "snapshot" })); + expect(again.getText("both").toDelta()).toEqual(latest.both); + expect(again.getText("after").toDelta()).toEqual(latest.after); + } + }); + + test("returns to the snapshot state after edits and imports elsewhere", () => { + const { bytes, versions } = styledText(); + const latest = versions.at(-1)!; + const doc = new LoroDoc(); + doc.import(bytes); + const remote = new LoroDoc(); + remote.import(bytes); + remote.setPeerId(2); + remote.getMap("other").set("remote", 1); + remote.commit(); + + doc.checkout(versions[4]!.frontiers); + doc.checkoutToLatest(); + doc.getMap("other").set("local", 1); + doc.commit(); + doc.checkout(versions[2]!.frontiers); + doc.import(remote.export({ mode: "update", from: doc.oplogVersion() })); + doc.checkoutToLatest(); + expect(doc.getText("both").toDelta()).toEqual(latest.both); + expect(doc.getText("after").toDelta()).toEqual(latest.after); + expect(doc.getMap("other").toJSON()).toEqual({ local: 1, remote: 1 }); + + doc.getText("after").insert(0, "A"); + doc.commit(); + expect(doc.getText("after").toString()).toBe("AellXo"); + doc.checkout(versions[6]!.frontiers); + expect(doc.getText("after").toDelta()).toEqual(latest.after); + }); + + test("diffs a Rust snapshot like a full-history document", () => { + const { bytes, versions } = styledText(); + const doc = new LoroDoc(); + doc.import(bytes); + const diff = doc.diff(versions[6]!.frontiers, versions[1]!.frontiers, false); + expect(doc.getText("both").toDelta()).toEqual(versions[6]!.both); + const mirror = new LoroDoc(); + mirror.configTextStyle({ keep: { expand: "both" }, bold: { expand: "after" } }); + mirror.getText("both").applyDelta(versions[6]!.both); + mirror.getText("after").applyDelta(versions[6]!.after); + mirror.applyDiff(diff); + expect(mirror.getText("both").toDelta()).toEqual(versions[1]!.both); + expect(mirror.getText("after").toDelta()).toEqual(versions[1]!.after); + }); +}); + +describe("checkout events after a lazy snapshot import", () => { + test("reports an unread nested container change as its delta", () => { + const build = (): { doc: LoroDoc; before: Frontiers } => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const parent = doc.getMap("root").setContainer("parent", new LoroMap()); + const child = parent.setContainer("child", new LoroText()); + child.insert(0, "abc"); + doc.commit(); + const before = doc.frontiers(); + child.insert(0, "X"); + doc.commit(); + return { doc, before }; + }; + const { doc, before } = build(); + const bytes = doc.export({ mode: "snapshot" }); + const diffs = (read: boolean): unknown[] => { + const loaded = new LoroDoc(); + loaded.import(bytes); + if (read) loaded.toJSON(); + const events: unknown[] = []; + loaded.subscribe((batch) => { + for (const event of batch.events) events.push(event.diff); + }); + loaded.checkout(before); + expect(loaded.toJSON()).toEqual({ root: { parent: { child: "abc" } } }); + return events; + }; + expect(diffs(false)).toEqual([{ type: "text", diff: [{ delete: 1 }] }]); + expect(diffs(true)).toEqual(diffs(false)); + }); + + test("exports unread nested containers after history is loaded", () => { + const { doc, versions } = buildHistory(); + for (const load of [ + (loaded: LoroDoc) => loaded.getAllChanges(), + (loaded: LoroDoc) => { + loaded.checkout(versions[0]!); + loaded.checkoutToLatest(); + }, + ]) { + const loaded = new LoroDoc(); + loaded.import(doc.export({ mode: "snapshot" })); + load(loaded); + const again = new LoroDoc(); + again.import(loaded.export({ mode: "snapshot" })); + expect(again.toJSON()).toEqual(doc.toJSON()); + } + }); +}); From f302370bb83236f7c4cab8a76df85acd60591400 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:08:15 +0800 Subject: [PATCH 03/15] fix(loro.js): keep snapshot state through a shallow export Shallow export rebuilds the root and latest states by full replay, which replaced snapshot state that the replay does not reproduce (Rust rich text) in both the export and the live document. The rebuilds now pass the version they leave, so snapshot states are stashed and reinstated, and unread lazy sequences are hydrated first. The import builds the root entry index only when it merges a latest state. Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 4 ++-- loro-js/src/runtime/document.ts | 31 +++++++++++++++++++++---- loro-js/tests/snapshot-checkout.test.ts | 18 ++++++++++++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index c001db273..b60278196 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -153,8 +153,8 @@ JavaScript constant factor. transitions and the snapshot state is reinstated at every version with the same operations on that container (`#settleSnapshotSequences`), so the snapshot version and exports stay exact. A full `#rebuildFromHistory`, still - used when a transition cannot be incremental, stashes installed snapshot states - the same way. + used when a transition cannot be incremental and by shallow export, stashes + snapshot states the same way; it hydrates unread sequence containers first. - First checkout after importing a 262,144-operation single-peer Text snapshot: about 60–70 ms on a loaded Apple M5 Pro, versus about 150 ms for the earlier whole-document replay; the replay of that one container dominates. A doc with diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index fd41ec00e..5b5966eb6 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -9,7 +9,11 @@ import { type DecodedOperationContent, } from "../codec/change-block-codec"; import type { ChangeLoroValue, ChangeValue } from "../codec/change-value"; -import { containerTypeFromRawByte, containerTypeToRawByte } from "../codec/container-id"; +import { + containerTypeFromRawByte, + containerTypeToRawByte, + decodeContainerId, +} from "../codec/container-id"; import { decodeDocument, decodeFastSnapshotBody, @@ -797,7 +801,9 @@ export class LoroDoc = Record = Record = Record { expect(doc.getText("after").toDelta()).toEqual(latest.after); }); + test("keeps the snapshot state through a shallow export", () => { + const { bytes, versions } = styledText(); + const latest = versions.at(-1)!; + for (const read of [false, true]) { + const doc = new LoroDoc(); + doc.import(bytes); + if (read) doc.toJSON(); + const shallow = new LoroDoc(); + shallow.import( + doc.export({ mode: "shallow-snapshot", frontiers: versions[3]!.frontiers }), + ); + for (const replica of [doc, shallow]) { + expect(replica.getText("both").toDelta()).toEqual(latest.both); + expect(replica.getText("after").toDelta()).toEqual(latest.after); + } + } + }); + test("diffs a Rust snapshot like a full-history document", () => { const { bytes, versions } = styledText(); const doc = new LoroDoc(); From 5c183052659595f42ff9307c7c76736d5a8d355f Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:12:46 +0800 Subject: [PATCH 04/15] fix(loro.js): rebuild an unread nested sequence that a checkout crosses The check for crossing snapshot operations ran before lazily encoded containers were hydrated, so an unread nested Text or List skipped its history rebuild and the checkout fell back to replaying the whole document. Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/src/runtime/document.ts | 37 ++++++++++++++++--------- loro-js/tests/snapshot-checkout.test.ts | 25 ++++++++++++++++- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 5b5966eb6..8f95f58b4 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -4924,24 +4924,26 @@ export class LoroDoc = Record(); - // Snapshot-hydrated containers whose snapshot operations the transition - // crosses. Operations applied after hydration are indexed like any others. - const crossesSnapshot = new Set(); + // The first counter per peer of the operations on each touched container. + const touched = new Map< + string, + { readonly id: CodecContainerId; readonly firstCounters: Map } + >(); for (const { change } of records) { for (const operation of change.operations) { const key = this.#containerKey(operation.container); - if (!touched.has(key)) touched.set(key, operation.container); - const entry = this.#snapshotSequences.get(key); - if ( - entry?.kind === "hydrated" && - operation.counter < (entry.version.get(change.id.peer) ?? 0) - ) { - crossesSnapshot.add(key); + let container = touched.get(key); + if (container === undefined) { + container = { id: operation.container, firstCounters: new Map() }; + touched.set(key, container); + } + const first = container.firstCounters.get(change.id.peer); + if (first === undefined || operation.counter < first) { + container.firstCounters.set(change.id.peer, operation.counter); } } } - for (const [key, id] of touched) { + for (const [key, { id, firstCounters }] of touched) { if (lazy !== undefined) { if ( this.#containers.has(key) || @@ -4955,7 +4957,16 @@ export class LoroDoc = Record counter >= (version.get(peer) ?? 0), + ) + ) { + continue; + } const snapshot = this.#completeSnapshotSequence(container, key, current); if (snapshot !== undefined && captureValues) { beforeValues.set(key, sequenceStateEventValue(container, snapshot)); diff --git a/loro-js/tests/snapshot-checkout.test.ts b/loro-js/tests/snapshot-checkout.test.ts index 9f290a1e2..bbdc84cee 100644 --- a/loro-js/tests/snapshot-checkout.test.ts +++ b/loro-js/tests/snapshot-checkout.test.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { LoroDoc, @@ -278,6 +278,29 @@ describe("checkout events after a lazy snapshot import", () => { expect(diffs(true)).toEqual(diffs(false)); }); + test("rebuilds only the unread nested sequence that a checkout crosses", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getMap("root").setContainer("text", new LoroText()); + text.insert(0, "abc"); + doc.getMap("other").set("k", 1); + doc.commit(); + const before = doc.frontiers(); + text.delete(1, 1); + doc.commit(); + + const loaded = new LoroDoc(); + loaded.import(doc.export({ mode: "snapshot" })); + const resets = [loaded.getMap("root"), loaded.getMap("other")].map((container) => + vi.spyOn(container, "_reset"), + ); + loaded.checkout(before); + expect(loaded.toJSON()).toEqual({ root: { text: "abc" }, other: { k: 1 } }); + loaded.checkoutToLatest(); + expect(loaded.toJSON()).toEqual({ root: { text: "ac" }, other: { k: 1 } }); + expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); + }); + test("exports unread nested containers after history is loaded", () => { const { doc, versions } = buildHistory(); for (const load of [ From a9c329c4a5e3aea56861d2af2b902ce230e696ea Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:19:14 +0800 Subject: [PATCH 05/15] docs(loro.js): record final snapshot checkout measurements Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index b60278196..7fe39bf85 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -155,13 +155,15 @@ JavaScript constant factor. snapshot version and exports stay exact. A full `#rebuildFromHistory`, still used when a transition cannot be incremental and by shallow export, stashes snapshot states the same way; it hydrates unread sequence containers first. -- First checkout after importing a 262,144-operation single-peer Text snapshot: - about 60–70 ms on a loaded Apple M5 Pro, versus about 150 ms for the earlier - whole-document replay; the replay of that one container dominates. A doc with - 32,768 child Maps that retreats one of them needs no replay: 63 ms versus - 170 ms, with the same 234 MiB peak RSS as before the fix. A shallow document - whose Map or Tree winner at the target is a trimmed root-time write replays - from the shallow root state. +- First checkout after importing a 262,144-operation single-peer Text snapshot + takes about 57 ms (medians of 5 alternating runs on a loaded Apple M5 Pro), + versus about 148 ms for the earlier whole-document replay; 65,536 operations + take 25 versus 43 ms, and a subscriber adds nothing measurable (earlier 192 + ms at 262,144). The replay of that one container dominates. Later checkouts + stay around 0.1–0.4 ms. A doc with 32,768 child Maps that retreats one of them + needs no replay: 57–61 ms versus 152 ms, with the same 234 MiB peak RSS as + before the fix (earlier 284 MiB). The coalesced inserts also make importing + the B4 trace as one update about 30% faster (about 195 versus 275 ms). When an element's deleted flag, tree parent/position, or map visibility changes, mutate it through its owning index helper. Direct mutation leaves subtree or From 7f66407580b99e2275d7cf6824f3de3e0d652466 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:22:09 +0800 Subject: [PATCH 06/15] test(loro.js): export the latest Rust rich-text state from a detached checkout Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/tests/snapshot-checkout.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/loro-js/tests/snapshot-checkout.test.ts b/loro-js/tests/snapshot-checkout.test.ts index bbdc84cee..a9049d4ac 100644 --- a/loro-js/tests/snapshot-checkout.test.ts +++ b/loro-js/tests/snapshot-checkout.test.ts @@ -212,6 +212,21 @@ describe("checkout after a Rust rich-text snapshot import", () => { expect(doc.getText("after").toDelta()).toEqual(latest.after); }); + test("exports the latest state from every detached checkout", () => { + const { bytes, versions } = styledText(); + const latest = versions.at(-1)!; + const doc = new LoroDoc(); + doc.import(bytes); + for (const version of [...versions].reverse()) { + doc.checkout(version.frontiers); + const again = new LoroDoc(); + again.import(doc.export({ mode: "snapshot" })); + expect(again.getText("both").toDelta()).toEqual(latest.both); + expect(again.getText("after").toDelta()).toEqual(latest.after); + expect(doc.getText("both").toDelta()).toEqual(version.both); + } + }); + test("keeps the snapshot state through a shallow export", () => { const { bytes, versions } = styledText(); const latest = versions.at(-1)!; From e56d37afc8cd1b56cc5b5f60285079fe1ae16b2d Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:22:19 +0800 Subject: [PATCH 07/15] docs(loro.js): note the detached snapshot export transition Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 7fe39bf85..1f01a7487 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -131,8 +131,9 @@ JavaScript constant factor. shallow root) has no tombstones, deletion index, or style, value, and move history for the snapshot's operations. `LoroDoc.#snapshotSequences` records each such container with its snapshot version. Before `checkout`, - `checkoutToLatest`, or `diff` transitions, - `#prepareSnapshotTransition` looks only at the containers the transition + `checkoutToLatest`, `diff`, or a detached snapshot export + (`#encodeLatestState`) transitions, `#prepareSnapshotTransition` looks only + at the containers the transition touches: it hydrates lazily encoded ones (an untouched lazy container still holds its latest state, which is its state at the current version) and, when the transition crosses a snapshot operation, rebuilds that one container from From 5e3244bbc040e844753b8d959a0074e49f4c0e41 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 04:36:32 +0800 Subject: [PATCH 08/15] fix(loro.js): never transition a replay that differs from the snapshot state A Text or List whose replay does not reproduce its snapshot state (Rust rich text, since loro.js does not count style anchors in positions) no longer installs the replay and pins the snapshot state back. Operations applied while the replay was installed, such as an update imported while detached, changed the latest state and exports. Such a container now keeps its snapshot state as the authority: every transition that touches it rebuilds it from that state plus later operations, so older versions are approximate but the latest state and exports never come from a replay. Full rebuilds and forkAt do the same. Transitions also deduplicate sequence elements by id. A packed Text span returns a new wrapper per lookup, so two concurrent deletes of one character deleted it twice and a subscribed checkout threw after half-updating the document. A checkout that throws now restores its previous version and state. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-snapshot-checkout.md | 2 +- context/loro-js-performance.md | 24 +- loro-js/src/runtime/document.ts | 767 +++++++++++------------ loro-js/tests/snapshot-checkout.test.ts | 101 ++- loro-js/tests/version-transition.test.ts | 75 +++ 5 files changed, 530 insertions(+), 439 deletions(-) diff --git a/.changeset/loro-js-snapshot-checkout.md b/.changeset/loro-js-snapshot-checkout.md index dfd53d8bd..cd691b1a3 100644 --- a/.changeset/loro-js-snapshot-checkout.md +++ b/.changeset/loro-js-snapshot-checkout.md @@ -2,4 +2,4 @@ "loro.js": patch --- -Fix checkout and `diff` on a `loro.js` document imported from a snapshot. Checking out an earlier version could return the latest value of a child container that had not been read yet, restore nothing for elements deleted before the snapshot, skip a delete when moving forward again, or throw `duplicate sequence id`. A transition now rebuilds only the Text and List containers it touches from their own history, once. Checkout events for containers that had not been read are now correct, and a snapshot exported after history was loaded no longer drops containers that had not been read. Rust-created rich text keeps its snapshot styles and text at the snapshot version. +Fix checkout and `diff` on a `loro.js` document imported from a snapshot. Checking out an earlier version could return the latest value of a child container that had not been read yet, restore nothing for elements deleted before the snapshot, skip a delete when moving forward again, or throw `duplicate sequence id`. A transition now rebuilds only the Text and List containers it touches from their own history, once. Rust-created rich text whose history `loro.js` cannot replay keeps its snapshot state, so its latest state and exports stay exact. Checkout events for containers that had not been read are now correct, and a snapshot exported after history was loaded no longer drops containers that had not been read. A character deleted concurrently by two peers is no longer deleted twice during checkout, and a checkout that throws leaves the document unchanged. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 1f01a7487..376368ffb 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -150,12 +150,24 @@ JavaScript constant factor. positions, so a replay of Rust-created styled text can differ from its snapshot state (for example, an insert right after a mark's end anchor). The completion therefore compares the replay with the snapshot state (visible ids, - plus the delta when styled). When they differ, the replayed state serves - transitions and the snapshot state is reinstated at every version with the - same operations on that container (`#settleSnapshotSequences`), so the - snapshot version and exports stay exact. A full `#rebuildFromHistory`, still - used when a transition cannot be incremental and by shallow export, stashes - snapshot states the same way; it hydrates unread sequence containers first. + plus the delta when styled). When they differ, the replay is discarded and the + container becomes `unreplayable`: it keeps its snapshot state, encoded once, + and every transition that touches it rebuilds it from that state + (`#rebuildFromSnapshotState`, O(container size + its operations)) instead of + transitioning a replay. The rebuild hides the state's inserted elements and + styles that the target excludes and applies later operations the target + includes. So the latest state, imports, and exports always equal the + snapshot state plus later operations, as without history; an older version + cannot restore text deleted before the snapshot. A full `#rebuildFromHistory` + (the non-incremental fallback, shallow export, `forkAt`) first checks the + snapshot-hydrated styled Text and MovableList containers, including lazily + encoded ones, and then rebuilds unreplayable containers the same way. Only the + root state of a shallow export uses the replay, since the snapshot state is + later than the root. +- Transitions deduplicate sequence elements by id (`SequenceElementSet`): a + packed Text span returns a new wrapper per lookup, so two concurrent deletes + of one character used to delete it twice. A checkout that throws restores its + previous version and state (`#transitionTo`). - First checkout after importing a 262,144-operation single-peer Text snapshot takes about 57 ms (medians of 5 alternating runs on a loaded Apple M5 Pro), versus about 148 ms for the earlier whole-document replay; 65,536 operations diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index e923f7389..4bd047952 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -28,6 +28,7 @@ import { import { decodeChangeBlockKey, encodeChangeBlockKey } from "../codec/id"; import { decodeSstable, encodeSstable, type SstableEntry } from "../codec/sstable"; import { + decodeContainerStateWrapper, decodeLazyStateSnapshotStore, decodeStateSnapshotStore, encodeStateSnapshotStore, @@ -69,7 +70,6 @@ import { type CausalVersion, type LastWriter, type RuntimeValue, - type SequenceContainerState, type SequenceElement, type SequenceMoveMeta, type SequenceValueMeta, @@ -203,27 +203,18 @@ type SnapshotSequenceState = // The snapshot state of `version` is installed. It has no history for the // operations of `version`; operations applied since then are indexed. | { readonly kind: "hydrated"; readonly version: VersionVector } - // A replayed state is installed. `snapshot` is the container's state at - // every version whose operations on it match `version`. `fingerprint` is set - // until a replay has been compared with it. + // loro.js cannot replay this container's history to its snapshot state + // (#completeSnapshotSequence). `state` is that state at `version`; every + // transition rebuilds the container from it (#rebuildFromSnapshotState). | { - readonly kind: "replayed"; - readonly snapshot: SequenceContainerState; - readonly version: VersionVector; - readonly fingerprint: SequenceFingerprint | undefined; - } - // The snapshot state is installed again; `replayed` is the replayed state for - // the same operations as `version`, valid until an operation is applied to - // the container. - | { - readonly kind: "pinned"; - readonly replayed: SequenceContainerState; + readonly kind: "unreplayable"; + readonly state: ContainerStateSnapshot; readonly version: VersionVector; }; interface SnapshotTransition { readonly beforeValues: Map; - readonly pins: readonly string[]; + readonly rebuilds: ReadonlySet; } interface SequenceFingerprint { @@ -309,9 +300,6 @@ export class LoroDoc = Record> | undefined; #hydratedSnapshotContainers = new Set(); #hydratingSnapshotContainers = new Set(); #snapshotContainerDepths = new Map(); @@ -1127,10 +1115,36 @@ export class LoroDoc = Record = Record = Record(); let calculated = new Map(); let mapKeysAtFrom = new Map>(); let mapKeysAtTo = new Map>(); if (useIncrementalTransition || retreatRecords.length === 0) { - const snapshotTransition = this.#prepareSnapshotTransition( - [...retreatRecords, ...forwardRecords], - fromVersion, - toVersion, - true, - ); const recording: EventRecording = { beforeValues: new Map(), eventStates: new Map(), }; + const rebuiltBefore = new Map(); + for (const key of rebuilds) { + const container = this.#containers.get(key); + if (container !== undefined) + rebuiltBefore.set(key, containerEventValue(container)); + } if (useIncrementalTransition) { this.#applyVersionTransition( retreatRecords, @@ -1219,10 +1222,10 @@ export class LoroDoc = Record = Record = Record = Record = Record(); fork.#recordTimestamp = this.#recordTimestamp; fork.#changeMergeInterval = this.#changeMergeInterval; @@ -1821,6 +1813,12 @@ export class LoroDoc = Record = Record(); - let preparedDiffs = new Map(); - this.#checkoutVersion = targetVersion; - this.#detached = true; - const movableMoveMode = movableMoveTransitionMode( retreatRecords, forwardRecords, - this.#movableMovePeers, + changed, + () => { + this.#checkoutVersion = targetVersion; + this.#detached = true; + }, ); - if (this.#canTransitionRecords(changedRecords, movableMoveMode, targetVersion)) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyVersionTransition( - retreatRecords, - forwardRecords, - targetVersion, - recording, - movableMoveMode, - ); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } else if ( - retreatRecords.length === 0 && - !hasMaterializedSequenceInsertions(forwardRecords, this.#containers) - ) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyRecords(forwardRecords, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } else { - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureContainerEventValues(changed); - } - this.#rebuildFromHistory(targetVersion, currentVersion); - } - this.#settleSnapshotSequences(targetVersion, snapshotTransition.pins); - for (const [key, value] of snapshotTransition.beforeValues) { - beforeValues.set(key, value); - preparedDiffs.delete(key); - } this.#emit( "checkout", undefined, @@ -1926,75 +1881,106 @@ export class LoroDoc = Record { + this.#checkoutVersion = undefined; + this.#detached = false; + }; + if (!wasDetached) { + attach(); + return; + } + const { beforeValues, preparedDiffs } = this.#transitionTo( + currentVersion, + latestVersion, + retreatRecords, + forwardRecords, + changed, + attach, + ); + this.#emit( + "checkout", + undefined, + before, + this.#frontiersCodec(), + changed, + beforeValues, + preparedDiffs, + ); + if (this.#detachedEditing) this.#renewPeerId(); + } + + /** + * Moves the state from `current` to `target` for a checkout and returns the + * values and diffs of its event. `setVersion` installs the new checkout + * version once the containers are prepared. If the transition throws, the + * checkout version and the state at `current` are restored before the error + * is rethrown, so a failed checkout leaves no partial update. + */ + #transitionTo( + current: VersionVector, + target: VersionVector, + retreatRecords: readonly HistoryRecord[], + forwardRecords: readonly HistoryRecord[], + changed: ReadonlySet, + setVersion: () => void, + ): { beforeValues: Map; preparedDiffs: Map } { + const subscribed = this.#hasEventSubscribers(); + const prepared = this.#prepareSnapshotTransition( + [...forwardRecords, ...retreatRecords], + current, + subscribed, + ); + const retreat = this.#withoutContainers(retreatRecords, prepared.rebuilds); + const forward = this.#withoutContainers(forwardRecords, prepared.rebuilds); + const previousCheckoutVersion = this.#checkoutVersion; + const previousDetached = this.#detached; + setVersion(); let beforeValues = new Map(); let preparedDiffs = new Map(); - this.#checkoutVersion = undefined; - this.#detached = false; - if (snapshotTransition !== undefined) { + try { const movableMoveMode = movableMoveTransitionMode( - retreatRecords, - forwardRecords, + retreat, + forward, this.#movableMovePeers, ); - if (this.#canTransitionRecords(changedRecords, movableMoveMode, latestVersion)) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; + const recording: EventRecording | undefined = subscribed + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + if (this.#canTransitionRecords([...forward, ...retreat], movableMoveMode, target)) { this.#applyVersionTransition( - retreatRecords, - forwardRecords, - latestVersion, + retreat, + forward, + target, recording, movableMoveMode, ); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } } else if ( - retreatRecords.length === 0 && - !hasMaterializedSequenceInsertions(forwardRecords, this.#containers) + retreat.length === 0 && + !hasMaterializedSequenceInsertions(forward, this.#containers) ) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyRecords(forwardRecords, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } + this.#applyRecords(forward, recording); } else { - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureContainerEventValues(changed); - } - this.#rebuildFromHistory(undefined, currentVersion); + if (subscribed) beforeValues = this.#captureContainerEventValues(changed); + this.#rebuildFromHistory(target, current); + return { beforeValues, preparedDiffs }; } - this.#settleSnapshotSequences(latestVersion, snapshotTransition.pins); - for (const [key, value] of snapshotTransition.beforeValues) { - beforeValues.set(key, value); - preparedDiffs.delete(key); + this.#rebuildFromSnapshotStates(prepared.rebuilds, target); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); } - this.#emit( - "checkout", - undefined, - before, - this.#frontiersCodec(), - changed, - beforeValues, - preparedDiffs, - ); - if (this.#detachedEditing) this.#renewPeerId(); + } catch (error) { + this.#checkoutVersion = previousCheckoutVersion; + this.#detached = previousDetached; + this.#rebuildFromHistory(current, current); + throw error; } + for (const [key, value] of prepared.beforeValues) { + beforeValues.set(key, value); + preparedDiffs.delete(key); + } + return { beforeValues, preparedDiffs }; } #renewPeerId(): void { @@ -3451,16 +3437,6 @@ export class LoroDoc = Record = Record>(); + const sequences = new Map(); const bulkSequenceRemovals = new Map(); const bulkSequenceRestorations = new Map(); const textAttributeRuns = new Map>(); const textStyleContainers = new Set(); - const movableValues = new Map>(); + const movableValues = new Map(); const mapKeys = new Map>(); const treeSubjects = new Map>(); const counters = new Map(); - const sequenceElements = (container: LoroList | LoroText): Set => { + const sequenceElements = (container: LoroList | LoroText): SequenceElementSet => { let elements = sequences.get(container); if (elements === undefined) { - elements = new Set(); + elements = new SequenceElementSet(); sequences.set(container, elements); } return elements; @@ -4255,7 +4231,7 @@ export class LoroDoc = Record = Record = Record = Record(); - const pins: string[] = []; + const rebuilds = new Set(); const lazy = this.#deferredSnapshotState; if (lazy === undefined && this.#snapshotSequences.size === 0) { - return { beforeValues, pins }; + return { beforeValues, rebuilds }; } // The first counter per peer of the operations on each touched container. const touched = new Map< @@ -4963,51 +4955,32 @@ export class LoroDoc = Record counter >= (version.get(peer) ?? 0), - ) + ) || + !this.#completeSnapshotSequence(container, key, current) ) { continue; } - const snapshot = this.#completeSnapshotSequence(container, key, current); - if (snapshot !== undefined && captureValues) { - beforeValues.set(key, sequenceStateEventValue(container, snapshot)); - } - } else if (entry.kind === "pinned") { - if (captureValues) beforeValues.set(key, containerEventValue(container)); - this.#snapshotSequences.set(key, { - kind: "replayed", - fingerprint: undefined, - snapshot: container._swapState(entry.replayed), - version: current.clone(), - }); - } - const replayed = this.#snapshotSequences.get(key); - if ( - replayed?.kind === "replayed" && - !this.#containerChangedBetween(key, replayed.version, target) - ) { - pins.push(key); - if (captureValues && !beforeValues.has(key)) { - beforeValues.set(key, containerEventValue(container)); - } } + rebuilds.add(key); + if (captureValues) beforeValues.set(key, containerEventValue(container)); } - return { beforeValues, pins }; + return { beforeValues, rebuilds }; } /** * Rebuilds a snapshot-hydrated sequence container from its own operations up - * to `version`, starting from its shallow root state in a shallow document. - * loro.js does not model Rust rich-text style anchors in operation positions, - * so a replay can differ from the snapshot state. In that case the replayed - * state stays installed for version transitions, and the snapshot state is - * returned and kept for every version with the same operations on this - * container (see context/loro-js-performance.md). + * to `version`, starting from its shallow root state in a shallow document, + * and returns whether the replay differed from the snapshot state. loro.js + * does not count Rust rich-text style anchors in operation positions, so a + * replay of Rust rich text can differ. Such a container keeps its snapshot + * state and becomes `unreplayable` (see #rebuildFromSnapshotState and + * context/loro-js-performance.md). */ #completeSnapshotSequence( container: LoroList | LoroText, key: string, version: VersionVector, - ): SequenceContainerState | undefined { + ): boolean { const expected = sequenceFingerprint(container); const snapshot = container._swapState(); try { @@ -5041,83 +5014,112 @@ export class LoroDoc = Record