From b58a81031f44941291c89f9a3945b3344fc8b317 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:42:48 +0800 Subject: [PATCH 01/13] fix(loro.js): converge plain-text edits with Rust A seeded differential suite now runs the same Text edits, syncs, checkouts, snapshots, cursors, undo, and revertTo in loro.js and loro-crdt (nodejs WASM) and compares every step; it skips when no WASM build exists. It found these divergences, fixed here: Fugue placement after a sibling subtree, deletes applied by recorded ID instead of position, doubly deleted text restored twice by checkout events, and cursor semantics. Ops are also written in Rust's shape (merged inserts, last delete run first). See context/loro-js-rust-differential.md and context/loro-js-performance.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-text-interop.md | 11 + AGENTS.md | 2 + context/CONTEXT-GAPS.md | 1 - context/loro-js-performance.md | 13 +- context/loro-js-rust-differential.md | 79 ++ loro-js/AGENTS.md | 30 + loro-js/CLAUDE.md | 1 + loro-js/README.md | 3 + loro-js/src/runtime/containers.ts | 119 ++- loro-js/src/runtime/document.ts | 142 ++- loro-js/src/runtime/sequence-index.ts | 47 + .../fixtures/rust/text-delete-positions.blob | Bin 0 -> 108 bytes .../fixtures/rust/text-delete-positions.json | 49 + loro-js/tests/richtext-differential.test.ts | 22 + .../tests/support/richtext-differential.ts | 981 ++++++++++++++++++ loro-js/tests/support/rust-reference.ts | 27 + loro-js/tests/text-interop.test.ts | 170 +++ package.json | 3 +- 18 files changed, 1661 insertions(+), 39 deletions(-) create mode 100644 .changeset/loro-js-text-interop.md create mode 100644 context/loro-js-rust-differential.md create mode 100644 loro-js/AGENTS.md create mode 120000 loro-js/CLAUDE.md create mode 100644 loro-js/tests/fixtures/rust/text-delete-positions.blob create mode 100644 loro-js/tests/fixtures/rust/text-delete-positions.json create mode 100644 loro-js/tests/richtext-differential.test.ts create mode 100644 loro-js/tests/support/richtext-differential.ts create mode 100644 loro-js/tests/support/rust-reference.ts create mode 100644 loro-js/tests/text-interop.test.ts diff --git a/.changeset/loro-js-text-interop.md b/.changeset/loro-js-text-interop.md new file mode 100644 index 000000000..ec51a6bc3 --- /dev/null +++ b/.changeset/loro-js-text-interop.md @@ -0,0 +1,11 @@ +--- +"loro.js": patch +--- + +Text edits now converge with Rust (`loro-crdt`) in cases a randomized comparison with the Rust WASM build found: + +- A concurrent insert after a sibling's subtree was placed after later, unrelated concurrent text. Replicas that merged such edits in different orders could end with different text. +- An imported delete is now applied by its position, as Rust does, instead of by its recorded `start_id`. Rust's WASM build records a wrong `start_id` when the deleted text spans astral characters (for example, emoji) inserted in separate edits, so loro.js deleted different characters than Rust. +- `checkout`, `diff`, and `revertTo` no longer restore a character twice when two concurrent deletes removed it. +- Text cursors follow Rust: a cursor at the end or in empty text has no target, `getCursorPos` returns the target's own offset for every side, and a deleted target reports the length of the text before it with side `-1`. The encoded cursor matches Rust's bytes. +- Consecutive inserts in one transaction are merged into one op, and a delete that spans several ID runs writes the last run first, as Rust does. diff --git a/AGENTS.md b/AGENTS.md index cb00f0dd4..beb65f99b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,8 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [skills/loro/SKILL.md](skills/loro/SKILL.md). - Pure TypeScript runtime indexes, complexity contracts, benchmarks, and remaining gaps: [context/loro-js-performance.md](context/loro-js-performance.md). +- loro.js vs Rust randomized Text differential tests and the Rust behaviors they + work around: [context/loro-js-rust-differential.md](context/loro-js-rust-differential.md). - WASM panic/OOM reporting channels, `__wbindgen_start` glue invariant, and trap-testing recipes: [context/wasm-error-reporting.md](context/wasm-error-reporting.md). diff --git a/context/CONTEXT-GAPS.md b/context/CONTEXT-GAPS.md index debfca641..9b92d88f0 100644 --- a/context/CONTEXT-GAPS.md +++ b/context/CONTEXT-GAPS.md @@ -5,4 +5,3 @@ fix the docs in that change. Format: `YYYY-MM-DD | | | why it was hard | suggested home` -2026-09-21 | How do I test that loro.js output imports into Rust/loro-crdt? | `loro-js/scripts/rewrite-rust-fixture.mjs` (`pnpm -C loro-js fixtures:rewrite`) writes `loro-js/tests/fixtures/rust/*.ts.blob`, consumed by `crates/loro/tests/loro_js_interop.rs`; loro.js state encoders live in `loro-js/src/runtime/document.ts` (`#containerState`) and must follow `docs/encoding-container-states.md` (tree sibling order: loro-dev/loro#1088) | root AGENTS only links loro.js performance context; the interop path is mentioned only in `loro-js/README.md` | a `loro-js/AGENTS.md` or a loro.js interop context article diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 896482166..38a37474a 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -1,6 +1,6 @@ # loro-js Performance Architecture -Verified against code 2026-07-22. +Verified against code 2026-09-28. The pure TypeScript runtime lives in `loro-js/src/runtime`. Its performance target is the asymptotic behavior of the Rust runtime, while accepting a larger @@ -103,7 +103,16 @@ JavaScript constant factor. keep their single-child edge implicit, and `SequenceIndex` can skip an entire future ID run while finding the next causally included element. Ordinary local edits keep the smaller unindexed path. MovableList continues to use the scan - because moves break the origin-tree physical preorder. + because moves break the origin-tree physical preorder. After the last sibling + the new element goes after that sibling's subtree (`fugueSubtreeEnd`), not at + the origin-right bound: when that bound is only the next causally included + element, later concurrent elements of an ancestor's subtree can sit before it. + The walk stops at the first element whose origin-left precedes the sibling. +- An imported Text delete is resolved by its position in the op's causal view, + like Rust's tracker (`LoroText._deleteTargets`): O(log n + runs) through + `visibleIdRuns` or the cached causal view. The recorded `start_id` is only a + fallback, because Rust's WASM build can record one that is off by the UTF-16 + length of astral text. Local deletes skip the lookup. - Merging adjacent changes appends only the new operations and key-table entries to the retained record. The cached operation length, peer end, frontier set, operation indexes, and subscriber update slice are updated incrementally, so diff --git a/context/loro-js-rust-differential.md b/context/loro-js-rust-differential.md new file mode 100644 index 000000000..898e19ce4 --- /dev/null +++ b/context/loro-js-rust-differential.md @@ -0,0 +1,79 @@ +# loro-js Differential Tests Against Rust + +Verified against code 2026-09-28. + +`loro-js/tests/richtext-differential.test.ts` runs random multi-peer Text +scenarios in loro.js and in Rust (`loro-crdt`, nodejs WASM build) side by side +and compares them after every step. It is the check for "does loro.js mean the +same thing as Rust" questions about Text positions, deletes, concurrency, +checkout, snapshots, cursors, and events. + +## Running it + +1. Build the reference: `pnpm -C crates/loro-wasm build-dev` (or + `build-release`). The tests load `crates/loro-wasm/nodejs/index.js`, or the + file named by `LORO_WASM_NODEJS`. Only the nodejs target is needed. +2. `pnpm --dir loro-js test` runs the suite; it is skipped when no build exists. + Set `LORO_REQUIRE_WASM_REFERENCE=1` to fail instead of skipping. +3. Scale with `LORO_DIFF_SEEDS` (default 40), `LORO_DIFF_STEPS` (default 80), + and `LORO_DIFF_FIRST_SEED`. 40 × 80 takes a few seconds. + +CI runs the loro.js suite with the reference required (`pnpm test-loro-js`, part +of `test-all` after `release-wasm`), so a Rust change that alters Text behavior +also shows up here. + +## What a scenario does + +`tests/support/richtext-differential.ts` generates plain-data actions from a +seed (`generateActions`), so a failure replays exactly. Each peer exists once +per runtime with the same peer ID. Actions: + +- `insert`/`delete` at UTF-16 or UTF-8 boundaries (BMP, CJK, astral, and + multi-scalar emoji), `mark`/`unmark` with before/after/both/none styles; +- `sync` by binary or JSON updates, within a runtime or crossed (Rust imports + the loro.js peer's updates and loro.js imports Rust's), so each side reads + the other's positions; +- `checkout` to a recorded version and back; +- full and shallow snapshot export from either runtime, imported into both; +- cursor creation, then cursor resolution after every later step; +- `revertTo` and undo/redo. + +After each step it compares the delta (normalized: merged runs, `null` +attributes dropped), UTF-16/Unicode/UTF-8 lengths, cursor positions, and the +atoms of the JSON history (every inserted scalar and mark with its entity +position; Rust groups atoms into ops depending on its encoding state). loro.js +events must rebuild the loro.js state; Rust events are not checked (see below). + +`revertTo` must produce the same state on forks of both runtimes, but its ops +may differ, so the replica runs it in one runtime and the other imports the +change. Undo/redo run in loro.js and Rust imports the result: loro.js's +`UndoManager` does not transform against remote edits like Rust's, so exact undo +parity is only an option (`undoParity`). + +## Shrinking a failure + +`shrinkActions(rust, actions, options)` removes actions while the scenario +still fails with the same first-line message (numbers ignored). A throwaway +Vitest file that calls `generateActions`, `runActions`, and `shrinkActions` +and prints the result is the quickest loop; paste the shrunk action list into a +regression test. + +## Rust behaviors the harness works around + +The Rust reference has bugs of its own in UTF-16 (WASM) builds: + +- A subscribed transaction that merges deletes of astral text can panic with + "Op/hint length mismatch" (`change_to_diff` in + `crates/loro-internal/src/txn.rs`), even after unsubscribing. The harness + subscribes only to loro.js documents. +- `get_text_entity_ranges` advanced the recorded delete `start_id` by the UTF-16 + length of astral text, so Rust can write a delete whose `start_id` names the + wrong elements. Rust applies deletes by position, and so does loro.js + (`LoroText._deleteTargets`). Rust still resolves a deleted cursor target + through the recorded IDs, so the harness skips cursors Rust cannot resolve. +- Rust's shallow import falls back to the recorded delete IDs when they disagree + with the positions, which can diverge from its own full-history import. The + harness compares loro.js with the source document instead. + +Snapshot-import checkout (`snapshotCheckout`) is off by default until loro.js +can check out a document imported from a snapshot (loro-dev/loro#1126). diff --git a/loro-js/AGENTS.md b/loro-js/AGENTS.md new file mode 100644 index 000000000..cd5b3661c --- /dev/null +++ b/loro-js/AGENTS.md @@ -0,0 +1,30 @@ +# loro.js + +Pure TypeScript implementation of Loro (`loro-js/src`). It must read and write +the same binary and JSON data as Rust (`loro-crdt`) and give the same results. + +## Context + +- Index structures, complexity contracts, and benchmarks: + [context/loro-js-performance.md](../context/loro-js-performance.md). +- Randomized Text comparison with Rust (build the nodejs WASM package first): + [context/loro-js-rust-differential.md](../context/loro-js-rust-differential.md). +- Encoding formats: [context/internal-encoding.md](../context/internal-encoding.md) + and `docs/encoding*.md`. + +## Interop fixtures + +- Rust output read by loro.js: `tests/fixtures/rust/*.blob` and `*.json`. +- loro.js output read by Rust: `pnpm fixtures:rewrite` runs + `scripts/rewrite-rust-fixture.mjs`, which writes `tests/fixtures/rust/*.ts.blob`; + `crates/loro/tests/loro_js_interop.rs` imports them. Container state encoders + live in `src/runtime/document.ts` (`#containerState`) and must follow + `docs/encoding-container-states.md`. + +## Commands + +From `loro-js/`: `pnpm test`, `pnpm typecheck`, `pnpm lint`, `pnpm format:check`, +`pnpm build`, or `pnpm check` for all of them. From the root, `pnpm test-loro-js` +runs the tests and fails if the Rust WASM reference is missing (CI runs it after +`release-wasm`). Benchmarks: `pnpm bench:b4`, +`pnpm bench:complexity -- 1000,2000,4000,8000`. diff --git a/loro-js/CLAUDE.md b/loro-js/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/loro-js/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/loro-js/README.md b/loro-js/README.md index 5c22ea5c4..6e037ad0e 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -94,6 +94,9 @@ The following paths are checked against the Rust implementation: - JSON update import/export and the postcard Awareness/EphemeralStore protocols. - Rust-produced fixtures imported by TypeScript and TypeScript-produced fixtures imported by Rust. +- Randomized multi-peer text edits run side by side with the `loro-crdt` WASM build + (`tests/richtext-differential.test.ts`): positions, deletes, concurrency, checkout, + snapshots, cursors, events, and `revertTo`. This is not yet a claim of complete behavioral equivalence with every `loro-crdt` edge case. Important current limits are: diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 9b3979b91..caa37a646 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -1722,19 +1722,25 @@ export class LoroText extends LoroContainer { return this._sequence.metricOffsetAtVisibleIndex(unicodeIndex, to); } + /** + * Matches Rust: an empty text or a position at the end yields an ID-less + * cursor (left side when empty, right side at the end); otherwise the cursor + * names the scalar that starts at the UTF-16 position. The encoded origin is + * Rust's too: the Unicode position, or the UTF-16 length at the end. + */ getCursor(pos: number, side: Side = 0): Cursor | undefined { if (!Number.isSafeInteger(pos) || pos < 0) return undefined; - if (pos >= this.length) { - return new Cursor( - this.id, - this._sequence.atVisible(this._sequence.visibleLength - 1)?.id, - 1, - this.length, - ); - } + const length = this.length; + if (length === 0) return new Cursor(this.id, undefined, side === 0 ? -1 : side, 0); + if (pos >= length) return new Cursor(this.id, undefined, 1, length); const unicodePosition = this.convertPos(pos, "utf16", "unicode"); if (unicodePosition === undefined) return undefined; - return new Cursor(this.id, this._sequence.atVisible(unicodePosition)!.id, side, pos); + return new Cursor( + this.id, + this._sequence.atVisible(unicodePosition)!.id, + side, + unicodePosition, + ); } getEditorOf(pos: number): string | undefined { @@ -1878,6 +1884,64 @@ export class LoroText extends LoroContainer { this._sequence.deleteIdSpan(startId, length, deletedBy); } + /** + * Resolves the elements a delete operation removes. Rust's tracker deletes the + * visible elements at `position` in the operation's causal view and uses + * `startId` only to name them, so the position wins when the two disagree + * (Rust's WASM build can record a `startId` that is off by the UTF-16 length + * of astral characters). The recorded IDs are used only when the position + * range does not fit the causal view. + */ + _deleteTargets( + position: number, + length: number, + startId: CodecId, + causalVersion: CausalVersion, + ): SequenceIdRun[] { + const recorded = [{ start: startId, length }]; + const current = this._sequence.isFullyIncluded(causalVersion); + if (current && length === 1) { + const element = this._sequence.atVisible(position); + if ( + element === undefined || + (element.id.peer === startId.peer && element.id.counter === startId.counter) + ) { + return recorded; + } + return [{ start: element.id, length: 1 }]; + } + const runs = current + ? this._sequence.visibleIdRuns(position, position + length) + : this._sequence.causalView(causalVersion).idRuns(position, position + length); + let resolved = 0; + for (const run of runs) resolved += run.length; + return resolved === length ? runs : recorded; + } + + /** + * Deletes resolved target runs. Operation counters follow Rust: a forward + * delete's k-th counter removes the k-th target, a reversed delete's k-th + * counter removes the k-th target from the right. + */ + _deleteTargetRuns( + runs: readonly SequenceIdRun[], + reversed: boolean, + deletedBy: CodecId, + ): void { + const total = runs.reduce((sum, run) => sum + run.length, 0); + let offset = 0; + for (const run of runs) { + const counter = reversed + ? deletedBy.counter + total - offset - run.length + : deletedBy.counter + offset; + this._deleteIdSpan(run.start, reversed ? -run.length : run.length, { + peer: deletedBy.peer, + counter, + }); + offset += run.length; + } + } + _applyMark( start: number, end: number, @@ -3054,7 +3118,9 @@ function indexedFugueInsertion( } if (!scanning) { - insertIndex = candidates[childIndex + 1]?.index ?? originRightIndex; + insertIndex = + candidates[childIndex + 1]?.index ?? + fugueSubtreeEnd(sequence, candidates[childIndex]!.index, originRightIndex); } } @@ -3066,6 +3132,39 @@ function indexedFugueInsertion( }; } +/** + * Returns the physical index right after the Fugue subtree rooted at + * `rootIndex`, bounded by `limit`. Elements between the last sibling and the + * origin-right bound need not descend from that sibling: when the bound is only + * the next causally included element, later concurrent elements that belong to + * an ancestor's subtree can sit before it. An element is a descendant exactly + * when its origin-left lies inside the scanned subtree. + */ +function fugueSubtreeEnd( + sequence: SequenceIndex, + rootIndex: number, + limit: number, +): number { + let end = rootIndex + 1; + if (end >= limit) return limit; + let previous = sequence.atPhysicalRaw(rootIndex)!.id; + sequence.forEachPhysicalFrom(end, (element, index) => { + if (index >= limit) return false; + const left = element.originLeft; + if (left === undefined) return false; + if (left.peer !== previous.peer || left.counter !== previous.counter) { + const leftElement = sequence.findByIdRaw(left); + const leftIndex = + leftElement === undefined ? undefined : sequence.physicalIndexOf(leftElement); + if (leftIndex === undefined || leftIndex < rootIndex) return false; + } + previous = element.id; + end = index + 1; + return undefined; + }); + return end; +} + function getFugueOriginIndex( sequence: SequenceIndex, ): FugueOriginIndex { diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 57f9a8a4d..a28d1d17f 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -1951,6 +1951,7 @@ export class LoroDoc = Record = Record true)) return undefined; + const offset = text._sequence.visibleMetricOffsetOf(target, "utf16") ?? 0; + const update = text.getCursor(offset, -1); + return update === undefined ? { offset, side: -1 } : { offset, side: -1, update }; + } + _subscribeContainer( container: LoroContainer, listener: (event: LoroEventBatch) => void, @@ -2086,7 +2117,14 @@ export class LoroDoc = Record = Record = Record = Record = Record ({ position: start, length: end - start })); } + #textDeleteTargets( + text: LoroText, + content: Extract, + causalVersion: CausalVersion, + ): SequenceIdRun[] { + const length = Number(content.length); + // A reversed (backspace) span stores its rightmost position. + return text._deleteTargets( + length < 0 ? content.position + length + 1 : content.position, + Math.abs(length), + content.startId, + causalVersion, + ); + } + #recordedEventDiffs( recording: EventRecording, includeMapTombstones = false, @@ -3132,12 +3179,18 @@ export class LoroDoc = Record= 0; index -= 1) { + const run = runs[index]!; + end -= run.length; this.#appendAndApply( container, { type: "text-delete", - position, + position: end, length: BigInt(run.length), startId: run.start, }, @@ -3288,6 +3341,7 @@ export class LoroDoc = Record = Record = Record>(); + // Keyed by ID: packed text spans hand out a fresh element view per lookup, + // so an element deleted by two concurrent ops must not be collected twice. + const sequences = new Map(); const bulkSequenceRemovals = new Map(); const bulkSequenceRestorations = new Map(); const textAttributeRuns = new Map>(); @@ -3875,10 +3933,10 @@ export class LoroDoc = Record>(); const treeSubjects = new Map>(); const counters = new Map(); - const sequenceElements = (container: LoroList | LoroText): Set => { + const sequenceElements = (container: LoroList | LoroText): SequenceElementSet => { let elements = sequences.get(container); if (elements === undefined) { - elements = new Set(); + elements = new SequenceElementSet(); sequences.set(container, elements); } return elements; @@ -6533,6 +6591,26 @@ function appendToTrailingListInsert( operation: DecodedOperation, ): boolean { const content = operation.content; + if (content.type === "text-insert") { + // Rust merges a text insert that continues the previous one in the change. + const previousIndex = operations.length - 1; + const previous = operations[previousIndex]; + if ( + previous === undefined || + previous.content.type !== "text-insert" || + !containerIdsEqual(previous.container, operation.container) || + previous.counter + previous.length !== operation.counter || + previous.content.position + previous.length !== content.position + ) { + return false; + } + operations[previousIndex] = { + ...previous, + length: previous.length + operation.length, + content: { ...previous.content, value: previous.content.value + content.value }, + }; + return true; + } if (content.type !== "list-insert" && content.type !== "movable-list-insert") { return false; } @@ -8128,6 +8206,20 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { return false; } +/** A set of sequence elements keyed by ID rather than by element identity. */ +class SequenceElementSet implements Iterable { + readonly #elements = new Map(); + + add(element: SequenceElement): void { + const key = idKey(element.id); + if (!this.#elements.has(key)) this.#elements.set(key, element); + } + + [Symbol.iterator](): Iterator { + return this.#elements.values(); + } +} + function subtractSequenceIdRuns( runs: readonly SequenceIdRun[], removedRuns: readonly SequenceIdRun[], diff --git a/loro-js/src/runtime/sequence-index.ts b/loro-js/src/runtime/sequence-index.ts index df09a73eb..9d7d92290 100644 --- a/loro-js/src/runtime/sequence-index.ts +++ b/loro-js/src/runtime/sequence-index.ts @@ -423,6 +423,53 @@ export class SequenceIndex { }); } + /** Visits elements in physical order from `start` until `visit` returns false. */ + forEachPhysicalFrom( + start: number, + visit: (element: T, index: number) => boolean | void, + ): void { + if (!Number.isSafeInteger(start) || start < 0 || start >= this.allLength) return; + const stack: SequenceNode[] = []; + let node = this.#root; + let base = 0; + let current: SequenceNode | undefined; + let offset = 0; + while (node !== undefined) { + const leftCount = allCount(node.left); + if (start < base + leftCount) { + stack.push(node); + node = node.left; + continue; + } + const ownStart = base + leftCount; + if (start < ownStart + nodeLength(node)) { + current = node; + offset = start - ownStart; + break; + } + base = ownStart + nodeLength(node); + node = node.right; + } + let index = start; + while (current !== undefined) { + for (; offset < nodeLength(current); offset += 1) { + if (visit(nodeElement(current, offset), index) === false) return; + index += 1; + } + let next = current.right; + if (next === undefined) { + current = stack.pop(); + } else { + while (next.left !== undefined) { + stack.push(next); + next = next.left; + } + current = next; + } + offset = 0; + } + } + findNextIncludedPhysical( start: number, version: ReadonlyMap, diff --git a/loro-js/tests/fixtures/rust/text-delete-positions.blob b/loro-js/tests/fixtures/rust/text-delete-positions.blob new file mode 100644 index 0000000000000000000000000000000000000000..4fd66bff8494a9d53b33918791f9fa399049c598 GIT binary patch literal 108 zcmd1FFUn^?0zN%!Wf@pP8Q2-v83h>Gp)!n&UEs literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/rust/text-delete-positions.json b/loro-js/tests/fixtures/rust/text-delete-positions.json new file mode 100644 index 000000000..1e952d15e --- /dev/null +++ b/loro-js/tests/fixtures/rust/text-delete-positions.json @@ -0,0 +1,49 @@ +{ + "text": "b", + "updates": { + "schema_version": 1, + "start_version": {}, + "peers": [ + "7" + ], + "changes": [ + { + "id": "0@0", + "timestamp": 0, + "deps": [], + "lamport": 0, + "msg": null, + "ops": [ + { + "container": "cid:root-text:Text", + "content": { + "type": "insert", + "pos": 0, + "text": "a😀b" + }, + "counter": 0 + }, + { + "container": "cid:root-text:Text", + "content": { + "type": "insert", + "pos": 2, + "text": "x" + }, + "counter": 3 + }, + { + "container": "cid:root-text:Text", + "content": { + "type": "delete", + "pos": 0, + "len": 3, + "start_id": "0@0" + }, + "counter": 4 + } + ] + } + ] + } +} diff --git a/loro-js/tests/richtext-differential.test.ts b/loro-js/tests/richtext-differential.test.ts new file mode 100644 index 000000000..3c84c83c2 --- /dev/null +++ b/loro-js/tests/richtext-differential.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "vitest"; + +import { generateActions, runActions } from "./support/richtext-differential"; +import { loadRustReference } from "./support/rust-reference"; + +const rust = loadRustReference(); +const seeds = Number(process.env.LORO_DIFF_SEEDS ?? 40); +const steps = Number(process.env.LORO_DIFF_STEPS ?? 80); +const firstSeed = Number(process.env.LORO_DIFF_FIRST_SEED ?? 1); +// Marks need loro.js to keep Rust's style anchors in the text sequence. +const options = { marks: false }; + +describe.skipIf(rust === undefined)("text differential against Rust", () => { + test.each(Array.from({ length: seeds }, (_, index) => firstSeed + index))( + "random concurrent text edits converge with Rust (seed %i)", + (seed) => { + expect(runActions(rust!, generateActions(seed, steps, options), options)).toBe( + undefined, + ); + }, + ); +}); diff --git a/loro-js/tests/support/richtext-differential.ts b/loro-js/tests/support/richtext-differential.ts new file mode 100644 index 000000000..47d0a75dd --- /dev/null +++ b/loro-js/tests/support/richtext-differential.ts @@ -0,0 +1,981 @@ +import * as Js from "../../src/index"; +import type { Delta, Frontiers, LoroEventBatch, Value } from "../../src/index"; +import type { RustReference } from "./rust-reference"; + +/** + * Differential rich-text scenarios between loro.js and the Rust implementation + * (`loro-crdt`). Every peer exists once per runtime; each action runs the same + * public API call on both copies and then compares text, delta, attributes, + * event-reconstructed state, and cursors. Syncs randomly move binary or JSON + * updates within a runtime or across runtimes, so positions written by one + * implementation are interpreted by the other. + * + * Actions are plain data with unresolved positions, so a failing sequence can be + * replayed, shrunk (`shrinkActions`), and pasted into a regression test. + */ + +type Runtime = "rust" | "js"; +type Doc = Js.LoroDoc; +type Text = Js.LoroText; +type UndoManager = Js.UndoManager; +type Cursor = Js.Cursor; + +export const STYLE_CONFIG = { + bold: { expand: "after" }, + link: { expand: "none" }, + em: { expand: "before" }, + hl: { expand: "both" }, +} as const; +const STYLE_KEYS = Object.keys(STYLE_CONFIG); +const STYLE_VALUES: Value[] = [true, false, 1, 2, "x", "y", { a: 1 }]; +const TEXTS = [ + "a", + "b", + "cd", + "xyz", + "é", + "中文", + "😀", + "a😀b", + "👍🏽", + "\n", + "ab\ncd", + "𝒳y", +]; + +export type Action = + | { + readonly type: "insert"; + readonly peer: number; + readonly pos: number; + readonly text: string; + readonly utf8?: boolean; + } + | { + readonly type: "delete"; + readonly peer: number; + readonly pos: number; + readonly len: number; + readonly utf8?: boolean; + } + | { + readonly type: "mark"; + readonly peer: number; + readonly pos: number; + readonly len: number; + readonly key: string; + readonly value: Value; + } + | { + readonly type: "unmark"; + readonly peer: number; + readonly pos: number; + readonly len: number; + readonly key: string; + } + | { readonly type: "commit"; readonly peer: number } + | { + readonly type: "sync"; + readonly from: number; + readonly to: number; + readonly json: boolean; + readonly cross: boolean; + } + | { readonly type: "checkout"; readonly peer: number; readonly version: number } + | { readonly type: "undo" | "redo"; readonly peer: number } + | { + readonly type: "revert"; + readonly peer: number; + readonly version: number; + readonly inRust: boolean; + } + | { + readonly type: "snapshot"; + readonly peer: number; + readonly fromRust: boolean; + readonly version: number; + } + | { + readonly type: "shallow"; + readonly peer: number; + readonly fromRust: boolean; + readonly version: number; + } + | { + readonly type: "cursor"; + readonly peer: number; + readonly pos: number; + readonly side: -1 | 0 | 1; + }; + +export interface ScenarioOptions { + readonly peers?: number; + readonly undo?: boolean; + readonly crossRuntimeSync?: boolean; + readonly snapshots?: boolean; + readonly snapshotCheckout?: boolean; + readonly checkout?: boolean; + readonly revert?: boolean; + readonly cursors?: boolean; + readonly events?: boolean; + readonly marks?: boolean; + readonly compareOps?: boolean; + /** + * Run undo/redo in both runtimes and compare, instead of running it in + * loro.js and importing the result into Rust. + */ + readonly undoParity?: boolean; +} + +const DEFAULT_OPTIONS: Required = { + peers: 3, + undo: true, + crossRuntimeSync: true, + snapshots: true, + // Checking out a document imported from a snapshot needs loro-dev/loro#1126. + snapshotCheckout: false, + checkout: true, + revert: true, + cursors: true, + events: true, + marks: true, + compareOps: true, + undoParity: false, +}; + +export class DifferentialFailure extends Error {} + +class Random { + #state: number; + + constructor(seed: number) { + this.#state = (seed * 2_654_435_761 + 0x9e37_79b9) >>> 0 || 1; + } + + next(): number { + let value = this.#state; + value ^= value << 13; + value ^= value >>> 17; + value ^= value << 5; + this.#state = value >>> 0; + return this.#state; + } + + int(bound: number): number { + return bound <= 0 ? 0 : this.next() % bound; + } + + pick(values: readonly T[]): T { + return values[this.int(values.length)]!; + } +} + +/** Generates a replayable action list. Positions are resolved at run time. */ +export function generateActions( + seed: number, + steps: number, + options: ScenarioOptions = {}, +): Action[] { + const config = { ...DEFAULT_OPTIONS, ...options }; + const random = new Random(seed); + const actions: Action[] = []; + const peerIds = Array.from({ length: config.peers }, (_, index) => index + 1); + const commitMaybe = (peer: number): void => { + if (random.int(5) !== 0) actions.push({ type: "commit", peer }); + }; + for (let step = 0; step < steps; step += 1) { + const peer = random.pick(peerIds); + const roll = random.int(100); + const pos = random.next(); + const len = 1 + random.int(6); + if (roll < 32) { + const text = random.int(4) === 0 ? `${step % 10}` : random.pick(TEXTS); + actions.push({ type: "insert", peer, pos, text, utf8: roll >= 26 }); + commitMaybe(peer); + } else if (roll < 48) { + actions.push({ type: "delete", peer, pos, len, utf8: roll >= 44 }); + commitMaybe(peer); + } else if (roll < 62 && config.marks) { + const key = random.pick(STYLE_KEYS); + const value = random.int(8) === 0 ? null : random.pick(STYLE_VALUES); + actions.push({ type: "mark", peer, pos, len, key, value }); + commitMaybe(peer); + } else if (roll < 68 && config.marks) { + actions.push({ type: "unmark", peer, pos, len, key: random.pick(STYLE_KEYS) }); + commitMaybe(peer); + } else if (roll < 80) { + const to = random.pick(peerIds.filter((id) => id !== peer)); + actions.push({ + type: "sync", + from: peer, + to, + json: random.int(2) === 0, + cross: config.crossRuntimeSync && random.int(2) === 0, + }); + } else if (roll < 84 && config.checkout) { + actions.push({ type: "checkout", peer, version: random.next() }); + } else if (roll < 88 && config.undo) { + actions.push({ type: random.int(3) === 0 ? "redo" : "undo", peer }); + } else if (roll < 90 && config.revert) { + actions.push({ + type: "revert", + peer, + version: random.next(), + inRust: random.int(2) === 0, + }); + } else if (roll < 93 && config.snapshots) { + actions.push({ + type: "snapshot", + peer, + fromRust: random.int(2) === 0, + version: random.next(), + }); + } else if (roll < 95 && config.snapshots) { + actions.push({ + type: "shallow", + peer, + fromRust: random.int(2) === 0, + version: random.next(), + }); + } else if (roll < 98 && config.cursors) { + actions.push({ type: "cursor", peer, pos, side: random.pick([-1, 0, 1] as const) }); + } else { + actions.push({ type: "commit", peer }); + } + } + return actions; +} + +interface Replica { + readonly runtime: Runtime; + readonly doc: Doc; + readonly shadow: EventShadow; + readonly undo: UndoManager | undefined; +} + +interface EventShadow { + delta: Delta[]; +} + +interface Peer { + readonly id: number; + readonly rust: Replica; + readonly js: Replica; + readonly versions: Frontiers[]; + readonly cursors: { rust: Cursor; js: Cursor }[]; +} + +export class RichtextScenario { + readonly #rust: typeof Js; + readonly #options: Required; + readonly #peers: Peer[] = []; + readonly #log: string[] = []; + + constructor(rust: RustReference, options: ScenarioOptions = {}) { + this.#rust = rust as unknown as typeof Js; + this.#options = { ...DEFAULT_OPTIONS, ...options }; + for (let index = 0; index < this.#options.peers; index += 1) { + const id = index + 1; + this.#peers.push({ + id, + rust: this.#replica("rust", id), + js: this.#replica("js", id), + versions: [], + cursors: [], + }); + } + } + + /** The replicas of a peer, for diagnostics. */ + replicas(peer: number): { rust: Doc; js: Doc } { + const selected = this.#peer(peer); + return { rust: selected.rust.doc, js: selected.js.doc }; + } + + run(actions: readonly Action[]): void { + for (const action of actions) this.#apply(action); + this.#syncAll(); + this.#compareAll("final sync"); + } + + #replica(runtime: Runtime, peer: number): Replica { + const module = runtime === "rust" ? this.#rust : Js; + const doc = new module.LoroDoc(); + doc.setPeerId(peer); + doc.configTextStyle(STYLE_CONFIG); + doc.getText("t"); + const shadow: EventShadow = { delta: [] }; + // Only loro.js events are checked: they must reproduce loro.js state, which + // is compared with Rust state. Rust itself panics ("Op/hint length + // mismatch", `txn.rs`) when a subscribed transaction merges deletes whose + // UTF-16 and Unicode lengths differ, even after unsubscribing. + if (this.#options.events && runtime === "js") { + doc.subscribe((batch: LoroEventBatch) => { + for (const event of batch.events) { + if (event.diff.type !== "text") continue; + shadow.delta = applyDelta(shadow.delta, event.diff.diff as Delta[]); + } + }); + } + const undo = + this.#options.undo && (runtime === "js" || this.#options.undoParity) + ? new module.UndoManager(doc, { mergeInterval: 0, maxUndoSteps: 100 }) + : undefined; + return { runtime, doc, shadow, undo }; + } + + #peer(id: number): Peer { + return this.#peers[(id - 1) % this.#peers.length]!; + } + + #apply(action: Action): void { + switch (action.type) { + case "insert": { + const peer = this.#peer(action.peer); + const position = pickBoundary(this.#currentText(peer), action.pos, action.utf8); + const text = action.text; + if (action.utf8) { + this.#both(peer, `insertUtf8(${position}, ${JSON.stringify(text)})`, (r) => + this.#text(r).insertUtf8(position, text), + ); + } else { + this.#both(peer, `insert(${position}, ${JSON.stringify(text)})`, (r) => + this.#text(r).insert(position, text), + ); + } + return; + } + case "delete": { + const peer = this.#peer(action.peer); + const range = pickRange( + this.#currentText(peer), + action.pos, + action.len, + action.utf8, + ); + if (range === undefined) return; + const [start, end] = range; + if (action.utf8) { + this.#both(peer, `deleteUtf8(${start}, ${end - start})`, (r) => + this.#text(r).deleteUtf8(start, end - start), + ); + } else { + this.#both(peer, `delete(${start}, ${end - start})`, (r) => + this.#text(r).delete(start, end - start), + ); + } + return; + } + case "mark": { + const peer = this.#peer(action.peer); + const range = pickRange(this.#currentText(peer), action.pos, action.len); + if (range === undefined) return; + const [start, end] = range; + const { key, value } = action; + this.#both( + peer, + `mark(${start}..${end}, ${key}, ${JSON.stringify(value)})`, + (r) => this.#text(r).mark({ start, end }, key, value), + ); + return; + } + case "unmark": { + const peer = this.#peer(action.peer); + const range = pickRange(this.#currentText(peer), action.pos, action.len); + if (range === undefined) return; + const [start, end] = range; + this.#both(peer, `unmark(${start}..${end}, ${action.key})`, (r) => + this.#text(r).unmark({ start, end }, action.key), + ); + return; + } + case "commit": + this.#commit(this.#peer(action.peer)); + return; + case "sync": + this.#sync(action); + return; + case "checkout": { + const peer = this.#peer(action.peer); + if (peer.versions.length === 0) return; + this.#commit(peer); + const version = peer.versions[action.version % peer.versions.length]!; + this.#both(peer, `checkout(${JSON.stringify(version)})`, (r) => + r.doc.checkout(version), + ); + this.#both(peer, "checkoutToLatest", (r) => r.doc.checkoutToLatest()); + return; + } + case "undo": + case "redo": { + // loro.js's undo is a separate implementation that does not produce + // Rust's ops (it does not transform against remote edits, for example). + // Run it in loro.js and check that Rust reads the resulting ops the same. + const peer = this.#peer(action.peer); + this.#commit(peer); + if (this.#options.undoParity) { + this.#both(peer, action.type, (replica) => { + if (action.type === "redo") replica.undo?.redo(); + else replica.undo?.undo(); + }); + this.#commit(peer); + return; + } + this.#oneSided(peer, peer.js, action.type, () => { + if (action.type === "redo") peer.js.undo?.redo(); + else peer.js.undo?.undo(); + }); + return; + } + case "revert": { + const peer = this.#peer(action.peer); + if (peer.versions.length === 0) return; + this.#commit(peer); + const version = peer.versions[action.version % peer.versions.length]!; + const label = `revertTo(${JSON.stringify(version)})`; + // The resulting state must match, but the ops need not: Rust applies its + // diff delta in document order, including where inserted text sits + // relative to deleted text, while loro.js composes its diff first. + const rustFork = peer.rust.doc.fork(); + const jsFork = peer.js.doc.fork(); + rustFork.revertTo(version); + jsFork.revertTo(version); + this.#expectDelta( + jsFork, + normalizeDelta(rustFork.getText("t").toDelta()), + `${label} on forks`, + ); + const actor = action.inRust ? peer.rust : peer.js; + this.#oneSided(peer, actor, label, () => actor.doc.revertTo(version)); + return; + } + case "snapshot": + this.#snapshotRoundTrip(action); + return; + case "shallow": + this.#shallowRoundTrip(action); + return; + case "cursor": + this.#createCursor(action); + return; + } + } + + #text(replica: Replica): Text { + return replica.doc.getText("t"); + } + + #currentText(peer: Peer): string { + return this.#text(peer.rust).toString(); + } + + #both(peer: Peer, label: string, action: (replica: Replica) => void): void { + this.#log.push(`peer ${peer.id}: ${label}`); + const errors: { runtime: Runtime; error: unknown }[] = []; + for (const replica of [peer.rust, peer.js]) { + try { + action(replica); + } catch (error) { + errors.push({ runtime: replica.runtime, error }); + } + } + if (errors.length === 1) { + this.#fail( + `${label}: only ${errors[0]!.runtime} threw: ${String(errors[0]!.error)}`, + ); + } + this.#compare(peer, label); + } + + /** + * Runs an edit on one replica only, then imports its change into the other + * replica of the same peer, so both keep identical histories. + */ + #oneSided(peer: Peer, actor: Replica, label: string, edit: () => void): void { + this.#log.push(`peer ${peer.id}: ${label} in ${actor.runtime}`); + try { + edit(); + actor.doc.commit(); + } catch (error) { + this.#fail(`${label} in ${actor.runtime} threw: ${String(error)}`); + } + const other = actor === peer.rust ? peer.js : peer.rust; + other.doc.import(actor.doc.export({ mode: "update" })); + this.#compare(peer, `${label} in ${actor.runtime}`); + peer.versions.push(peer.rust.doc.frontiers()); + } + + #commit(peer: Peer): void { + this.#both(peer, "commit", (replica) => replica.doc.commit()); + const version = peer.rust.doc.frontiers(); + const previous = peer.versions.at(-1); + if (previous === undefined || JSON.stringify(previous) !== JSON.stringify(version)) { + peer.versions.push(version); + } + } + + #sync(action: Extract): void { + const from = this.#peer(action.from); + const to = this.#peer(action.to); + if (from === to) return; + this.#commit(from); + this.#commit(to); + const cross = action.cross && this.#options.crossRuntimeSync; + this.#log.push( + `sync ${from.id} -> ${to.id} (${action.json ? "json" : "binary"}${cross ? ", cross" : ""})`, + ); + const move = (source: Replica, target: Replica): void => { + if (action.json) { + target.doc.importJsonUpdates(JSON.stringify(source.doc.exportJsonUpdates())); + } else { + target.doc.import(source.doc.export({ mode: "update" })); + } + }; + move(cross ? from.js : from.rust, to.rust); + move(cross ? from.rust : from.js, to.js); + this.#compare(to, `after sync ${from.id} -> ${to.id}`); + to.versions.push(to.rust.doc.frontiers()); + } + + #syncAll(): void { + this.#log.push("sync all"); + for (const from of this.#peers) { + for (const to of this.#peers) { + if (from === to) continue; + this.#commit(from); + const cross = this.#options.crossRuntimeSync; + to.rust.doc.import((cross ? from.js : from.rust).doc.export({ mode: "update" })); + to.js.doc.import((cross ? from.rust : from.js).doc.export({ mode: "update" })); + } + } + } + + #snapshotRoundTrip(action: Extract): void { + const peer = this.#peer(action.peer); + this.#commit(peer); + const source = action.fromRust ? peer.rust : peer.js; + const bytes = source.doc.export({ mode: "snapshot" }); + this.#log.push(`peer ${peer.id}: snapshot from ${source.runtime}`); + const version = + peer.versions.length === 0 + ? undefined + : peer.versions[action.version % peer.versions.length]!; + this.#compareImported(peer, bytes, `snapshot from ${source.runtime}`, version); + } + + #shallowRoundTrip(action: Extract): void { + const peer = this.#peer(action.peer); + if (peer.versions.length === 0) return; + this.#commit(peer); + const source = action.fromRust ? peer.rust : peer.js; + const index = action.version % peer.versions.length; + const frontiers = peer.versions[index]!; + const bytes = source.doc.export({ mode: "shallow-snapshot", frontiers }); + this.#log.push( + `peer ${peer.id}: shallow snapshot from ${source.runtime} at ${JSON.stringify(frontiers)}`, + ); + const later = peer.versions.slice(index); + this.#compareImported( + peer, + bytes, + `shallow snapshot from ${source.runtime}`, + later[action.version % later.length], + ); + } + + /** + * Imports `bytes` into fresh loro.js and Rust documents. loro.js must match the + * Rust import of the same bytes; both must match the source unless Rust + * disagrees with itself (a Rust shallow document can apply a delete whose + * recorded IDs and positions disagree differently from a full one). + */ + #compareImported( + peer: Peer, + bytes: Uint8Array, + label: string, + version: Frontiers | undefined, + ): void { + const js = new Js.LoroDoc(); + js.import(bytes); + const rust = new this.#rust.LoroDoc(); + rust.import(bytes); + const reference = normalizeDelta(rust.getText("t").toDelta()); + const source = normalizeDelta(this.#text(peer.rust).toDelta()); + if (JSON.stringify(reference) !== JSON.stringify(source)) { + // Rust's shallow import falls back to the recorded delete IDs when they + // disagree with the positions, which diverges from its own full-history + // result. loro.js keeps the full-history (position) result. + this.#log.push(" (the Rust import differs from its source)"); + } + this.#expectDelta(js, source, `${label}: loro.js import vs source`); + if (!this.#options.snapshotCheckout || version === undefined) return; + const errors: string[] = []; + for (const doc of [rust, js]) { + try { + doc.checkout(version); + } catch (error) { + errors.push(String(error)); + } + } + if (errors.length === 1) { + this.#fail( + `${label}: checkout ${JSON.stringify(version)} threw only in one runtime`, + ); + } + if (errors.length === 0) { + this.#expectDelta( + js, + normalizeDelta(rust.getText("t").toDelta()), + `${label}: checkout ${JSON.stringify(version)}`, + ); + } + } + + #createCursor(action: Extract): void { + const peer = this.#peer(action.peer); + const position = pickBoundary(this.#currentText(peer), action.pos); + const side = action.side; + const rust = this.#text(peer.rust).getCursor(position, side); + const js = this.#text(peer.js).getCursor(position, side); + this.#log.push(`peer ${peer.id}: cursor(${position}, ${side})`); + if ((rust === undefined) !== (js === undefined)) { + this.#fail(`cursor(${position}, ${side}) existence differs`); + } + if (rust === undefined || js === undefined) return; + if ( + JSON.stringify(rust.pos()) !== JSON.stringify(js.pos()) || + rust.side() !== js.side() + ) { + this.#fail( + `cursor(${position}, ${side}) differs: rust ${JSON.stringify(rust.pos())}/${rust.side()} js ${JSON.stringify(js.pos())}/${js.side()}`, + ); + } + peer.cursors.push({ rust, js }); + } + + #compareCursors(peer: Peer, label: string): void { + // Rust resolves a deleted target against the oplog head but converts the + // position with the checked-out state, so detached results are not compared. + if (peer.rust.doc.getPendingTxnLength() > 0 || peer.rust.doc.isDetached()) return; + for (const cursor of peer.cursors) { + const rust = peer.rust.doc.getCursorPos(cursor.rust); + const js = peer.js.doc.getCursorPos(cursor.js); + if ( + rust === undefined && + js !== undefined && + !recordedAsDeleted(peer.rust.doc, cursor.rust) + ) { + // Rust finds a deleted target through the recorded delete IDs, which its + // WASM build can get wrong for astral text; loro.js resolves deletes by + // position and still knows where the target was. + continue; + } + if (rust?.offset !== js?.offset || rust?.side !== js?.side) { + this.#fail( + `${label}: cursor ${JSON.stringify(cursor.rust.pos())}/${cursor.rust.side()} position differs: rust ${JSON.stringify(rust && { offset: rust.offset, side: rust.side })} js ${JSON.stringify(js && { offset: js.offset, side: js.side })}`, + ); + } + } + } + + #expectDelta(doc: Doc, expected: unknown, label: string): void { + const actual = normalizeDelta(doc.getText("t").toDelta()); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + this.#fail( + `${label}: delta ${JSON.stringify(actual)} != expected ${JSON.stringify(expected)}`, + ); + } + } + + #compare(peer: Peer, label: string): void { + const rustText = this.#text(peer.rust); + const jsText = this.#text(peer.js); + if (jsText.toDelta().some((item) => "insert" in item && item.insert.length === 0)) { + this.#fail(`${label}: peer ${peer.id} loro.js delta has an empty insert`); + } + const rustDelta = normalizeDelta(rustText.toDelta()); + const jsDelta = normalizeDelta(jsText.toDelta()); + if (JSON.stringify(rustDelta) !== JSON.stringify(jsDelta)) { + this.#fail( + `${label}: peer ${peer.id} delta differs\n rust ${JSON.stringify(rustDelta)}\n js ${JSON.stringify(jsDelta)}`, + ); + } + if (rustText.length !== jsText.length) { + this.#fail(`${label}: length ${rustText.length} != ${jsText.length}`); + } + for (const unit of ["unicode", "utf8"] as const) { + const rustLength = rustText.convertPos(rustText.length, "utf16", unit); + const jsLength = jsText.convertPos(jsText.length, "utf16", unit); + if (rustLength !== jsLength) { + this.#fail(`${label}: ${unit} length ${rustLength} != ${jsLength}`); + } + } + if (this.#options.events) { + for (const replica of [peer.js]) { + if (replica.doc.getPendingTxnLength() > 0) continue; + const events = normalizeDelta(replica.shadow.delta); + const state = normalizeDelta(this.#text(replica).toDelta()); + if (JSON.stringify(events) !== JSON.stringify(state)) { + this.#fail( + `${label}: ${replica.runtime} events do not reproduce the state\n events ${JSON.stringify(events)}\n state ${JSON.stringify(state)}`, + ); + } + } + } + if (this.#options.cursors) this.#compareCursors(peer, label); + if (this.#options.compareOps) this.#compareOps(peer, label); + } + + #compareOps(peer: Peer, label: string): void { + const rust = atomicOps(peer.rust.doc); + const js = atomicOps(peer.js.doc); + const index = rust.findIndex((atom, position) => atom !== js[position]); + if (index >= 0 || rust.length !== js.length) { + const at = index >= 0 ? index : Math.min(rust.length, js.length); + this.#fail( + `${label}: peer ${peer.id} ops differ at atom ${at}\n rust ${rust[at]}\n js ${js[at]}`, + ); + } + } + + #compareAll(label: string): void { + const expected = normalizeDelta(this.#text(this.#peers[0]!.rust).toDelta()); + for (const peer of this.#peers) { + this.#compare(peer, label); + this.#expectDelta(peer.js.doc, expected, `${label}: converge peer ${peer.id}`); + } + } + + #fail(message: string): never { + throw new DifferentialFailure( + `${message}\nsteps:\n ${this.#log.slice(-40).join("\n ")}`, + ); + } +} + +/** + * The JSON history as atomic ops sorted by ID: one entry per inserted scalar + * with its entity position, plus every other non-delete op. Rust merges adjacent + * ops of merged changes depending on internal encoding state, so this form + * ignores how atoms are grouped into ops. + */ +function atomicOps(doc: Doc): string[] { + const json = doc.exportJsonUpdates(); + const peers = json.peers; + const peerOf = (value: string): string => + peers == null ? value : peers[Number(value)]!; + const expand = (value: string): [number, string] => { + const [counter, peer] = value.split("@"); + return [Number(counter), peerOf(peer!)]; + }; + const atoms: { peer: string; counter: number; text: string }[] = []; + for (const change of json.changes) { + const [, peer] = expand(change.id); + for (const op of change.ops) { + const content = op.content as Record; + const push = (offset: number, text: string): void => { + atoms.push({ peer, counter: op.counter + offset, text }); + }; + if (content.type === "insert" && typeof content.text === "string") { + // One atom per Unicode scalar, each with its entity position. + let index = 0; + for (const char of content.text) { + push(index, `insert ${Number(content.pos) + index} ${JSON.stringify(char)}`); + index += 1; + } + } else if (content.type === "delete") { + // Deletes are compared through state: Rust's WASM build can record + // delete IDs that disagree with the positions it applies (astral text). + continue; + } else { + push(0, JSON.stringify(content)); + } + } + } + return atoms + .sort((left, right) => + left.peer < right.peer + ? -1 + : left.peer > right.peer + ? 1 + : left.counter - right.counter, + ) + .map((atom) => `${atom.counter}@${atom.peer} ${atom.text}`); +} + +/** Whether some recorded delete op names the cursor's target, as Rust requires. */ +function recordedAsDeleted(doc: Doc, cursor: Cursor): boolean { + const id = cursor.pos(); + if (id === undefined) return true; + const json = doc.exportJsonUpdates(); + const peers = json.peers; + for (const change of json.changes) { + for (const op of change.ops) { + const content = op.content as { type: string; len?: number; start_id?: string }; + if (content.type !== "delete" || content.start_id === undefined) continue; + const [counter, peerIndex] = content.start_id.split("@"); + const peer = peers == null ? peerIndex : peers[Number(peerIndex)]; + const start = Number(counter); + const length = Math.abs(content.len ?? 0); + if (peer === id.peer && id.counter >= start && id.counter < start + length) { + return true; + } + } + } + return false; +} + +/** Runs one scenario and returns the failure message, if any. */ +export function runActions( + rust: RustReference, + actions: readonly Action[], + options: ScenarioOptions = {}, +): string | undefined { + try { + new RichtextScenario(rust, options).run(actions); + return undefined; + } catch (error) { + return error instanceof DifferentialFailure + ? error.message + : error instanceof Error + ? (error.stack ?? error.message) + : String(error); + } +} + +/** + * Greedily removes actions while the scenario still fails the same way (same + * first failure line after replacing numbers), so shrinking does not wander + * into an unrelated divergence. + */ +export function shrinkActions( + rust: RustReference, + actions: readonly Action[], + options: ScenarioOptions = {}, +): Action[] { + let current = [...actions]; + const original = runActions(rust, current, options); + if (original === undefined) return current; + const kind = failureKind(original); + const stillFails = (candidate: readonly Action[]): boolean => { + const failure = runActions(rust, candidate, options); + return failure !== undefined && failureKind(failure) === kind; + }; + for (let chunk = Math.max(1, current.length >> 1); chunk >= 1; chunk >>= 1) { + let index = 0; + while (index < current.length) { + const candidate = [...current.slice(0, index), ...current.slice(index + chunk)]; + if (stillFails(candidate)) current = candidate; + else index += chunk; + } + } + return current; +} + +function failureKind(failure: string): string { + const line = failure.split("\n")[0]!; + const colon = line.indexOf(": "); + return (colon < 0 ? line : line.slice(colon + 2)).replace(/[0-9]+/g, "N").slice(0, 40); +} + +function boundaries(value: string, utf8 = false): number[] { + const output = [0]; + let offset = 0; + for (const scalar of value) { + offset += utf8 ? new TextEncoder().encode(scalar).length : scalar.length; + output.push(offset); + } + return output; +} + +function pickBoundary(value: string, raw: number, utf8 = false): number { + const all = boundaries(value, utf8); + return all[raw % all.length]!; +} + +function pickRange( + value: string, + raw: number, + length: number, + utf8 = false, +): [number, number] | undefined { + const all = boundaries(value, utf8); + if (all.length < 2) return undefined; + const start = raw % (all.length - 1); + const end = Math.min(all.length - 1, start + length); + return [all[start]!, all[end]!]; +} + +export function normalizeDelta(delta: readonly Delta[]): Delta[] { + const output: { insert: string; attributes?: Record }[] = []; + for (const item of delta) { + if (!("insert" in item) || item.insert.length === 0) continue; + const attributes = normalizeAttributes(item.attributes); + const previous = output.at(-1); + if ( + previous !== undefined && + JSON.stringify(previous.attributes) === JSON.stringify(attributes) + ) { + previous.insert += item.insert; + } else { + output.push( + attributes === undefined + ? { insert: item.insert } + : { insert: item.insert, attributes }, + ); + } + } + return output; +} + +function normalizeAttributes( + attributes: Readonly> | undefined, +): Record | undefined { + if (attributes === undefined) return undefined; + const entries = Object.entries(attributes) + .filter(([, value]) => value !== null && value !== undefined) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)); + return entries.length === 0 ? undefined : Object.fromEntries(entries); +} + +/** Applies a Quill-style text delta; `null` attribute values remove the key. */ +export function applyDelta( + document: readonly Delta[], + change: readonly Delta[], +): Delta[] { + const chars: { value: string; attributes: Record }[] = []; + for (const item of document) { + if (!("insert" in item)) continue; + for (let offset = 0; offset < item.insert.length; offset += 1) { + chars.push({ value: item.insert[offset]!, attributes: { ...item.attributes } }); + } + } + let index = 0; + for (const item of change) { + if ("insert" in item) { + const attributes = normalizeAttributes(item.attributes) ?? {}; + for (let offset = 0; offset < item.insert.length; offset += 1) { + chars.splice(index, 0, { + value: item.insert[offset]!, + attributes: { ...attributes }, + }); + index += 1; + } + } else if ("delete" in item) { + chars.splice(index, item.delete); + } else { + for (let offset = 0; offset < item.retain; offset += 1) { + const target = chars[index + offset]; + if (target === undefined) break; + for (const [key, value] of Object.entries(item.attributes ?? {})) { + if (value === null || value === undefined) delete target.attributes[key]; + else target.attributes[key] = value; + } + } + index += item.retain; + } + } + return normalizeDelta( + chars.map((char) => ({ insert: char.value, attributes: char.attributes })), + ); +} diff --git a/loro-js/tests/support/rust-reference.ts b/loro-js/tests/support/rust-reference.ts new file mode 100644 index 000000000..b3eef5ee7 --- /dev/null +++ b/loro-js/tests/support/rust-reference.ts @@ -0,0 +1,27 @@ +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; + +/** + * Loads the Rust implementation (`loro-crdt`, nodejs WASM target) as a + * differential-testing reference. Build it with + * `pnpm -C crates/loro-wasm build-dev` (or `build-release`), or point + * `LORO_WASM_NODEJS` at another build's `nodejs/index.js`. + * + * Returns `undefined` when no build is available so the differential suites can + * skip instead of failing on machines without a WASM toolchain. + */ +export function loadRustReference(): RustReference | undefined { + const path = + process.env.LORO_WASM_NODEJS ?? + fileURLToPath(new URL("../../../crates/loro-wasm/nodejs/index.js", import.meta.url)); + if (!existsSync(path)) { + if (process.env.LORO_REQUIRE_WASM_REFERENCE === "1") { + throw new Error(`Rust WASM reference build is missing: ${path}`); + } + return undefined; + } + return createRequire(import.meta.url)(path) as RustReference; +} + +export type RustReference = typeof import("../../src/index"); diff --git a/loro-js/tests/text-interop.test.ts b/loro-js/tests/text-interop.test.ts new file mode 100644 index 000000000..944b7f5f9 --- /dev/null +++ b/loro-js/tests/text-interop.test.ts @@ -0,0 +1,170 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, test } from "vitest"; + +import { LoroDoc, type Delta, type JsonSchema } from "../src/index"; +import { applyDelta, normalizeDelta } from "./support/richtext-differential"; + +const fixture = (name: string): Uint8Array => + new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); + +function doc(peer: number): LoroDoc { + const created = new LoroDoc(); + created.setPeerId(peer); + return created; +} + +function sync(from: LoroDoc, to: LoroDoc): void { + to.import(from.export({ mode: "update" })); +} + +describe("text interoperability with Rust", () => { + test("applies a remote delete by its position, like Rust's tracker", () => { + // Rust's WASM build recorded `start_id` 0@7 for a delete of a, 😀, x: it + // advanced the ID by the UTF-16 length of 😀, so the recorded IDs name b + // instead of x. Rust applies the delete by position and keeps "b". + const { text, updates } = JSON.parse( + readFileSync( + new URL("./fixtures/rust/text-delete-positions.json", import.meta.url), + "utf8", + ), + ) as { text: string; updates: JsonSchema }; + expect(text).toBe("b"); + + const fromJson = new LoroDoc(); + fromJson.importJsonUpdates(updates); + expect(fromJson.getText("text").toString()).toBe("b"); + + const fromBinary = new LoroDoc(); + fromBinary.import(fixture("text-delete-positions.blob")); + expect(fromBinary.getText("text").toString()).toBe("b"); + }); + + test("orders a concurrent insert before a later sibling subtree", () => { + const peer1 = doc(1); + const peer2 = doc(2); + const peer3 = doc(3); + peer2.getText("t").insert(0, "9"); + peer2.commit(); + peer1.getText("t").insert(0, "b"); + peer1.commit(); + sync(peer1, peer2); + expect(peer2.getText("t").toString()).toBe("b9"); + peer2.getText("t").insert(1, "é"); + peer2.commit(); + sync(peer1, peer3); + peer3.getText("t").insert(1, "a"); + peer3.commit(); + + // "a" follows b and é (b's subtree) but precedes 9, which is not in it. + for (const target of [peer1, peer2, peer3]) { + for (const source of [peer1, peer2, peer3]) { + if (source !== target) sync(source, target); + } + expect(target.getText("t").toString()).toBe("béa9"); + } + }); + + test("writes multi-run deletes last run first, like Rust", () => { + const owner = doc(1); + const text = owner.getText("t"); + text.insert(0, "abcd"); + text.insert(2, "X"); + owner.commit(); + text.delete(1, 3); // b, X, c: three ID runs + owner.commit(); + const ops = owner + .exportJsonUpdates() + .changes.at(-1)! + .ops.map((op) => op.content); + // Output of Rust (loro-crdt) for the same edits; peer 1 is index 0. + expect(ops.slice(-3)).toEqual([ + { type: "delete", pos: 3, len: 1, start_id: "2@0" }, + { type: "delete", pos: 2, len: 1, start_id: "4@0" }, + { type: "delete", pos: 1, len: 1, start_id: "1@0" }, + ]); + expect(text.toString()).toBe("ad"); + }); + + test("merges consecutive text inserts in a transaction, like Rust", () => { + const owner = doc(1); + const text = owner.getText("t"); + text.insert(0, "中文"); + owner.commit(); + text.insert(1, "😀"); + text.insert(3, "ab"); + owner.commit(); + const ops = owner + .exportJsonUpdates() + .changes.flatMap((change) => change.ops.map((op) => op.content)); + expect(ops).toEqual([ + { type: "insert", pos: 0, text: "中文" }, + { type: "insert", pos: 1, text: "😀ab" }, + ]); + }); + + test("checkout events restore an element deleted twice only once", () => { + const peer1 = doc(1); + const peer2 = doc(2); + peer2.getText("t").insert(0, "中文"); + peer2.commit(); + const base = peer2.frontiers(); + sync(peer2, peer1); + peer2.getText("t").delete(0, 2); + peer2.commit(); + peer1.getText("t").delete(1, 1); + peer1.commit(); + sync(peer2, peer1); + peer1.getText("t").insert(0, "xyz"); + peer1.commit(); + + let shadow: Delta[] = peer1.getText("t").toDelta(); + peer1.subscribe((batch) => { + for (const event of batch.events) { + if (event.diff.type === "text") { + shadow = applyDelta(shadow, event.diff.diff as Delta[]); + } + } + }); + peer1.checkout(base); + expect(peer1.getText("t").toString()).toBe("中文"); + expect(normalizeDelta(shadow)).toEqual(normalizeDelta(peer1.getText("t").toDelta())); + peer1.checkoutToLatest(); + expect(normalizeDelta(shadow)).toEqual(normalizeDelta(peer1.getText("t").toDelta())); + }); + + test("text cursors follow Rust's position semantics", () => { + const owner = doc(1); + const text = owner.getText("t"); + const empty = text.getCursor(0, 0)!; + expect(empty.pos()).toBeUndefined(); + expect(empty.side()).toBe(-1); + + text.insert(0, "a😀bc"); + owner.commit(); + const end = text.getCursor(text.length, -1)!; + expect(end.pos()).toBeUndefined(); + expect(end.side()).toBe(1); + expect(owner.getCursorPos(end)).toEqual({ offset: 5, side: 1 }); + expect(text.getCursor(2, 0)).toBeUndefined(); + + // The encoded cursor matches Rust byte for byte: its origin is the Unicode + // position (2), not the UTF-16 position (3). + expect([...text.getCursor(3, 1)!.encode()]).toEqual([1, 1, 4, 0, 1, 116, 0, 2, 2]); + expect([...text.getCursor(5, 1)!.encode()]).toEqual([0, 0, 1, 116, 0, 2, 5]); + + // A visible target reports its own offset for every side. + const right = text.getCursor(3, 1)!; + expect(right.pos()).toEqual({ peer: "1", counter: 2 }); + expect(owner.getCursorPos(right)).toEqual({ offset: 3, side: 1 }); + + // A deleted target reports the length before it, on the left side. + text.delete(1, 3); + owner.commit(); + const deleted = owner.getCursorPos(right)!; + expect(deleted.offset).toBe(1); + expect(deleted.side).toBe(-1); + expect(deleted.update?.pos()).toEqual({ peer: "1", counter: 3 }); + expect(deleted.update?.side()).toBe(-1); + }); +}); diff --git a/package.json b/package.json index f7aae33fe..cd851ff4a 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,10 @@ "check-all": "cargo hack check --each-feature", "build": "cargo build", "test": "cargo nextest run --features=test_utils,jsonpath --no-fail-fast && cargo test --doc", - "test-all": "pnpm test && pnpm test-fractional-index && pnpm release-wasm && pnpm test-loom", + "test-all": "pnpm test && pnpm test-fractional-index && pnpm release-wasm && pnpm test-loro-js && pnpm test-loom", "test-wasm": "cd crates/loro-wasm && pnpm i && pnpm build-dev", "test-fractional-index": "pnpm --filter @loro-dev/fractional-index install --frozen-lockfile && pnpm --filter @loro-dev/fractional-index check", + "test-loro-js": "pnpm --filter loro.js install --frozen-lockfile && LORO_REQUIRE_WASM_REFERENCE=1 pnpm --filter loro.js test", "test-loom": "LOOM_MAX_PREEMPTIONS=2 RUSTFLAGS='--cfg loom' cargo test -p loro --test multi_thread_test --release", "test-loom-reproduce": "LOOM_MAX_PREEMPTIONS=2 LOOM_LOG=info LOOM_LOCATION=1 LOOM_CHECKPOINT_INTERVAL=1 LOOM_CHECKPOINT_FILE=loom_test.json RUSTFLAGS='--cfg loom' cargo test -p loro --test multi_thread_test --release", "watch-loom": "RUSTFLAGS='--cfg loom' bacon", From 829db4fe16e7c2073529cc735074b7b5b6c11c7a Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 04:29:44 +0800 Subject: [PATCH 02/13] fix(loro.js): stop a concurrent insert at the end of its origin's subtree The indexed Fugue path jumped from the last direct child of `originLeft` to the origin-right element. Rust's scan stops at the first concurrent element whose origin is further left, so the insert could land after that element. Replicas then disagreed depending on import order, and all of them could disagree with loro-crdt. Check whether the interval's last element descends from `originLeft` and otherwise binary-search the boundary. The descent test skips implicit runs through per-peer sorted explicit counters. Rationale and cost: context/loro-js-performance.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-fugue-origin-subtree.md | 5 + context/loro-js-performance.md | 11 ++- loro-js/src/runtime/containers.ts | 104 +++++++++++++++++++- loro-js/tests/fugue-index.test.ts | 106 ++++++++++++++++++++- 4 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 .changeset/loro-js-fugue-origin-subtree.md diff --git a/.changeset/loro-js-fugue-origin-subtree.md b/.changeset/loro-js-fugue-origin-subtree.md new file mode 100644 index 000000000..fecda9748 --- /dev/null +++ b/.changeset/loro-js-fugue-origin-subtree.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +Fix a `loro.js` Text/List convergence bug with concurrent inserts. When an insert was concurrent with a run typed after the same character and with another concurrent element whose origin was further left, `loro.js` put the insert after that element instead of at the end of the run. Replicas could disagree with each other (depending on the order in which they received the updates) and with `loro-crdt`. `loro.js` now places these inserts where Rust does. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 896482166..14dda5920 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -103,7 +103,16 @@ JavaScript constant factor. keep their single-child edge implicit, and `SequenceIndex` can skip an entire future ID run while finding the next causally included element. Ordinary local edits keep the smaller unindexed path. MovableList continues to use the scan - because moves break the origin-tree physical preorder. + because moves break the origin-tree physical preorder. Sibling subtrees are + contiguous, so the gap between two direct children belongs to the earlier + child. After the last child the interval can also hold concurrent elements + whose origin is left of `originLeft`; Rust's scan stops before them. The index + therefore checks whether the interval's last element descends from + `originLeft` and otherwise binary-searches the boundary. The descent test + walks origin-left links but jumps over each implicit run through per-peer + sorted counters of explicit (non-consecutive) elements, so it costs + O(explicit links · log n), like Rust's span-based scan, instead of one probe + per scalar in a long concurrent run. - Merging adjacent changes appends only the new operations and key-table entries to the retained record. The cached operation length, peer end, frontier set, operation indexes, and subscriber update slice are updated incrementally, so diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 9b3979b91..c6a49a6e2 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -2870,6 +2870,12 @@ interface FugueOriginEntry { interface FugueOriginIndex { structureVersion: number; readonly explicitChildren: Map; + /** + * Sorted counters, per peer, of the elements that are not the implicit child + * of their predecessor ID. Every other element is linked to `counter - 1`, so + * an origin-left walk can jump from any element to the head of its run. + */ + readonly explicitCounters: Map; } interface FugueInsertionResult { @@ -3036,6 +3042,7 @@ function indexedFugueInsertion( const parentRightIndex = directRightParentIndex(sequence, originLeft, originRight); let insertIndex = startIndex; let scanning = false; + let afterLastChild = false; for (let childIndex = 0; childIndex < candidates.length; childIndex += 1) { const { entry: other } = candidates[childIndex]!; if (sameOptionalId(other.originRight, originRight)) { @@ -3054,7 +3061,38 @@ function indexedFugueInsertion( } if (!scanning) { - insertIndex = candidates[childIndex + 1]?.index ?? originRightIndex; + const next = candidates[childIndex + 1]; + insertIndex = next?.index ?? originRightIndex; + afterLastChild = next === undefined; + } + } + + // Sibling subtrees are contiguous, so the gap between two direct children + // belongs to the earlier one. After the last child, the interval can also + // hold concurrent elements whose origin is left of `originLeft`; the scan + // stops before them, so the insertion must too. + const lastChild = candidates.at(-1); + if (afterLastChild && originLeft !== undefined && lastChild !== undefined) { + const left = sequence.findByIdRaw(originLeft); + const leftIndex = left === undefined ? undefined : sequence.physicalIndexOf(left); + if (leftIndex === undefined) return undefined; + const inSubtree = (index: number): boolean => + fugueDescendsFrom( + sequence, + originIndex, + sequence.atPhysicalRaw(index)!.id, + originLeft, + leftIndex, + ); + if (!inSubtree(originRightIndex - 1)) { + let low = lastChild.index + 1; + let high = originRightIndex - 1; + while (low < high) { + const middle = (low + high) >>> 1; + if (inSubtree(middle)) low = middle + 1; + else high = middle; + } + insertIndex = low; } } @@ -3066,6 +3104,54 @@ function indexedFugueInsertion( }; } +/** + * Whether `ancestor`, at physical index `ancestorIndex`, is on the origin-left + * chain of `id`. Ancestors precede their descendants, so the walk stops at the + * first origin left of `ancestorIndex`; runs of implicit children are skipped + * through `explicitCounters`. + */ +function fugueDescendsFrom( + sequence: SequenceIndex, + index: FugueOriginIndex, + id: CodecId, + ancestor: CodecId, + ancestorIndex: number, +): boolean { + let current = id; + for (;;) { + const counters = index.explicitCounters.get(current.peer); + let head = current.counter; + if (counters !== undefined) { + let low = 0; + let high = counters.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (counters[middle]! <= current.counter) low = middle + 1; + else high = middle; + } + if (low > 0) head = counters[low - 1]!; + } + if ( + ancestor.peer === current.peer && + head <= ancestor.counter && + ancestor.counter < current.counter + ) { + return true; + } + const parent = sequence.findByIdRaw({ + peer: current.peer, + counter: head, + })?.originLeft; + if (parent === undefined) return false; + if (parent.peer === ancestor.peer && parent.counter === ancestor.counter) return true; + const element = sequence.findByIdRaw(parent); + const parentIndex = + element === undefined ? undefined : sequence.physicalIndexOf(element); + if (parentIndex === undefined || parentIndex < ancestorIndex) return false; + current = parent; + } +} + function getFugueOriginIndex( sequence: SequenceIndex, ): FugueOriginIndex { @@ -3079,6 +3165,7 @@ function getFugueOriginIndex( const rebuilt: FugueOriginIndex = { structureVersion: sequence.structureVersion, explicitChildren: new Map(), + explicitCounters: new Map(), }; sequence.forEachPhysicalRaw((element) => { recordFugueOriginEntry(rebuilt, element.id, element.originLeft, element.originRight); @@ -3147,6 +3234,21 @@ function recordFugueOriginEntry( ) { return; } + const counters = index.explicitCounters.get(id.peer); + if (counters === undefined) { + index.explicitCounters.set(id.peer, [id.counter]); + } else if (counters.at(-1)! < id.counter) { + counters.push(id.counter); + } else { + let low = 0; + let high = counters.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (counters[middle]! < id.counter) low = middle + 1; + else high = middle; + } + if (counters[low] !== id.counter) counters.splice(low, 0, id.counter); + } const key = optionalSequenceIdKey(originLeft); const children = index.explicitChildren.get(key); const entry = { diff --git a/loro-js/tests/fugue-index.test.ts b/loro-js/tests/fugue-index.test.ts index 7b59fa27c..dcac55c6c 100644 --- a/loro-js/tests/fugue-index.test.ts +++ b/loro-js/tests/fugue-index.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test, vi } from "vitest"; -import { LoroText } from "../src/index"; +import { LoroDoc, LoroText } from "../src/index"; import { SequenceIndex } from "../src/runtime/sequence-index"; describe("Fugue origin index", () => { @@ -23,4 +23,108 @@ describe("Fugue origin index", () => { expect(longRunProbes).toBeLessThanOrEqual(shortRunProbes + 2); expect(longRunProbes).toBeLessThan(8); }); + + test("finds the end of a concurrent run under the origin without probing it", () => { + const probeCount = (length: number): number => { + const text = new LoroText(); + text._insertFugue(0, "x", { peer: 1n, counter: 0 }, 0, new Map()); + // A concurrent root: it sorts after x and stays after x's subtree. + text._insertFugue(0, "n", { peer: 4n, counter: 0 }, 0, new Map()); + text._insertFugue( + 1, + "a".repeat(length), + { peer: 1n, counter: 1 }, + 1, + new Map([[1n, 1]]), + ); + expect(text.toString()).toBe(`x${"a".repeat(length)}n`); + + const atPhysical = vi.spyOn(SequenceIndex.prototype, "atPhysicalRaw"); + text._insertFugue(1, "b", { peer: 2n, counter: 0 }, 1, new Map([[1n, 1]])); + const probes = atPhysical.mock.calls.length; + atPhysical.mockRestore(); + + expect(text.toString()).toBe(`x${"a".repeat(length)}bn`); + return probes; + }; + + const shortRunProbes = probeCount(128); + const longRunProbes = probeCount(16_384); + // A binary search over the interval, not a scan of the run. + expect(longRunProbes).toBeLessThanOrEqual(shortRunProbes + 16); + expect(longRunProbes).toBeLessThan(64); + }); + + // An insert concurrent with a run under its origin must stop at the first + // concurrent element whose origin is further left, as Rust's scan does. + // The indexed path used to jump to the origin-right element instead. + test.each(["list", "text"] as const)( + "orders a concurrent %s insert before an element from an older origin", + (kind) => { + const peers = [1, 2, 3, 4].map((peer) => { + const doc = new LoroDoc(); + doc.setPeerId(peer); + return doc; + }); + const [p1, p2, p3, p4] = peers as [LoroDoc, LoroDoc, LoroDoc, LoroDoc]; + const insert = (doc: LoroDoc, pos: number, value: string): void => { + if (kind === "list") doc.getList("seq").insert(pos, value); + else doc.getText("seq").insert(pos, value); + doc.commit(); + }; + const read = (doc: LoroDoc): string => + kind === "list" + ? (doc.getList("seq").toArray() as string[]).join("") + : doc.getText("seq").toString(); + + insert(p1, 0, "A"); + insert(p2, 0, "B"); + insert(p4, 0, "N"); + p3.import(p1.export({ mode: "update" })); + p3.import(p4.export({ mode: "update" })); + expect(read(p3)).toBe("AN"); + // Y: origin left A, origin right N. + insert(p3, 1, "Y"); + p1.import(p2.export({ mode: "update" })); + p1.import(p4.export({ mode: "update" })); + expect(read(p1)).toBe("ABN"); + // X: origin left A, origin right B. + insert(p1, 1, "X"); + p1.import(p3.export({ mode: "update" })); + + const replay = new LoroDoc(); + for (const peer of [p4, p3, p2, p1]) replay.import(peer.export({ mode: "update" })); + // loro-crdt gives AXYBN for every import order. + expect(read(replay)).toBe("AXYBN"); + expect(read(p1)).toBe("AXYBN"); + }, + ); + + test("places an insert concurrent with a run before a concurrent root, as Rust does", () => { + const [p1, p2, p4] = [1, 2, 4].map((peer) => { + const doc = new LoroDoc(); + doc.setPeerId(peer); + return doc; + }) as [LoroDoc, LoroDoc, LoroDoc]; + p1.getText("text").insert(0, "x"); + p1.commit(); + p2.import(p1.export({ mode: "update" })); + p1.getText("text").insert(1, "aaaa"); + p1.commit(); + p4.getText("text").insert(0, "n"); + p4.commit(); + p2.getText("text").insert(1, "b"); + p2.commit(); + + for (const order of [ + [p1, p4, p2], + [p2, p4, p1], + [p4, p1, p2], + ]) { + const doc = new LoroDoc(); + for (const peer of order) doc.import(peer.export({ mode: "update" })); + // loro-crdt gives xaaaabn; the indexed path used to give xaaaanb. + expect(doc.getText("text").toString()).toBe("xaaaabn"); + } + }); }); From ae2a67fd04999302e45c3438b5edaf63d58924e8 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 09:08:43 +0800 Subject: [PATCH 03/13] perf(loro.js): jump between ID runs when finding a Fugue subtree end An element right after its ID predecessor has it as origin-left, so only run starts can end a sibling's subtree (context/loro-js-performance.md). Importing a concurrent character after a 512k-character run drops from ~9.5 ms to ~0.2 ms. Adds text-concurrent-insert-after-long-run to bench:complexity. Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 4 ++ loro-js/benchmarks/complexity.mjs | 18 +++++ loro-js/src/runtime/containers.ts | 38 ++++++----- loro-js/src/runtime/sequence-index.ts | 96 ++++++++++++++------------- loro-js/tests/indexes.test.ts | 45 +++++++++++++ loro-js/tests/text-interop.test.ts | 39 +++++++++++ 6 files changed, 177 insertions(+), 63 deletions(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 38a37474a..4c1ec6283 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -108,6 +108,10 @@ JavaScript constant factor. the origin-right bound: when that bound is only the next causally included element, later concurrent elements of an ancestor's subtree can sit before it. The walk stops at the first element whose origin-left precedes the sibling. + Inside a physical ID run every element's origin-left is its predecessor, so + the walk jumps between run starts (`SequenceIndex.nextPhysicalIdRunStart`, + skipping single-run treap subtrees): O((runs in the subtree + 1) log n), and + O(log n) after a long typed run (`text-concurrent-insert-after-long-run`). - An imported Text delete is resolved by its position in the op's causal view, like Rust's tracker (`LoroText._deleteTargets`): O(log n + runs) through `visibleIdRuns` or the cached causal view. The recorded `start_id` is only a diff --git a/loro-js/benchmarks/complexity.mjs b/loro-js/benchmarks/complexity.mjs index 46abfa8e1..489164db0 100644 --- a/loro-js/benchmarks/complexity.mjs +++ b/loro-js/benchmarks/complexity.mjs @@ -509,6 +509,24 @@ for (const size of sizes) { }); unsubscribeImport(); + // A concurrent character that Fugue orders after a sibling's whole run of + // 64 * size characters. Peer 1's character first builds the origin index. + const concurrentUpdate = (peer, text) => { + const doc = new LoroDoc(); + doc.setPeerId(peer); + doc.getText("text").insert(0, text); + doc.commit(); + return doc.export({ mode: "update" }); + }; + const concurrentTarget = new LoroDoc(); + concurrentTarget.import(concurrentUpdate(2, "x".repeat(size * 64))); + concurrentTarget.import(concurrentUpdate(1, "y")); + const concurrentTail = concurrentUpdate(3, "z"); + measure("text-concurrent-insert-after-long-run", size, () => { + concurrentTarget.import(concurrentTail); + return concurrentTarget.getText("text").length; + }); + const checkoutTextDoc = new LoroDoc(); checkoutTextDoc.setPeerId(3); checkoutTextDoc.getText("text").insert(0, "x".repeat(size)); diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index caa37a646..51c59c6d6 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -3139,30 +3139,32 @@ function indexedFugueInsertion( * the next causally included element, later concurrent elements that belong to * an ancestor's subtree can sit before it. An element is a descendant exactly * when its origin-left lies inside the scanned subtree. + * + * Only the first element of a physical ID run needs that check. An element + * right after its ID predecessor has that predecessor as origin-left: the op + * was placed right after its origin-left in its causal view, the predecessor + * is in that view, and only concurrent elements can come between an element + * and its origin-left. So the walk jumps from run start to run start and costs + * O((runs in the subtree + 1) log n), not one step per scalar. */ function fugueSubtreeEnd( sequence: SequenceIndex, rootIndex: number, limit: number, ): number { - let end = rootIndex + 1; - if (end >= limit) return limit; - let previous = sequence.atPhysicalRaw(rootIndex)!.id; - sequence.forEachPhysicalFrom(end, (element, index) => { - if (index >= limit) return false; - const left = element.originLeft; - if (left === undefined) return false; - if (left.peer !== previous.peer || left.counter !== previous.counter) { - const leftElement = sequence.findByIdRaw(left); - const leftIndex = - leftElement === undefined ? undefined : sequence.physicalIndexOf(leftElement); - if (leftIndex === undefined || leftIndex < rootIndex) return false; - } - previous = element.id; - end = index + 1; - return undefined; - }); - return end; + let index = rootIndex + 1; + while (index < limit) { + const start = sequence.nextPhysicalIdRunStart(index, limit); + if (start >= limit) return limit; + const left = sequence.atPhysicalRaw(start)!.originLeft; + if (left === undefined) return start; + const leftElement = sequence.findByIdRaw(left); + const leftIndex = + leftElement === undefined ? undefined : sequence.physicalIndexOf(leftElement); + if (leftIndex === undefined || leftIndex < rootIndex) return start; + index = start + 1; + } + return limit; } function getFugueOriginIndex( diff --git a/loro-js/src/runtime/sequence-index.ts b/loro-js/src/runtime/sequence-index.ts index 9d7d92290..249eef1fc 100644 --- a/loro-js/src/runtime/sequence-index.ts +++ b/loro-js/src/runtime/sequence-index.ts @@ -423,51 +423,16 @@ export class SequenceIndex { }); } - /** Visits elements in physical order from `start` until `visit` returns false. */ - forEachPhysicalFrom( - start: number, - visit: (element: T, index: number) => boolean | void, - ): void { - if (!Number.isSafeInteger(start) || start < 0 || start >= this.allLength) return; - const stack: SequenceNode[] = []; - let node = this.#root; - let base = 0; - let current: SequenceNode | undefined; - let offset = 0; - while (node !== undefined) { - const leftCount = allCount(node.left); - if (start < base + leftCount) { - stack.push(node); - node = node.left; - continue; - } - const ownStart = base + leftCount; - if (start < ownStart + nodeLength(node)) { - current = node; - offset = start - ownStart; - break; - } - base = ownStart + nodeLength(node); - node = node.right; - } - let index = start; - while (current !== undefined) { - for (; offset < nodeLength(current); offset += 1) { - if (visit(nodeElement(current, offset), index) === false) return; - index += 1; - } - let next = current.right; - if (next === undefined) { - current = stack.pop(); - } else { - while (next.left !== undefined) { - stack.push(next); - next = next.left; - } - current = next; - } - offset = 0; - } + /** + * Returns the first physical index in `[start, limit)` whose element does not + * continue the ID run of the element before it, or `limit` if there is none. + * Expected O(log n): treap subtrees that are one ID run are skipped whole. + */ + nextPhysicalIdRunStart(start: number, limit = this.allLength): number { + const end = Math.min(limit, this.allLength); + if (start >= end) return end; + const found = findPhysicalIdRunStart(this.#root, 0, start, undefined); + return found === undefined || found > end ? end : found; } findNextIncludedPhysical( @@ -2233,6 +2198,47 @@ function nodeCounter( : (node.element as T).id.counter; } +/** + * Finds the first index `>= start` in this subtree whose element starts an ID + * run. `before` is the ID of the element physically before the subtree. + */ +function findPhysicalIdRunStart( + node: SequenceNode | undefined, + nodeStart: number, + start: number, + before: SequenceId | undefined, +): number | undefined { + if (node === undefined || nodeStart + node.allCount <= start) return undefined; + if (node.idRunCount === 1) { + return start <= nodeStart && + (before === undefined || !sequenceIdsContinue(before, node.firstId)) + ? nodeStart + : undefined; + } + + const fromLeft = findPhysicalIdRunStart(node.left, nodeStart, start, before); + if (fromLeft !== undefined) return fromLeft; + + const ownStart = nodeStart + allCount(node.left); + let previous = node.left?.lastId ?? before; + const length = nodeLength(node); + const firstOffset = Math.max(0, start - ownStart); + if (firstOffset < length) { + if (firstOffset > 0) previous = nodeId(node, firstOffset - 1); + for (let offset = firstOffset; offset < length; offset += 1) { + const id = nodeId(node, offset); + if (previous === undefined || !sequenceIdsContinue(previous, id)) { + return ownStart + offset; + } + // The rest of a single-run node continues this element. + if (node.ownIdRunCount === 1) break; + previous = id; + } + } + + return findPhysicalIdRunStart(node.right, ownStart + length, start, node.ownLastId); +} + function findNextIncludedPhysical( node: SequenceNode | undefined, nodeStart: number, diff --git a/loro-js/tests/indexes.test.ts b/loro-js/tests/indexes.test.ts index 72834f149..9bb9aea08 100644 --- a/loro-js/tests/indexes.test.ts +++ b/loro-js/tests/indexes.test.ts @@ -571,6 +571,51 @@ describe("runtime indexes", () => { } }); + test("finds physical ID run starts", () => { + const sequence = new SequenceIndex(); + const physical: TestElement[] = []; + let random = 0x51_7c_c1_b7; + const nextRandom = (): number => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return random >>> 0; + }; + const nextCounter = new Map(); + for (let step = 0; step < 400; step += 1) { + const peer = BigInt(nextRandom() % 3); + const length = 1 + (nextRandom() % 40); + const start = nextCounter.get(peer) ?? 0; + nextCounter.set(peer, start + length); + const elements = Array.from({ length }, (_, offset) => ({ + id: { peer, counter: start + offset }, + deleted: false, + deletedBy: [], + value: "x", + })); + const position = nextRandom() % (physical.length + 1); + sequence.insertAtPhysical(position, elements); + physical.splice(position, 0, ...elements); + } + + const startsRun = (index: number): boolean => { + if (index === 0) return true; + const previous = physical[index - 1]!.id; + const current = physical[index]!.id; + return previous.peer !== current.peer || previous.counter + 1 !== current.counter; + }; + for (let start = 0; start <= physical.length; start += 1) { + const limit = start + (nextRandom() % 200); + let expected = start; + while (expected < Math.min(limit, physical.length) && !startsRun(expected)) { + expected += 1; + } + expect(sequence.nextPhysicalIdRunStart(start, limit)).toBe( + Math.min(expected, limit, physical.length), + ); + } + }); + test("ranges across an excluded element", () => { const sequence = new SequenceIndex(); const base = Array.from({ length: 6 }, (_, counter) => ({ diff --git a/loro-js/tests/text-interop.test.ts b/loro-js/tests/text-interop.test.ts index 944b7f5f9..1a10fa42b 100644 --- a/loro-js/tests/text-interop.test.ts +++ b/loro-js/tests/text-interop.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test } from "vitest"; import { LoroDoc, type Delta, type JsonSchema } from "../src/index"; import { applyDelta, normalizeDelta } from "./support/richtext-differential"; +import { loadRustReference, type RustReference } from "./support/rust-reference"; const fixture = (name: string): Uint8Array => new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); @@ -65,6 +66,44 @@ describe("text interoperability with Rust", () => { } }); + test("orders a concurrent insert after a sibling subtree of several ID runs", () => { + // z goes after the whole subtree of peer 2's first "a": its later run of a, + // the B run, and peer 4's c run, whose starts are the only elements + // `fugueSubtreeEnd` checks. + const scenario = ({ LoroDoc: Doc }: { LoroDoc: typeof LoroDoc }): string => { + const make = (peer: number): LoroDoc => { + const created = new Doc(); + created.setPeerId(peer); + return created; + }; + const [p1, p2, p3, p4, target] = [1, 2, 3, 4, 9].map(make) as [ + LoroDoc, + LoroDoc, + LoroDoc, + LoroDoc, + LoroDoc, + ]; + p2.getText("t").insert(0, "a".repeat(100)); + p2.commit(); + sync(p2, p4); + p2.getText("t").insert(50, "B".repeat(40)); + p2.commit(); + p4.getText("t").insert(80, "c".repeat(30)); + p4.commit(); + p1.getText("t").insert(0, "y"); + p1.commit(); + p3.getText("t").insert(0, "z"); + p3.commit(); + for (const source of [p2, p4, p1, p3]) sync(source, target); + return target.getText("t").toString(); + }; + const expected = `ya${"a".repeat(49)}${"B".repeat(40)}${"a".repeat(30)}${"c".repeat(30)}${"a".repeat(20)}z`; + expect(scenario({ LoroDoc })).toBe(expected); + // The same edits in Rust, when its WASM build is available. + const rust: RustReference | undefined = loadRustReference(); + expect(rust === undefined ? expected : scenario(rust)).toBe(expected); + }); + test("writes multi-run deletes last run first, like Rust", () => { const owner = doc(1); const text = owner.getText("t"); From b6db4d10a712dee76301fe4f2c06f15c3b401f51 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 09:13:50 +0800 Subject: [PATCH 04/13] docs(loro.js): read 0.2 data like Rust and document the upgrade Decision and rationale: context/loro-js-rust-differential.md ("Data written by loro.js 0.2"). The changeset becomes major; README "Upgrading from 0.2" lists what changes and how to migrate. Fixtures written by loro.js 0.2.1 pin how loro.js and Rust read its updates, snapshots, and cursors. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-text-interop.md | 13 ++- context/loro-js-rust-differential.md | 38 +++++--- loro-js/AGENTS.md | 5 + loro-js/README.md | 41 ++++++++ loro-js/scripts/write-legacy-fixtures.mjs | 65 +++++++++++++ .../loro-js-0.2/cursor-emoji.cursor.blob | Bin 0 -> 9 bytes .../loro-js-0.2/cursor-emoji.snapshot.blob | Bin 0 -> 248 bytes .../tests/fixtures/loro-js-0.2/expected.json | 4 + .../fixtures/loro-js-0.2/text-concurrent.json | 88 ++++++++++++++++++ .../loro-js-0.2/text-concurrent.snapshot.blob | Bin 0 -> 369 bytes .../loro-js-0.2/text-concurrent.update.blob | Bin 0 -> 282 bytes loro-js/tests/legacy-data.test.ts | 62 ++++++++++++ 12 files changed, 300 insertions(+), 16 deletions(-) create mode 100644 loro-js/scripts/write-legacy-fixtures.mjs create mode 100644 loro-js/tests/fixtures/loro-js-0.2/cursor-emoji.cursor.blob create mode 100644 loro-js/tests/fixtures/loro-js-0.2/cursor-emoji.snapshot.blob create mode 100644 loro-js/tests/fixtures/loro-js-0.2/expected.json create mode 100644 loro-js/tests/fixtures/loro-js-0.2/text-concurrent.json create mode 100644 loro-js/tests/fixtures/loro-js-0.2/text-concurrent.snapshot.blob create mode 100644 loro-js/tests/fixtures/loro-js-0.2/text-concurrent.update.blob create mode 100644 loro-js/tests/legacy-data.test.ts diff --git a/.changeset/loro-js-text-interop.md b/.changeset/loro-js-text-interop.md index ec51a6bc3..7e9b31ab6 100644 --- a/.changeset/loro-js-text-interop.md +++ b/.changeset/loro-js-text-interop.md @@ -1,11 +1,14 @@ --- -"loro.js": patch +"loro.js": major --- -Text edits now converge with Rust (`loro-crdt`) in cases a randomized comparison with the Rust WASM build found: +Text now follows Rust (`loro-crdt`) everywhere, including when reading documents written by earlier loro.js versions. This breaks documents edited only with loro.js that have concurrent text edits or stored cursors: read "Upgrading from 0.2" in the README before upgrading. -- A concurrent insert after a sibling's subtree was placed after later, unrelated concurrent text. Replicas that merged such edits in different orders could end with different text. -- An imported delete is now applied by its position, as Rust does, instead of by its recorded `start_id`. Rust's WASM build records a wrong `start_id` when the deleted text spans astral characters (for example, emoji) inserted in separate edits, so loro.js deleted different characters than Rust. +- **Breaking:** concurrent text inserts are ordered like Rust. Earlier versions could place an insert after a sibling's subtree behind later, unrelated concurrent text, so a history with such inserts can now read differently (for example `béa9` instead of `bé9a`). Earlier loro.js replicas that merged such edits in different orders, and documents shared with `loro-crdt` peers, had already diverged; they now agree with Rust. +- **Breaking:** text cursors follow Rust. An encoded cursor stores the Unicode position of its target, not the UTF-16 position, so a cursor encoded by an earlier version can resolve to another offset. A cursor at the end or in empty text has no target, `getCursorPos` returns the target's own offset for every side, and a deleted target reports the length of the text before it with side `-1`. Get new cursors after upgrading. +- An imported delete is applied by its position, as Rust does, instead of by its recorded `start_id` (Rust's WASM build up to 1.16.3 can record a wrong one around astral characters, loro-dev/loro#1149). - `checkout`, `diff`, and `revertTo` no longer restore a character twice when two concurrent deletes removed it. -- Text cursors follow Rust: a cursor at the end or in empty text has no target, `getCursorPos` returns the target's own offset for every side, and a deleted target reports the length of the text before it with side `-1`. The encoded cursor matches Rust's bytes. - Consecutive inserts in one transaction are merged into one op, and a delete that spans several ID runs writes the last run first, as Rust does. +- Importing a concurrent insert next to a long run of text no longer walks the whole run. + +Upgrading: a snapshot written by an earlier version keeps its current text, but its history (updates, JSON updates, checkout of older versions) is read like Rust, and a replica loaded from a snapshot and one loaded from updates can keep different text after new edits. Migrate every replica the same way; the safest path is to read the final state with the old version and build a new document from it. diff --git a/context/loro-js-rust-differential.md b/context/loro-js-rust-differential.md index 898e19ce4..c72addc6f 100644 --- a/context/loro-js-rust-differential.md +++ b/context/loro-js-rust-differential.md @@ -60,20 +60,36 @@ regression test. ## Rust behaviors the harness works around -The Rust reference has bugs of its own in UTF-16 (WASM) builds: +WASM builds before loro-dev/loro#1135 had bugs of their own. The harness still +tolerates them, so it also runs against older reference builds: -- A subscribed transaction that merges deletes of astral text can panic with +- A subscribed transaction that merges deletes of astral text could panic with "Op/hint length mismatch" (`change_to_diff` in - `crates/loro-internal/src/txn.rs`), even after unsubscribing. The harness - subscribes only to loro.js documents. + `crates/loro-internal/src/txn.rs`). The harness subscribes only to loro.js + documents. - `get_text_entity_ranges` advanced the recorded delete `start_id` by the UTF-16 - length of astral text, so Rust can write a delete whose `start_id` names the - wrong elements. Rust applies deletes by position, and so does loro.js - (`LoroText._deleteTargets`). Rust still resolves a deleted cursor target - through the recorded IDs, so the harness skips cursors Rust cannot resolve. -- Rust's shallow import falls back to the recorded delete IDs when they disagree - with the positions, which can diverge from its own full-history import. The - harness compares loro.js with the source document instead. + length of astral text, so Rust could write a delete whose `start_id` names the + wrong elements; such deletes stay in existing histories (loro-dev/loro#1149). + Rust applies deletes by position, and so does loro.js + (`LoroText._deleteTargets`). Rust resolves a deleted cursor target through the + recorded IDs, so the harness skips cursors Rust cannot resolve. +- Rust's shallow import labels placeholders with the recorded delete IDs, which + can diverge from its own full-history import when they are wrong. The harness + compares loro.js with the source document instead. Snapshot-import checkout (`snapshotCheckout`) is off by default until loro.js can check out a document imported from a snapshot (loro-dev/loro#1126). + +## Data written by loro.js 0.2 + +loro.js 0.2 ordered some concurrent text inserts differently from Rust (a +concurrent insert after a sibling's subtree, now `fugueSubtreeEnd`) and encoded +text cursors by UTF-16 position. Decision (2026-09-28): later versions read all +data with Rust's semantics and add no version marker. The same bytes must mean +the same thing in both runtimes, documents shared with `loro-crdt` peers had +already diverged, and a marker would need a format change that Rust does not +have. The cost falls on documents edited only with 0.2: their history can read +differently, and a replica loaded from a 0.2 snapshot keeps the 0.2 text while +one loaded from updates follows Rust. `README.md` ("Upgrading from 0.2") +describes the migration paths; `tests/legacy-data.test.ts` pins the readings +with fixtures written by `scripts/write-legacy-fixtures.mjs`. diff --git a/loro-js/AGENTS.md b/loro-js/AGENTS.md index cd5b3661c..61232601b 100644 --- a/loro-js/AGENTS.md +++ b/loro-js/AGENTS.md @@ -20,6 +20,11 @@ the same binary and JSON data as Rust (`loro-crdt`) and give the same results. `crates/loro/tests/loro_js_interop.rs` imports them. Container state encoders live in `src/runtime/document.ts` (`#containerState`) and must follow `docs/encoding-container-states.md`. +- loro.js 0.2 output read by later versions: `scripts/write-legacy-fixtures.mjs` + writes `tests/fixtures/loro-js-0.2/` with a 0.2 build; `tests/legacy-data.test.ts` + pins that loro.js and Rust read it the same way. Later versions read 0.2 data + with Rust's semantics, with no version marker (README "Upgrading from 0.2"; the + decision is in [context/loro-js-rust-differential.md](../context/loro-js-rust-differential.md)). ## Commands diff --git a/loro-js/README.md b/loro-js/README.md index 6e037ad0e..ccbb38d73 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -121,6 +121,47 @@ Treat the package as experimental when data can be produced by untrusted or olde clients. Keep a `loro-crdt` interoperability test for the formats and operations your application depends on. +## Upgrading from 0.2 + +1.0 reads every document the way Rust (`loro-crdt`) does, including documents +written by 0.2. The data has no version marker, so a document edited only with 0.2 +can read differently after the upgrade: + +- **Concurrent text inserts.** 0.2 could order an insert after a sibling's subtree + differently from Rust. 1.0 orders it like Rust, so such a history can give other + text (`bé9a` in 0.2, `béa9` in 1.0 and Rust). +- **Cursors.** 0.2 encoded the UTF-16 position of a cursor's target; Rust and 1.0 use + the Unicode position. A cursor encoded by 0.2 can resolve to another offset (on + `ab😀`, 4 in 0.2 and 2 in 1.0). Get new cursors from the upgraded document. + +Documents also edited by `loro-crdt` peers already disagreed with Rust in these cases; +1.0 makes them agree. + +How 1.0 reads data written by 0.2 (measured on the example above): + +| 0.2 data | Current text in 1.0 | History in 1.0 | +| --------------------- | ----------------------- | -------------------------------------------------------------------------- | +| Updates, JSON | Rust's reading (`béa9`) | Rust's reading | +| Full/shallow snapshot | Kept (`bé9a`) | Rust's reading: exported updates and checkouts of older versions follow it | + +**Migrate every replica the same way.** Replicas loaded from the same 0.2 snapshot +agree with each other, but a replica loaded from 0.2 updates holds other text, and +the two stay different after exchanging new edits (`béQ9aZ` and `béQa9Z`). A peer that +later joins through exported updates instead of a snapshot diverges the same way. + +Recommended paths: + +1. **Keep the content, drop the history (safest).** With 0.2, check out the latest + version and read the final state (`doc.toJSON()`, or `text.toDelta()` for rich + text). In 1.0, build one new document from it and share that document with every + replica. Do not import old updates or snapshots into it. +2. **Keep the history, accept Rust's reading.** Import the 0.2 updates (not snapshots) + on every replica, and check the content, because it can differ from what 0.2 + showed. + +A 0.2 snapshot keeps its current text, so loading it is only safe when every replica +loads that snapshot and new peers join through snapshots, never through updates. + ## Development checks From this directory: diff --git a/loro-js/scripts/write-legacy-fixtures.mjs b/loro-js/scripts/write-legacy-fixtures.mjs new file mode 100644 index 000000000..6889f676d --- /dev/null +++ b/loro-js/scripts/write-legacy-fixtures.mjs @@ -0,0 +1,65 @@ +// Writes documents with loro.js 0.2 into tests/fixtures/loro-js-0.2, so that +// tests/legacy-data.test.ts pins how later versions read data written before +// loro.js adopted Rust's text semantics ("Upgrading from 0.2" in README.md). +// +// npm install --prefix /tmp/loro-js-0.2 loro.js@0.2.1 +// node scripts/write-legacy-fixtures.mjs /tmp/loro-js-0.2/node_modules/loro.js/dist/index.js + +import { mkdirSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const legacyEntry = process.argv[2]; +if (legacyEntry === undefined) { + throw new Error("pass the path of loro.js 0.2's dist/index.js"); +} +const { LoroDoc } = await import(pathToFileURL(legacyEntry).href); + +const directory = new URL("../tests/fixtures/loro-js-0.2/", import.meta.url); +mkdirSync(directory, { recursive: true }); +const write = (name, bytes) => writeFileSync(new URL(name, directory), bytes); +const expected = {}; + +function doc(peer) { + const created = new LoroDoc(); + created.setPeerId(peer); + return created; +} + +function sync(from, to) { + to.import(from.export({ mode: "update" })); +} + +// Concurrent inserts that 0.2 ordered differently from Rust: "a" (peer 3) and +// "é" (peer 2) both follow "b", and 0.2 put "a" after "9". +{ + const [peer1, peer2, peer3] = [1, 2, 3].map(doc); + peer2.getText("t").insert(0, "9"); + peer2.commit(); + peer1.getText("t").insert(0, "b"); + peer1.commit(); + sync(peer1, peer2); + peer2.getText("t").insert(1, "é"); + peer2.commit(); + sync(peer1, peer3); + peer3.getText("t").insert(1, "a"); + peer3.commit(); + sync(peer3, peer2); + write("text-concurrent.update.blob", peer2.export({ mode: "update" })); + write("text-concurrent.snapshot.blob", peer2.export({ mode: "snapshot" })); + write("text-concurrent.json", JSON.stringify(peer2.exportJsonUpdates(), null, 2)); + expected["text-concurrent"] = peer2.getText("t").toString(); +} + +// A cursor on the end of "ab😀": 0.2 encoded its UTF-16 position. +{ + const owner = doc(1); + const text = owner.getText("c"); + text.insert(0, "ab😀"); + owner.commit(); + const cursor = text.getCursor(4, 1); + write("cursor-emoji.snapshot.blob", owner.export({ mode: "snapshot" })); + write("cursor-emoji.cursor.blob", cursor.encode()); + expected["cursor-emoji"] = owner.getCursorPos(cursor).offset; +} + +write("expected.json", `${JSON.stringify(expected, null, 2)}\n`); diff --git a/loro-js/tests/fixtures/loro-js-0.2/cursor-emoji.cursor.blob b/loro-js/tests/fixtures/loro-js-0.2/cursor-emoji.cursor.blob new file mode 100644 index 0000000000000000000000000000000000000000..584f5e8f57c9e871df7db184d03af9c823164a77 GIT binary patch literal 9 QcmZQ%WMN=TW?*6g00GYcaR2}S literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/loro-js-0.2/cursor-emoji.snapshot.blob b/loro-js/tests/fixtures/loro-js-0.2/cursor-emoji.snapshot.blob new file mode 100644 index 0000000000000000000000000000000000000000..964b55e069a71bc2a4a41660b40ca504fa9348ca GIT binary patch literal 248 zcmd1FFUn^?0xcC=_A@Xy14VrNgZvp7m>HNE1sEBjDxeHjAfJ(ejgf_siGhKMF`0pn zk%fs7$O0l(CProkcDBT%5A$a(z3n#R@k1Ry&Yh{4VPnhZ4&?i6Mg7LaZr!Nde*0UZMZj10|Px#wZVGcz;+ Q9SA0cVkaE4fT&>r0KEboD*ylh literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/loro-js-0.2/expected.json b/loro-js/tests/fixtures/loro-js-0.2/expected.json new file mode 100644 index 000000000..79106f599 --- /dev/null +++ b/loro-js/tests/fixtures/loro-js-0.2/expected.json @@ -0,0 +1,4 @@ +{ + "text-concurrent": "bé9a", + "cursor-emoji": 4 +} diff --git a/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.json b/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.json new file mode 100644 index 000000000..c51d81803 --- /dev/null +++ b/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "start_version": {}, + "peers": [ + "1", + "2", + "3" + ], + "changes": [ + { + "id": "0@0", + "timestamp": 0, + "deps": [], + "lamport": 0, + "msg": null, + "ops": [ + { + "container": "cid:root-t:Text", + "content": { + "type": "insert", + "pos": 0, + "text": "b" + }, + "counter": 0 + } + ] + }, + { + "id": "0@1", + "timestamp": 0, + "deps": [], + "lamport": 0, + "msg": null, + "ops": [ + { + "container": "cid:root-t:Text", + "content": { + "type": "insert", + "pos": 0, + "text": "9" + }, + "counter": 0 + } + ] + }, + { + "id": "1@1", + "timestamp": 0, + "deps": [ + "0@0", + "0@1" + ], + "lamport": 1, + "msg": null, + "ops": [ + { + "container": "cid:root-t:Text", + "content": { + "type": "insert", + "pos": 1, + "text": "é" + }, + "counter": 1 + } + ] + }, + { + "id": "0@2", + "timestamp": 0, + "deps": [ + "0@0" + ], + "lamport": 1, + "msg": null, + "ops": [ + { + "container": "cid:root-t:Text", + "content": { + "type": "insert", + "pos": 1, + "text": "a" + }, + "counter": 0 + } + ] + } + ] +} \ No newline at end of file diff --git a/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.snapshot.blob b/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.snapshot.blob new file mode 100644 index 0000000000000000000000000000000000000000..074087f5ee706d680895b9e7226cfe1e3b236672 GIT binary patch literal 369 zcmYjNu}T9$5PdVVduI+YN+d+IkgF9e6Dz$UO=6 zu(7rD2gF8e8{eHCI54v_Z{NI`-Scdk;lJJ8f5YC1^oOG%Tzu3zJ(xYRzJ)+!RTyD4 zd@Xqa(?FakpmKo~amqXFd6f{ylh8wVUb3iMv6Q)QFX>0_Hl(5v?P|Ge9`FU^@NS^ zbjti%0e~CldH3V&?U_YNFCWV(ezKT%##WLDQ?xXzPUk~{T11jV!ptj{AhpY8HPJBR NpOi7mp3wV~_yI);Clvqy literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.update.blob b/loro-js/tests/fixtures/loro-js-0.2/text-concurrent.update.blob new file mode 100644 index 0000000000000000000000000000000000000000..c13a0f884d79caf798865f63b4418b84a41aabd1 GIT binary patch literal 282 zcmd1FFUn^?0$<(Pbs1Q!7#JBC83h;_p)ybgE0E8~z{beJ$i%?F#8|?>$H>CO2xI{f zD-%!=6JrugD^LX(5U0NnBZEWMgW}x0w5kv7cfCxz|3@br7Kv2G!rvaKf)J8 J8^gqy2mptd3H1N~ literal 0 HcmV?d00001 diff --git a/loro-js/tests/legacy-data.test.ts b/loro-js/tests/legacy-data.test.ts new file mode 100644 index 000000000..e89f70a2b --- /dev/null +++ b/loro-js/tests/legacy-data.test.ts @@ -0,0 +1,62 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, test } from "vitest"; + +import { Cursor, LoroDoc, type JsonSchema } from "../src/index"; +import { loadRustReference } from "./support/rust-reference"; + +// Written by loro.js 0.2.1 (scripts/write-legacy-fixtures.mjs). Later versions +// read them the way Rust does; README.md "Upgrading from 0.2" documents the +// differences. +const fixtureUrl = (name: string): URL => + new URL(`./fixtures/loro-js-0.2/${name}`, import.meta.url); +const fixture = (name: string): Uint8Array => + new Uint8Array(readFileSync(fixtureUrl(name))); +const json = (name: string): unknown => + JSON.parse(readFileSync(fixtureUrl(name), "utf8")); +const legacy = json("expected.json") as Record; + +interface Runtime { + readonly LoroDoc: typeof LoroDoc; + readonly Cursor: typeof Cursor; +} + +const rust = loadRustReference(); +const runtimes: [string, Runtime][] = [["loro.js", { LoroDoc, Cursor }]]; +if (rust !== undefined) runtimes.push(["loro-crdt", rust]); + +describe.each(runtimes)("documents written by loro.js 0.2, read by %s", (_, runtime) => { + const read = (load: (doc: LoroDoc) => void): LoroDoc => { + const doc = new runtime.LoroDoc(); + load(doc); + return doc; + }; + + test("orders concurrent inserts from history like Rust", () => { + expect(legacy["text-concurrent"]).toBe("bé9a"); + const fromUpdate = read((doc) => doc.import(fixture("text-concurrent.update.blob"))); + expect(fromUpdate.getText("t").toString()).toBe("béa9"); + const fromJson = read((doc) => + doc.importJsonUpdates(json("text-concurrent.json") as JsonSchema), + ); + expect(fromJson.getText("t").toString()).toBe("béa9"); + }); + + test("keeps a snapshot's current text but reads its history like Rust", () => { + const fromSnapshot = read((doc) => + doc.import(fixture("text-concurrent.snapshot.blob")), + ); + expect(fromSnapshot.getText("t").toString()).toBe("bé9a"); + const history = read((doc) => doc.import(fromSnapshot.export({ mode: "update" }))); + expect(history.getText("t").toString()).toBe("béa9"); + }); + + test("resolves a 0.2 cursor by its Unicode position", () => { + expect(legacy["cursor-emoji"]).toBe(4); + const owner = read((doc) => doc.import(fixture("cursor-emoji.snapshot.blob"))); + const cursor = runtime.Cursor.decode(fixture("cursor-emoji.cursor.blob")); + expect(owner.getCursorPos(cursor)?.offset).toBe(2); + const fresh = owner.getText("c").getCursor(4, 1)!; + expect(owner.getCursorPos(fresh)?.offset).toBe(4); + }); +}); From ee884206104b5d8da37850738bacf0149107384c Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 09:29:45 +0800 Subject: [PATCH 05/13] test(loro.js): check out per-op versions in the differential suite Visits two versions per checkout, including versions right after a random op, so transitions between historical versions and mid-change versions are compared with Rust. README "Upgrading from 0.2" now reflects what a document loaded from a 0.2 snapshot does after its history is used. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-text-interop.md | 2 +- loro-js/README.md | 44 +++++++++---------- .../tests/support/richtext-differential.ts | 37 +++++++++++++--- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/.changeset/loro-js-text-interop.md b/.changeset/loro-js-text-interop.md index 7e9b31ab6..74e6b59af 100644 --- a/.changeset/loro-js-text-interop.md +++ b/.changeset/loro-js-text-interop.md @@ -11,4 +11,4 @@ Text now follows Rust (`loro-crdt`) everywhere, including when reading documents - Consecutive inserts in one transaction are merged into one op, and a delete that spans several ID runs writes the last run first, as Rust does. - Importing a concurrent insert next to a long run of text no longer walks the whole run. -Upgrading: a snapshot written by an earlier version keeps its current text, but its history (updates, JSON updates, checkout of older versions) is read like Rust, and a replica loaded from a snapshot and one loaded from updates can keep different text after new edits. Migrate every replica the same way; the safest path is to read the final state with the old version and build a new document from it. +Upgrading: updates and JSON updates written by an earlier version are read like Rust. A snapshot written by an earlier version keeps its current text, but its history reads like Rust, so a document loaded from it must not stay in use (exported updates, replicas loaded from updates, and checkouts can show other content). Migrate every replica the same way: read the final state (with the old version, or right after importing an old snapshot) and build a new document from it, or import the old updates everywhere and accept Rust's reading. See "Upgrading from 0.2" in the README. diff --git a/loro-js/README.md b/loro-js/README.md index ccbb38d73..b966de1f1 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -139,28 +139,28 @@ Documents also edited by `loro-crdt` peers already disagreed with Rust in these How 1.0 reads data written by 0.2 (measured on the example above): -| 0.2 data | Current text in 1.0 | History in 1.0 | -| --------------------- | ----------------------- | -------------------------------------------------------------------------- | -| Updates, JSON | Rust's reading (`béa9`) | Rust's reading | -| Full/shallow snapshot | Kept (`bé9a`) | Rust's reading: exported updates and checkouts of older versions follow it | - -**Migrate every replica the same way.** Replicas loaded from the same 0.2 snapshot -agree with each other, but a replica loaded from 0.2 updates holds other text, and -the two stay different after exchanging new edits (`béQ9aZ` and `béQa9Z`). A peer that -later joins through exported updates instead of a snapshot diverges the same way. - -Recommended paths: - -1. **Keep the content, drop the history (safest).** With 0.2, check out the latest - version and read the final state (`doc.toJSON()`, or `text.toDelta()` for rich - text). In 1.0, build one new document from it and share that document with every - replica. Do not import old updates or snapshots into it. -2. **Keep the history, accept Rust's reading.** Import the 0.2 updates (not snapshots) - on every replica, and check the content, because it can differ from what 0.2 - showed. - -A 0.2 snapshot keeps its current text, so loading it is only safe when every replica -loads that snapshot and new peers join through snapshots, never through updates. +| 0.2 data | 1.0 reads | +| --------------------- | ----------------------------------------------------------------------- | +| Updates, JSON updates | Rust's reading: `béa9` | +| Full/shallow snapshot | The 0.2 text (`bé9a`) as current state, but the history reads like Rust | + +A document loaded from a 0.2 snapshot therefore holds a current state that its own +history disagrees with. Its current state is safe to read right after the import, +but do not keep using the document: exporting it as updates gives `béa9`, a +replica loaded from updates holds other text and stays different after new edits +(`béQ9aZ` and `béQa9Z`), and checking out older versions and back can change the +content (Rust can then even show text that matches neither reading). + +**Migrate every replica the same way.** Recommended paths: + +1. **Keep the content, drop the history (safest).** Read the final state, either + with 0.2 after checking out the latest version, or with 1.0 right after importing + a 0.2 snapshot: `doc.toJSON()`, or `text.toDelta()` for rich text. Build one new + document from it in 1.0 and share that document with every replica. Do not + import old updates or snapshots into it. +2. **Keep the history, accept Rust's reading.** Import the 0.2 updates (not + snapshots) on every replica, and check the content, because it can differ from + what 0.2 showed. ## Development checks diff --git a/loro-js/tests/support/richtext-differential.ts b/loro-js/tests/support/richtext-differential.ts index 47d0a75dd..c719b8380 100644 --- a/loro-js/tests/support/richtext-differential.ts +++ b/loro-js/tests/support/richtext-differential.ts @@ -397,12 +397,19 @@ export class RichtextScenario { return; case "checkout": { const peer = this.#peer(action.peer); - if (peer.versions.length === 0) return; this.#commit(peer); - const version = peer.versions[action.version % peer.versions.length]!; - this.#both(peer, `checkout(${JSON.stringify(version)})`, (r) => - r.doc.checkout(version), - ); + // Two versions in a row, so transitions also run between historical + // versions, not only from and back to the latest one. + for (const roll of [ + action.version, + Math.imul(action.version, 0x9e37_79b1) >>> 0, + ]) { + const version = this.#versionToVisit(peer, roll); + if (version === undefined) continue; + this.#both(peer, `checkout(${JSON.stringify(version)})`, (r) => + r.doc.checkout(version), + ); + } this.#both(peer, "checkoutToLatest", (r) => r.doc.checkoutToLatest()); return; } @@ -461,6 +468,26 @@ export class RichtextScenario { } } + /** + * A recorded commit version, or for odd rolls the version right after a + * random op in the history. The latter can split a change or an op, such as + * a mark whose end anchor is not included yet. + */ + #versionToVisit(peer: Peer, roll: number): Frontiers | undefined { + if (roll % 2 === 1) { + const counters = [...peer.rust.doc.oplogVersion().toJSON()].filter( + ([, end]) => end > 0, + ); + if (counters.length > 0) { + const [id, end] = counters[(roll >>> 1) % counters.length]!; + return [{ peer: id, counter: (roll >>> 5) % end }]; + } + } + return peer.versions.length === 0 + ? undefined + : peer.versions[roll % peer.versions.length]; + } + #text(replica: Replica): Text { return replica.doc.getText("t"); } From ccd06e02d8d0f70319be29c943f26d82889b2865 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 09:39:23 +0800 Subject: [PATCH 06/13] docs(loro.js): describe per-op checkouts in the differential suite Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-rust-differential.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/context/loro-js-rust-differential.md b/context/loro-js-rust-differential.md index c72addc6f..31372048a 100644 --- a/context/loro-js-rust-differential.md +++ b/context/loro-js-rust-differential.md @@ -33,7 +33,10 @@ per runtime with the same peer ID. Actions: - `sync` by binary or JSON updates, within a runtime or crossed (Rust imports the loro.js peer's updates and loro.js imports Rust's), so each side reads the other's positions; -- `checkout` to a recorded version and back; +- `checkout` to two versions in a row, then back to the latest: each is a + recorded commit version or the version right after a random op, which can + split a change or an op (a mark with only its start anchor, for example), so + transitions also run between historical and mid-change versions; - full and shallow snapshot export from either runtime, imported into both; - cursor creation, then cursor resolution after every later step; - `revertTo` and undo/redo. From b1e4bade6ee0144111b8654242603d54d0382a96 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 09:45:14 +0800 Subject: [PATCH 07/13] docs(loro.js): record Fugue subtree-end measurements Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 4c1ec6283..b2a78a3bb 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -111,7 +111,12 @@ JavaScript constant factor. Inside a physical ID run every element's origin-left is its predecessor, so the walk jumps between run starts (`SequenceIndex.nextPhysicalIdRunStart`, skipping single-run treap subtrees): O((runs in the subtree + 1) log n), and - O(log n) after a long typed run (`text-concurrent-insert-after-long-run`). + O(log n) after a long typed run. With a warm origin index, importing one + concurrent character after a 512k-character run takes 0.2–0.3 ms (0.2–0.45 ms + on `main`, which stopped at the origin-right bound and could misorder; 8.9–14.8 + ms for the per-scalar walk), and `text-concurrent-insert-after-long-run` stays + at 0.15–0.24 ms from 64k to 512k characters (2.4 → 12.8 ms per-scalar; + September 28, Node 22, 1-minute load 5–25, interleaved). - An imported Text delete is resolved by its position in the op's causal view, like Rust's tracker (`LoroText._deleteTargets`): O(log n + runs) through `visibleIdRuns` or the cached causal view. The recorded `start_id` is only a From c14f3164ef3b979bfef2143ea7c8eb3a5e40c979 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 17:38:45 +0800 Subject: [PATCH 08/13] docs(loro.js): record Fugue measurements after adopting #1139 Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 803f04d04..af0588eb1 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -119,8 +119,11 @@ JavaScript constant factor. search takes 7.3 ms versus 11.1 ms, but on a deep chain of explicit links (two positions typed alternately, 64k elements) 60.6 ms versus 15.5 ms, because each of its O(log n) probes walks the chain. Realistic traces favor - the binary search. `text-concurrent-insert-after-long-run` covers the long - typed run. + the binary search. With a warm origin index, importing one concurrent + character after a 512k-character typed run takes 0.28–0.31 ms (0.29–0.30 ms + on `main`, which misorders other cases; a per-scalar walk took 9.7–10.7 ms), + and `text-concurrent-insert-after-long-run` stays at 0.18–0.30 ms from 64k to + 512k characters (September 28, Node 22, interleaved, 1-minute load 7–17). - An imported Text delete is resolved by its position in the op's causal view, like Rust's tracker (`LoroText._deleteTargets`): O(log n + runs) through `visibleIdRuns` or the cached causal view. The recorded `start_id` is only a From 4140169f068ddbd66bf3f3384fc3d053338a1dce Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 15:07:35 +0800 Subject: [PATCH 09/13] docs(loro.js): describe 0.2 snapshots after checkout, fork, and shallow export With #1126 on main, a checkout keeps a 0.2 snapshot's state while a fork or shallow export replays its history (Rust's reading); the test pins the fork difference from Rust. Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/README.md | 6 ++- loro-js/tests/legacy-data.test.ts | 79 +++++++++++++++++-------------- 2 files changed, 48 insertions(+), 37 deletions(-) diff --git a/loro-js/README.md b/loro-js/README.md index b966de1f1..fc6ae0665 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -148,8 +148,10 @@ A document loaded from a 0.2 snapshot therefore holds a current state that its o history disagrees with. Its current state is safe to read right after the import, but do not keep using the document: exporting it as updates gives `béa9`, a replica loaded from updates holds other text and stays different after new edits -(`béQ9aZ` and `béQa9Z`), and checking out older versions and back can change the -content (Rust can then even show text that matches neither reading). +(`béQ9aZ` and `béQa9Z`), older versions it checks out can be approximate, and a +fork or a shallow-snapshot export can rebuild it from its history, which switches +it to Rust's reading (Rust can even show text that matches neither reading after a +checkout there and back). **Migrate every replica the same way.** Recommended paths: diff --git a/loro-js/tests/legacy-data.test.ts b/loro-js/tests/legacy-data.test.ts index e89f70a2b..9f17b0e01 100644 --- a/loro-js/tests/legacy-data.test.ts +++ b/loro-js/tests/legacy-data.test.ts @@ -25,38 +25,47 @@ const rust = loadRustReference(); const runtimes: [string, Runtime][] = [["loro.js", { LoroDoc, Cursor }]]; if (rust !== undefined) runtimes.push(["loro-crdt", rust]); -describe.each(runtimes)("documents written by loro.js 0.2, read by %s", (_, runtime) => { - const read = (load: (doc: LoroDoc) => void): LoroDoc => { - const doc = new runtime.LoroDoc(); - load(doc); - return doc; - }; - - test("orders concurrent inserts from history like Rust", () => { - expect(legacy["text-concurrent"]).toBe("bé9a"); - const fromUpdate = read((doc) => doc.import(fixture("text-concurrent.update.blob"))); - expect(fromUpdate.getText("t").toString()).toBe("béa9"); - const fromJson = read((doc) => - doc.importJsonUpdates(json("text-concurrent.json") as JsonSchema), - ); - expect(fromJson.getText("t").toString()).toBe("béa9"); - }); - - test("keeps a snapshot's current text but reads its history like Rust", () => { - const fromSnapshot = read((doc) => - doc.import(fixture("text-concurrent.snapshot.blob")), - ); - expect(fromSnapshot.getText("t").toString()).toBe("bé9a"); - const history = read((doc) => doc.import(fromSnapshot.export({ mode: "update" }))); - expect(history.getText("t").toString()).toBe("béa9"); - }); - - test("resolves a 0.2 cursor by its Unicode position", () => { - expect(legacy["cursor-emoji"]).toBe(4); - const owner = read((doc) => doc.import(fixture("cursor-emoji.snapshot.blob"))); - const cursor = runtime.Cursor.decode(fixture("cursor-emoji.cursor.blob")); - expect(owner.getCursorPos(cursor)?.offset).toBe(2); - const fresh = owner.getText("c").getCursor(4, 1)!; - expect(owner.getCursorPos(fresh)?.offset).toBe(4); - }); -}); +describe.each(runtimes)( + "documents written by loro.js 0.2, read by %s", + (name, runtime) => { + const read = (load: (doc: LoroDoc) => void): LoroDoc => { + const doc = new runtime.LoroDoc(); + load(doc); + return doc; + }; + + test("orders concurrent inserts from history like Rust", () => { + expect(legacy["text-concurrent"]).toBe("bé9a"); + const fromUpdate = read((doc) => + doc.import(fixture("text-concurrent.update.blob")), + ); + expect(fromUpdate.getText("t").toString()).toBe("béa9"); + const fromJson = read((doc) => + doc.importJsonUpdates(json("text-concurrent.json") as JsonSchema), + ); + expect(fromJson.getText("t").toString()).toBe("béa9"); + }); + + test("keeps a snapshot's current text but reads its history like Rust", () => { + const fromSnapshot = read((doc) => + doc.import(fixture("text-concurrent.snapshot.blob")), + ); + expect(fromSnapshot.getText("t").toString()).toBe("bé9a"); + const history = read((doc) => doc.import(fromSnapshot.export({ mode: "update" }))); + expect(history.getText("t").toString()).toBe("béa9"); + // State and history disagree here, so the runtimes' forks differ: loro.js + // replays the history, Rust copies the state. + const fork = fromSnapshot.fork(); + expect(fork.getText("t").toString()).toBe(name === "loro.js" ? "béa9" : "bé9a"); + }); + + test("resolves a 0.2 cursor by its Unicode position", () => { + expect(legacy["cursor-emoji"]).toBe(4); + const owner = read((doc) => doc.import(fixture("cursor-emoji.snapshot.blob"))); + const cursor = runtime.Cursor.decode(fixture("cursor-emoji.cursor.blob")); + expect(owner.getCursorPos(cursor)?.offset).toBe(2); + const fresh = owner.getText("c").getCursor(4, 1)!; + expect(owner.getCursorPos(fresh)?.offset).toBe(4); + }); + }, +); From c95042a263304fd14b85d316665c2a8a9451b228 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 15:20:41 +0800 Subject: [PATCH 10/13] perf(loro.js): check a transition's sequence elements once per container #canTransitionRecords walked the whole sequence once per insert op, so checking out across scattered edits was O(ops * elements): 668 ms at 8k characters with 2k edits on main, 4 ms now. Adds text-scattered-edits-checkout to bench:complexity. Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/benchmarks/complexity.mjs | 21 ++++++++++++ loro-js/src/runtime/document.ts | 46 +++++++++++++++------------ loro-js/src/runtime/sequence-index.ts | 2 +- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/loro-js/benchmarks/complexity.mjs b/loro-js/benchmarks/complexity.mjs index 489164db0..a62447c66 100644 --- a/loro-js/benchmarks/complexity.mjs +++ b/loro-js/benchmarks/complexity.mjs @@ -527,6 +527,27 @@ for (const size of sizes) { return concurrentTarget.getText("text").length; }); + // Checking out across size / 4 scattered inserts: the transition checks each + // op's elements, once per container rather than one tree walk per op. + const scatteredDoc = new LoroDoc(); + scatteredDoc.setPeerId(4); + const scatteredText = scatteredDoc.getText("text"); + scatteredText.insert(0, "x".repeat(size)); + scatteredDoc.commit(); + const scatteredBase = scatteredDoc.frontiers(); + let scatteredSeed = 1; + for (let index = 0; index < size >>> 2; index += 1) { + scatteredSeed = (scatteredSeed * 1_103_515_245 + 12_345) >>> 0; + scatteredText.insert(scatteredSeed % scatteredText.length, "y"); + if (index % 16 === 0) scatteredDoc.commit(); + } + scatteredDoc.commit(); + measure("text-scattered-edits-checkout", size, () => { + scatteredDoc.checkout(scatteredBase); + scatteredDoc.checkoutToLatest(); + return scatteredText.length; + }); + const checkoutTextDoc = new LoroDoc(); checkoutTextDoc.setPeerId(3); checkoutTextDoc.getText("text").insert(0, "x".repeat(size)); diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 8b4f093fb..247d4dd90 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -4095,6 +4095,19 @@ export class LoroDoc = Record } >(); + // The inserted elements each sequence must still hold, checked once per + // container below: one tree walk per operation made a checkout + // O(operations * elements). + const requiredRuns = new Map(); + const requireRun = ( + container: LoroList | LoroText, + start: CodecId, + length: number, + ): void => { + const runs = requiredRuns.get(container); + if (runs === undefined) requiredRuns.set(container, [{ start, length }]); + else runs.push({ start, length }); + }; for (const { change } of records) { const causalVersion = this.#causalVersionAt(change.dependencies); for (const operation of change.operations) { @@ -4177,31 +4190,21 @@ export class LoroDoc = Record = Record( } let count = countPhysicalIdsInRanges(node.left, targets); for (let offset = 0; offset < nodeLength(node); offset += 1) { - const id = nodeElement(node, offset).id; + const id = nodeId(node, offset); if (counterRangeRelation(targets, id.peer, id.counter, id.counter + 1) === 1) { count += 1; } From 981c12e666cfd0196ca52d357ce6a928195efa37 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 15:31:12 +0800 Subject: [PATCH 11/13] docs(loro.js): record the transition check measurements Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-performance.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 17a1ffcf1..38450b831 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -253,6 +253,14 @@ JavaScript constant factor. that throws restores its previous version and state (`#transitionTo`, which also prepares inside its `try`), and `diff` restores the current state with a full rebuild when it or its move back throws. +- Before a transition, `#canTransitionRecords` checks that each sequence still + holds the elements that the crossed insert operations name. It collects the + runs per container and calls `containsIdRuns` once per container, reading IDs + without building element views: O(elements + runs log runs). One call per + operation made a checkout O(operations × elements): across 2k scattered + inserts in an 8k text it took 698 ms on `main` and takes 4.2 ms now + (`text-scattered-edits-checkout`, 13 → 698 ms from 1k to 8k before, 1.0 → + 4.2 ms now; September 29, Node 22, 1-minute load 6–9). - First checkout after importing a 262,144-operation single-peer Text snapshot takes about 57 ms (medians of 5 alternating runs on a loaded Apple M5 Pro), versus about 148 ms for the earlier whole-document replay; 65,536 operations From eabcddfd87c4c712a532e70512ce2b221aa78d9b Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 17:33:51 +0800 Subject: [PATCH 12/13] docs(loro.js): correct the 0.2 cursor and migration notes Cursors move for side 1 and end-of-text, not astral text; reading the final state with 1.0 needs the snapshot to be the final version; a shallow export switches a 0.2-snapshot document to Rust's reading. Co-Authored-By: Claude Opus 5.5 --- .changeset/loro-js-text-interop.md | 4 ++-- context/loro-js-rust-differential.md | 4 ++-- loro-js/README.md | 28 ++++++++++++++--------- loro-js/scripts/write-legacy-fixtures.mjs | 2 +- loro-js/tests/legacy-data.test.ts | 2 +- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.changeset/loro-js-text-interop.md b/.changeset/loro-js-text-interop.md index 0f17142ab..521f1c6c7 100644 --- a/.changeset/loro-js-text-interop.md +++ b/.changeset/loro-js-text-interop.md @@ -5,9 +5,9 @@ Text now follows Rust (`loro-crdt`) everywhere, including when reading documents written by earlier loro.js versions. This breaks documents edited only with loro.js that have concurrent text edits or stored cursors: read "Upgrading from 0.2" in the README before upgrading. - **Breaking:** concurrent text inserts are ordered like Rust (the fix is shared with loro-dev/loro#1139). Earlier versions could place an insert after a sibling's subtree behind later, unrelated concurrent text, so a history with such inserts can now read differently (for example `béa9` instead of `bé9a`). Earlier loro.js replicas that merged such edits in different orders, and documents shared with `loro-crdt` peers, had already diverged; they now agree with Rust. -- **Breaking:** text cursors follow Rust. An encoded cursor stores the Unicode position of its target, not the UTF-16 position, so a cursor encoded by an earlier version can resolve to another offset. A cursor at the end or in empty text has no target, `getCursorPos` returns the target's own offset for every side, and a deleted target reports the length of the text before it with side `-1`. Get new cursors after upgrading. +- **Breaking:** text cursors follow Rust: a cursor reports its target's own offset, so a cursor encoded by an earlier version with side 1, or at the end of the text, resolves one character earlier (the end cursor of `abc` at 2 instead of 3). Other cursors resolve as before. The encoded origin is now the Unicode position, as in Rust. A cursor at the end or in empty text has no target, `getCursorPos` returns the target's own offset for every side, and a deleted target reports the length of the text before it with side `-1`. Get new cursors after upgrading. - An imported delete is applied by its position, as Rust does, instead of by its recorded `start_id` (Rust's WASM build up to 1.16.3 can record a wrong one around astral characters, loro-dev/loro#1149). - `checkout`, `diff`, and `revertTo` no longer restore a character twice when two concurrent deletes removed it. - Consecutive inserts in one transaction are merged into one op, and a delete that spans several ID runs writes the last run first, as Rust does. -Upgrading: updates and JSON updates written by an earlier version are read like Rust. A snapshot written by an earlier version keeps its current text, but its history reads like Rust, so a document loaded from it must not stay in use (exported updates, replicas loaded from updates, and checkouts can show other content). Migrate every replica the same way: read the final state (with the old version, or right after importing an old snapshot) and build a new document from it, or import the old updates everywhere and accept Rust's reading. See "Upgrading from 0.2" in the README. +Upgrading: updates and JSON updates written by an earlier version are read like Rust. A snapshot written by an earlier version keeps its current text, but its history reads like Rust, so a document loaded from it must not stay in use (exported updates, replicas loaded from updates, and checkouts can show other content). Exporting a shallow snapshot from such a document switches the document itself to Rust's reading. Migrate every replica the same way: read the final state with the old version and build a new document from it (reading it with 1.0 right after importing an old snapshot only works when that snapshot is the final version, with no updates stored after it), or import the old updates everywhere and accept Rust's reading. See "Upgrading from 0.2" in the README. diff --git a/context/loro-js-rust-differential.md b/context/loro-js-rust-differential.md index 1f954c5ba..c417d8c68 100644 --- a/context/loro-js-rust-differential.md +++ b/context/loro-js-rust-differential.md @@ -86,8 +86,8 @@ can check out a document imported from a snapshot (loro-dev/loro#1126). ## Data written by loro.js 0.2 loro.js 0.2 ordered some concurrent text inserts differently from Rust (a -concurrent insert after a sibling's subtree, fixed by loro-dev/loro#1139) and encoded -text cursors by UTF-16 position. Decision (2026-09-28): later versions read all +concurrent insert after a sibling's subtree, fixed by loro-dev/loro#1139) and resolved +side-1 and end-of-text cursors one character after their target. Decision (2026-09-28): later versions read all data with Rust's semantics and add no version marker. The same bytes must mean the same thing in both runtimes, documents shared with `loro-crdt` peers had already diverged, and a marker would need a format change that Rust does not diff --git a/loro-js/README.md b/loro-js/README.md index fc6ae0665..65b80487d 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -130,9 +130,11 @@ can read differently after the upgrade: - **Concurrent text inserts.** 0.2 could order an insert after a sibling's subtree differently from Rust. 1.0 orders it like Rust, so such a history can give other text (`bé9a` in 0.2, `béa9` in 1.0 and Rust). -- **Cursors.** 0.2 encoded the UTF-16 position of a cursor's target; Rust and 1.0 use - the Unicode position. A cursor encoded by 0.2 can resolve to another offset (on - `ab😀`, 4 in 0.2 and 2 in 1.0). Get new cursors from the upgraded document. +- **Cursors.** 1.0 resolves cursors like Rust: a cursor reports its target's own + offset. A cursor encoded by 0.2 with side 1, or at the end of the text, therefore + resolves one character earlier, whatever the characters are (on `abc`, the end + cursor is at 3 in 0.2 and 2 in 1.0; a side-1 cursor on `b` at 2 and 1). Other + cursors resolve as before. Get new cursors from the upgraded document. Documents also edited by `loro-crdt` peers already disagreed with Rust in these cases; 1.0 makes them agree. @@ -149,17 +151,21 @@ history disagrees with. Its current state is safe to read right after the import but do not keep using the document: exporting it as updates gives `béa9`, a replica loaded from updates holds other text and stays different after new edits (`béQ9aZ` and `béQa9Z`), older versions it checks out can be approximate, and a -fork or a shallow-snapshot export can rebuild it from its history, which switches -it to Rust's reading (Rust can even show text that matches neither reading after a -checkout there and back). +fork can rebuild it from its history, which switches it to Rust's reading. +Exporting a shallow snapshot from it (`export({ mode: "shallow-snapshot" })`) does +the same to the document itself: its current state becomes Rust's reading (`bé9a` +turns into `béa9`). Rust can even show text that matches neither reading after a +checkout there and back. **Migrate every replica the same way.** Recommended paths: -1. **Keep the content, drop the history (safest).** Read the final state, either - with 0.2 after checking out the latest version, or with 1.0 right after importing - a 0.2 snapshot: `doc.toJSON()`, or `text.toDelta()` for rich text. Build one new - document from it in 1.0 and share that document with every replica. Do not - import old updates or snapshots into it. +1. **Keep the content, drop the history (safest).** Read the final state with 0.2, + after checking out the latest version: `doc.toJSON()`, or `text.toDelta()` for + rich text. Build one new document from it in 1.0 and share that document with + every replica. Do not import old updates or snapshots into it. Reading the state + with 1.0 right after importing a 0.2 snapshot gives the same result only when + that snapshot is the final version: if updates were stored after it, 1.0 (like + Rust) reads those updates the Rust way. 2. **Keep the history, accept Rust's reading.** Import the 0.2 updates (not snapshots) on every replica, and check the content, because it can differ from what 0.2 showed. diff --git a/loro-js/scripts/write-legacy-fixtures.mjs b/loro-js/scripts/write-legacy-fixtures.mjs index 6889f676d..1b807cdde 100644 --- a/loro-js/scripts/write-legacy-fixtures.mjs +++ b/loro-js/scripts/write-legacy-fixtures.mjs @@ -50,7 +50,7 @@ function sync(from, to) { expected["text-concurrent"] = peer2.getText("t").toString(); } -// A cursor on the end of "ab😀": 0.2 encoded its UTF-16 position. +// The end cursor of "ab😀": 0.2 resolved it after its target, Rust and 1.0 at it. { const owner = doc(1); const text = owner.getText("c"); diff --git a/loro-js/tests/legacy-data.test.ts b/loro-js/tests/legacy-data.test.ts index 9f17b0e01..dccc17c97 100644 --- a/loro-js/tests/legacy-data.test.ts +++ b/loro-js/tests/legacy-data.test.ts @@ -59,7 +59,7 @@ describe.each(runtimes)( expect(fork.getText("t").toString()).toBe(name === "loro.js" ? "béa9" : "bé9a"); }); - test("resolves a 0.2 cursor by its Unicode position", () => { + test("resolves a 0.2 end cursor one character earlier, like Rust", () => { expect(legacy["cursor-emoji"]).toBe(4); const owner = read((doc) => doc.import(fixture("cursor-emoji.snapshot.blob"))); const cursor = runtime.Cursor.decode(fixture("cursor-emoji.cursor.blob")); From 84431ba4a6855ea5baaa691cb4f830541665ec95 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 17:58:04 +0800 Subject: [PATCH 13/13] test(loro.js): make the differential checkout's second version independent The second checkout version came from `imul(roll, 0x9e3779b1)`, whose low bits repeat the roll's (the multiplier is 1 modulo 16), so it usually picked the first version again. It now uses the product's high bits. Co-Authored-By: Claude Opus 5.5 --- loro-js/tests/support/richtext-differential.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/loro-js/tests/support/richtext-differential.ts b/loro-js/tests/support/richtext-differential.ts index c719b8380..f373f7861 100644 --- a/loro-js/tests/support/richtext-differential.ts +++ b/loro-js/tests/support/richtext-differential.ts @@ -400,10 +400,7 @@ export class RichtextScenario { this.#commit(peer); // Two versions in a row, so transitions also run between historical // versions, not only from and back to the latest one. - for (const roll of [ - action.version, - Math.imul(action.version, 0x9e37_79b1) >>> 0, - ]) { + for (const roll of [action.version, secondRoll(action.version)]) { const version = this.#versionToVisit(peer, roll); if (version === undefined) continue; this.#both(peer, `checkout(${JSON.stringify(version)})`, (r) => @@ -786,6 +783,14 @@ export class RichtextScenario { * ops of merged changes depending on internal encoding state, so this form * ignores how atoms are grouped into ops. */ +/** + * A second roll derived from `roll`. The product's high bits, because the + * multiplier is 1 modulo 16, so its low bits repeat `roll`'s. + */ +function secondRoll(roll: number): number { + return Math.imul(roll, 0x9e37_79b1) >>> 16; +} + function atomicOps(doc: Doc): string[] { const json = doc.exportJsonUpdates(); const peers = json.peers;