From c4fd87f4a96d0282938f255e290ea7807c2c8f6e Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:19:37 +0800 Subject: [PATCH 1/7] fix(loro.js): root shallow snapshots at the latest critical version (#1144) loro.js rooted a shallow snapshot at the requested version, or at the meet of its heads. A branch merged after that version but forked below it is concurrent with it, so a retained op straddled the root. Checking out retained versions and returning could then reorder concurrent Text/List elements, and the root differed from a Rust export. Port Rust's latest_single_head_critical_version (dag.rs) and use it with the latest frontiers, as calc_shallow_doc_start does. The StyleStart nudge and the clamp to an existing shallow root are unchanged. Co-authored-by: Claude Opus 5.5 (1M context) --- .changeset/loro-js-shallow-critical-root.md | 5 + loro-js/src/runtime/document.ts | 145 +++++++++++++-- loro-js/tests/shallow-root-critical.test.ts | 186 ++++++++++++++++++++ 3 files changed, 324 insertions(+), 12 deletions(-) create mode 100644 .changeset/loro-js-shallow-critical-root.md create mode 100644 loro-js/tests/shallow-root-critical.test.ts diff --git a/.changeset/loro-js-shallow-critical-root.md b/.changeset/loro-js-shallow-critical-root.md new file mode 100644 index 000000000..7a7fe3aa8 --- /dev/null +++ b/.changeset/loro-js-shallow-critical-root.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +`loro.js` shallow snapshots now pick the same root as `loro-crdt`: the latest single-head critical version of the requested frontiers and the latest version (loro-dev/loro#1095). Before, `loro.js` used the requested version (or the meet of its heads) even when a retained op was concurrent with it. Checking out retained versions of such a snapshot and returning to the latest version could reorder concurrent Text/List elements, and the snapshot disagreed with a Rust export of the same document. diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 81937856f..ecb5bba98 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -7713,18 +7713,15 @@ export class LoroDoc = Record 1) { - const versions = start.map((frontier) => this.#causalVersionAt([frontier])); - const peers = new Set(versions.flatMap((version) => [...version.keys()])); - const common = new VersionVector(); - for (const peer of peers) { - const counter = Math.min(...versions.map((version) => version.get(peer) ?? 0)); - if (counter > 0) common.set(peer, counter); - } - const commonFrontiers = this.#frontiersForVersion(common); - start = commonFrontiers.length === 1 ? commonFrontiers : []; - } + // Every retained op must be causally before or after the root, so a + // branch merged after `requested` that forked below it moves the root + // down (loro-dev/loro#1095). + let start = + parsed.length === 0 + ? [] + : this.#latestSingleHeadCriticalVersion(parsed, [ + ...this.#historyFrontiers.values(), + ]); if (start.length === 1) { const operation = this.#operationAt(start[0]!); @@ -7743,6 +7740,130 @@ export class LoroDoc = Record { + const record = this.#recordContaining(id); + if (record === undefined) return undefined; + const change = record.change; + return { + peer: change.id.peer, + counter: change.id.counter, + lamport: change.lamport, + len: id.counter - change.id.counter + 1, + deps: change.dependencies, + }; + }; + const lastCounter = (span: Span): number => span.counter + span.len - 1; + const lastLamport = (span: Span): number => span.lamport + span.len - 1; + const contains = (span: Span, peer: bigint, counter: number): boolean => + span.peer === peer && span.counter <= counter && counter <= lastCounter(span); + // Max-heap order: last lamport, then peer; a shorter span with the same + // last op is greater. + const greater = (a: Span, b: Span): boolean => + lastLamport(a) !== lastLamport(b) + ? lastLamport(a) > lastLamport(b) + : a.peer !== b.peer + ? a.peer > b.peer + : a.len < b.len; + const heap: Span[] = []; + const push = (span: Span): void => { + heap.push(span); + for (let index = heap.length - 1; index > 0; ) { + const parent = (index - 1) >> 1; + if (!greater(heap[index]!, heap[parent]!)) break; + [heap[index], heap[parent]] = [heap[parent]!, heap[index]!]; + index = parent; + } + }; + const pop = (): Span => { + const top = heap[0]!; + const last = heap.pop()!; + if (heap.length > 0) { + heap[0] = last; + for (let index = 0; ; ) { + const leftChild = index * 2 + 1; + const rightChild = leftChild + 1; + let largest = index; + if (leftChild < heap.length && greater(heap[leftChild]!, heap[largest]!)) { + largest = leftChild; + } + if (rightChild < heap.length && greater(heap[rightChild]!, heap[largest]!)) { + largest = rightChild; + } + if (largest === index) break; + [heap[index], heap[largest]] = [heap[largest]!, heap[index]!]; + index = largest; + } + } + return top; + }; + for (const id of [...left, ...right]) { + const span = spanOf(id); + if (span === undefined) return []; + push(span); + } + while (heap.length > 0) { + const node = pop(); + while ( + heap.length > 0 && + heap[0]!.peer === node.peer && + lastCounter(heap[0]!) === lastCounter(node) + ) { + pop(); + } + if (heap.length === 0) return [{ peer: node.peer, counter: lastCounter(node) }]; + const other = heap[0]!; + if (contains(node, other.peer, lastCounter(other))) { + node.len = lastCounter(other) - node.counter + 1; + push(node); + continue; + } + if (node.len > 1) { + node.len = + lastLamport(other) >= node.lamport + ? Math.min(lastLamport(other) - node.lamport + 1, node.len - 1) + : 1; + push(node); + continue; + } + const deps: Span[] = []; + for (const dependency of node.deps) { + const span = spanOf(dependency); + if (span === undefined) return []; + deps.push(span); + } + if (node.counter > 0) { + const previous = spanOf({ peer: node.peer, counter: node.counter - 1 }); + if ( + previous !== undefined && + !deps.some((dep) => contains(dep, previous.peer, lastCounter(previous))) + ) { + deps.push(previous); + } + } + if (deps.length === 0) return []; + for (const dep of deps) push(dep); + } + return []; + } + #causalVersionForKnownFrontiers(frontiers: readonly CodecId[]): VersionVector { const version = new VersionVector(); for (const [peer, counter] of this.#causalVersionAt(frontiers)) { diff --git a/loro-js/tests/shallow-root-critical.test.ts b/loro-js/tests/shallow-root-critical.test.ts new file mode 100644 index 000000000..3c17fc04c --- /dev/null +++ b/loro-js/tests/shallow-root-critical.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc } from "../src/index"; +import type { Frontiers } from "../src/index"; + +// Ports of crates/loro/tests/shallow_root_critical.rs. A shallow snapshot keeps +// every op from its root to the latest version, so the root must be a critical +// version: every retained op is causally after it, never concurrent with it +// (loro-dev/loro#1095). The roots below are the ones loro-crdt picks. + +function shallowOf(doc: LoroDoc, frontiers: Frontiers): LoroDoc { + const shallow = new LoroDoc(); + shallow.import(doc.export({ mode: "shallow-snapshot", frontiers })); + return shallow; +} + +/** Every op of `doc` is in the root's causal past or has the root in its own. */ +function expectCritical(doc: LoroDoc, root: Frontiers): void { + if (root.length === 0) return; + const rootVersion = doc.frontiersToVV(root); + for (const [peer, end] of doc.oplogVersion().toJSON()) { + for (let counter = rootVersion.get(peer) ?? 0; counter < end; counter += 1) { + const opVersion = doc.frontiersToVV([{ peer, counter }]); + const order = rootVersion.compare(opVersion); + if (order !== -1 && order !== 0) { + throw new Error(`${counter}@${peer} is concurrent with the root`); + } + } + } +} + +describe("shallow snapshot root", () => { + test("keeps the full history for independent heads", () => { + for (let peers = 2; peers <= 4; peers += 1) { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("t").insert(0, "a"); + doc.commit(); + for (let peer = 2; peer <= peers; peer += 1) { + const other = new LoroDoc(); + other.setPeerId(peer); + other.getText("t").insert(0, "b"); + other.commit(); + doc.import(other.export({ mode: "update" })); + } + const shallow = shallowOf(doc, doc.oplogFrontiers()); + expect(shallow.shallowSinceFrontiers()).toEqual([]); + expect(shallow.toJSON()).toEqual(doc.toJSON()); + } + }); + + test("still trims to a root shared by every head", () => { + const base = new LoroDoc(); + base.setPeerId(100); + base.getText("t").insert(0, "root"); + base.commit(); + const snapshot = base.export({ mode: "snapshot" }); + for (let peer = 1; peer <= 5; peer += 1) { + const fork = new LoroDoc(); + fork.setPeerId(peer); + fork.import(snapshot); + fork.getText("t").insert(0, "x"); + fork.commit(); + base.import(fork.export({ mode: "update" })); + } + const shallow = shallowOf(base, base.oplogFrontiers()); + expect(shallow.shallowSinceFrontiers()).toEqual([{ peer: "100", counter: 3 }]); + expect(shallow.toJSON()).toEqual(base.toJSON()); + }); + + test("moves below a past version when a later branch forked below it", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("t").insert(0, "0"); + doc.commit(); + const atA0 = doc.export({ mode: "snapshot" }); + doc.getText("t").insert(0, "1"); + doc.commit(); + doc.getText("t").insert(0, "2"); + doc.commit(); + const target = doc.oplogFrontiers(); + const fork = new LoroDoc(); + fork.setPeerId(2); + fork.import(atA0); + fork.getText("t").insert(1, "B"); + fork.commit(); + doc.import(fork.export({ mode: "update" })); + + const shallow = shallowOf(doc, target); + // The requested version 2@1 is concurrent with 0@2; loro-crdt uses 0@1. + expect(shallow.shallowSinceFrontiers()).toEqual([{ peer: "1", counter: 0 }]); + expectCritical(doc, shallow.shallowSinceFrontiers()); + shallow.checkout(target); + expect(shallow.getText("t").toString()).toBe("210"); + shallow.checkoutToLatest(); + expect(shallow.getText("t").toString()).toBe("210B"); + }); + + test("keeps concurrent list order through checkouts of the retained range", () => { + const p2 = new LoroDoc(); + p2.setPeerId(2); + const p4 = new LoroDoc(); + p4.setPeerId(4); + p4.getList("l").insert(0, false); + p4.commit(); + p2.getList("l").insert(0, 45); + p2.commit(); + p2.getMap("m").set("k", 1); + p2.commit(); + p2.getMap("m").set("k", 2); + p2.commit(); + p2.import(p4.export({ mode: "update" })); + expect(p2.getList("l").toArray()).toEqual([45, false]); + + // 4@0 is concurrent with 2@1, so the root cannot be 2@1. + const shallow = shallowOf(p2, [{ peer: "2", counter: 1 }]); + expect(shallow.shallowSinceFrontiers()).toEqual([]); + for (const frontiers of [ + [ + { peer: "4", counter: 0 }, + { peer: "2", counter: 2 }, + ], + [{ peer: "2", counter: 2 }], + [{ peer: "2", counter: 1 }], + ] as Frontiers[]) { + shallow.checkout(frontiers); + const full = new LoroDoc(); + full.import(p2.export({ mode: "update" })); + full.checkout(frontiers); + expect(shallow.toJSON()).toEqual(full.toJSON()); + } + shallow.checkoutToLatest(); + // Before the fix the retained concurrent insert came back as [false, 45]. + expect(shallow.getList("l").toArray()).toEqual([45, false]); + }); + + test("is critical on random multi-peer histories", () => { + let random = 0x2c1b_3c6d; + const next = (limit: number): number => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return (random >>> 0) % limit; + }; + for (let round = 0; round < 40; round += 1) { + const docs = [1, 2, 3].map((peer) => { + const doc = new LoroDoc(); + doc.setPeerId(peer); + return doc; + }); + const targets: Frontiers[] = []; + for (let step = 0; step < 24; step += 1) { + const doc = docs[next(docs.length)]!; + if (next(4) === 0) { + const other = docs[next(docs.length)]!; + if (other !== doc) doc.import(other.export({ mode: "update" })); + } else { + const text = doc.getText("t"); + text.insert(next(text.length + 1), String.fromCharCode(97 + next(26))); + doc.commit(); + if (next(3) === 0) targets.push(doc.oplogFrontiers()); + } + } + const doc = docs[0]!; + for (const other of docs.slice(1)) doc.import(other.export({ mode: "update" })); + for (const target of targets) { + const shallow = shallowOf(doc, target); + const root = shallow.shallowSinceFrontiers(); + expectCritical(doc, root); + expect(shallow.toJSON()).toEqual(doc.toJSON()); + const rootVersion = doc.frontiersToVV(root); + for (const version of targets) { + const order = rootVersion.compare(doc.frontiersToVV(version)); + if (order !== -1 && order !== 0) continue; + const full = new LoroDoc(); + full.import(doc.export({ mode: "update" })); + full.checkout(version); + shallow.checkout(version); + expect(shallow.toJSON()).toEqual(full.toJSON()); + } + shallow.checkoutToLatest(); + expect(shallow.toJSON()).toEqual(doc.toJSON()); + } + } + }); +}); From 99e7d84bfa2a4a09c9a643800b9ea0cedf10cac4 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:19:54 +0800 Subject: [PATCH 2/7] fix(loro.js): record the same local ops as loro-crdt (#1145) Three API calls produced different ops than Rust, so the same edits got different op IDs, and a concurrent map set/delete could resolve differently: - Map.delete of an absent key recorded nothing; Rust records a delete. - Counter.increment(0) and attaching a detached counter with value 0 recorded nothing; Rust records the increment. - A text delete across several ID runs was emitted left to right; Rust's TextHandler::delete emits the runs right to left. The old "no semantic no-op edits" test asserted the loro.js behavior; it now asserts Rust's. MovableList set-to-same-value (Rust records it) is left to the MovableList rework. Co-authored-by: Claude Opus 5.5 (1M context) --- .changeset/loro-js-local-op-parity.md | 8 +++++ loro-js/src/runtime/document.ts | 8 ++--- loro-js/tests/runtime.test.ts | 42 ++++++++++++++++++++------- 3 files changed, 43 insertions(+), 15 deletions(-) create mode 100644 .changeset/loro-js-local-op-parity.md diff --git a/.changeset/loro-js-local-op-parity.md b/.changeset/loro-js-local-op-parity.md new file mode 100644 index 000000000..8cb0bf580 --- /dev/null +++ b/.changeset/loro-js-local-op-parity.md @@ -0,0 +1,8 @@ +--- +"loro.js": patch +--- + +`loro.js` now records the same local ops as `loro-crdt` for these API calls, so a document edited through either package gets the same op IDs and the same concurrent outcomes: + +- `LoroMap.delete(key)` records a delete even when the key is absent. Like in Rust, that delete can win against a concurrent `set` of the same key. +- `LoroCounter.increment(0)` records an increment, and attaching a detached counter records an increment by its value, even 0. diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index ecb5bba98..3dbf9c523 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -4026,8 +4026,8 @@ export class LoroDoc = Record = Record { expect(() => root.ensureMergeableMap("regular")).toThrow(/non-mergeable value/u); }); - test("does not record semantic no-op edits", () => { + // loro-crdt skips setting a map key to its current value, but records a + // delete of an absent key and a zero counter increment. Matching it keeps op + // IDs and concurrent outcomes the same for the same API calls. + test("records the same local ops as loro-crdt for no-op-looking edits", () => { const doc = new LoroDoc(); const map = doc.getMap("map"); map.set("value", { nested: [1, true, null] }); - const movable = doc.getMovableList("movable"); - movable.push("same"); - doc.commit(); - const opCount = doc.opCount(); - - map.set("value", { nested: [1, true, null] }); - map.delete("missing"); - movable.set(0, "same"); - doc.getCounter("counter").increment(0); doc.commit(); + const count = (edit: () => void): number => { + const before = doc.opCount(); + edit(); + doc.commit(); + return doc.opCount() - before; + }; - expect(doc.opCount()).toBe(opCount); + expect(count(() => map.set("value", { nested: [1, true, null] }))).toBe(0); + expect(count(() => map.delete("missing"))).toBe(1); + expect(count(() => doc.getCounter("counter").increment(0))).toBe(1); + // Attaching a detached counter increments it by its value, even 0. + expect(count(() => map.setContainer("child", new LoroCounter()))).toBe(2); + }); + + test("lets a delete of an absent map key win against a concurrent set", () => { + const a = new LoroDoc(); + a.setPeerId(1); + const b = new LoroDoc(); + b.setPeerId(2); + a.getMap("map").set("key", 1); + a.commit(); + b.getMap("map").delete("key"); + b.commit(); + a.import(b.export({ mode: "update" })); + b.import(a.export({ mode: "update" })); + // Same lamport; the larger peer's delete wins, as in loro-crdt. + expect(a.toJSON()).toEqual({ map: {} }); + expect(b.toJSON()).toEqual({ map: {} }); }); test("resurfaces preserved state and switches mergeable kinds", () => { From 3435bac26c4fd5994a47e98f2b46ad611ee8f537 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:20:28 +0800 Subject: [PATCH 3/7] fix(loro.js): revive a List child that a replay-fallback event re-inserts (#1150) * fix(loro.js): include reattached child state in diff() and revertTo() A child container without ops in the range was emitted only as a reference, so applyDiff/revertTo recreated it empty. Emit its full state, as Rust does. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): emit full state only for containers that become reachable in diff() Emit a child's whole state only when it is unreachable at `from`, replace a partial diff for such a child, include revived tree node metadata, and order the result parent-first. Moved movable-list children and mergeable children keep only their own ops. applyDiff now moves an existing movable-list child instead of recreating it. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): apply movable-list moves minimally, undo them, and revive tree subtrees in diff() Port Rust's MovableListHandler::apply_delta so a moved child becomes one move instead of reordering the whole list, undo movable-list moves through their recorded neighbors, and emit creates for every node that becomes reachable, including descendants of revived nodes and nodes moved out of a deleted ancestor. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): read a trimmed tree node's placement from the shallow root in diff() On a shallow doc a node created before the root has no retained op at the root version; use its root-state placement instead of treating it as revived. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): resolve movable-list moves by element identity, undo repeated moves, and restore mergeable markers on undo applyDiff now finds moved children through their inserted ids and places every inserted item after its final-order predecessor, so unmerged delete items, several children moved out of one range, and scalar-only deletes are handled without index bookkeeping or scans. Undo reverts earlier moves of an element that only this document moved since. A wide revived subtree is walked iteratively. Undoing a mergeable key deletion re-ensures the marker. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): delete a moved-in live node before recreating a revived subtree in diff() A node alive at `from` inside a subtree revived by the range (e.g. a child moved out of a deleted parent and then reverted back under it) was created again while its old copy stayed. Delete the topmost such nodes first, as Rust's diff does, then create the subtree. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): tracked-span move undo, exact physical undo placement, delta validation, and linear tree revival - Undo only lets later moves the UndoManager tracks (its recorded edits and the inverses it wrote) pass; excluded-origin and remote moves block. - Each move records its physical successor; undo puts the element back there, so redo after mixed moves and inserts restores the original order. - applyDiff validates list deltas for the whole batch before applying, hydrates lazily imported lists before looking for moved children, and deletes only the gaps between kept positions. - Tree revival memoizes "alive at from" along ancestor paths and emits the deletes of moved-in live nodes first, with their index at `from`. - Undo restores a mergeable marker over a later overwrite. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): send the full state of child containers attached by an event When a checkout or import attached a child container again (a map key set back to an older text, a restored list element), the child's event was missing or was a delta against the state it had while hidden. A listener starts an attached child from empty, so it ended up with wrong content, or with an invalid delta ("text span beyond 0"). Like Rust's DocState::apply_diff, revive every child that a batch attaches: its event becomes its whole state, recursively, and events are ordered parent first. A movable-list child that only moved keeps its delta. Uses #attachedChildren and #reachableThroughParentAt from the diff() fix this branch is based on. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): undo a move next to its old physical neighbor that has not moved since The old successor may have been moved by an untracked op; prefer the old predecessor, then the successor, and write no op when the element is already there. Matches Rust in 70 of 72 neighbor-change cases (the other two: Rust revives a deleted element as an empty container). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): bound the undo manager's tracked ranges by its stacks Drop tracked ranges that end before the earliest stacked op of their peer, and all of them on clear/destroy; only later moves are ever checked. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): compute tree delete indexes at `from` once per parent, in apply order Put the diff's own deletes and the moved-in deletes first, resolve each parent's sibling order at `from` from history (nodes that only exist at `from` included), and count indexes with a Fenwick tree as the deletes are applied. Fixes indexes next to new roots and the quadratic rescans. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): validate an applyDiff batch against simulated list lengths Simulate list lengths in batch order before any write: lists created by an earlier entry start empty, repeated entries see the previous result, and the lookup creates nothing (no empty root after a rejected batch). Entries whose bounds were checked skip the second validation. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): align mergeable diff, applyDiff fullState, and events with Rust #1134 Co-Authored-By: Claude Opus 5.5 (1M context) * docs: point the mergeable context at the loro.js alignment Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): validate a hidden mergeable list against its hidden length Co-Authored-By: Claude Opus 5.5 * fix(loro.js): prune undo tracked ranges from the stack bottoms by lamport Pruning visited both stacks on every commit (quadratic sessions), and compared a peer's ranges only with that peer's items, so after a peer change it could drop a range a stacked move still needed. Co-Authored-By: Claude Opus 5.5 * fix(loro.js): emit tree diffs in apply order and reject tree items before writing Tree diff items now apply in order, as in Rust: moves of live nodes (parent first), then the topmost deletes, then revived subtrees, with every index counted in the tree the earlier items leave. applyDiff applies tree items in order and dry-runs them with the list checks, so a failing batch writes nothing. A node hidden at the target version is emitted as a delete. Co-Authored-By: Claude Opus 5.5 * perf(loro.js): memoize hidden tree nodes so deep trees diff in linear time The reachability filter and tree revival checked each node's ancestors without a cache (quadratic in depth). Fractional indexes are now built as flat strings. Co-Authored-By: Claude Opus 5.5 * fix(loro.js): resolve applyDiff tree items and map children as Rust does A create of a node that is alive here moves it, a move of a missing or deleted node creates a new one, and a delete of a deleted node does nothing, so default applyDiff on a doc holding a mergeable tree's hidden state no longer duplicates its nodes. A regular map child is always set as a new container, so its full state is not appended to a hidden copy. Co-Authored-By: Claude Opus 5.5 * docs(loro.js): document fullState for event mirrors and the batch checks Co-Authored-By: Claude Opus 5.5 * docs: add the loro.js tree diff context Co-Authored-By: Claude Opus 5.5 * test(loro.js): check deep-tree linearity by scaling, not wall time The CI runner is several times slower than a dev machine, so the 1.5 s bound flaked; compare 8,000-deep with 2,000-deep instead. Co-Authored-By: Claude Opus 5.5 * test(loro.js): shrink the deep-tree scaling test for slow CI runners Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .changeset/loro-js-revived-child-events.md | 5 + loro-js/src/runtime/document.ts | 9 +- loro-js/tests/revived-child-events.test.ts | 183 +++++++++++++++++++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 .changeset/loro-js-revived-child-events.md create mode 100644 loro-js/tests/revived-child-events.test.ts diff --git a/.changeset/loro-js-revived-child-events.md b/.changeset/loro-js-revived-child-events.md new file mode 100644 index 000000000..8e2cfcce9 --- /dev/null +++ b/.changeset/loro-js-revived-child-events.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +Fix a `loro.js` checkout event for a child container in a List when the checkout falls back to replaying history, for example after updates were imported while the document was detached. The List diff could delete and re-insert an unchanged child container without sending its content, so a listener that starts an inserted child from empty lost that child's content. The child's whole state is now sent, as for any other re-attached child. diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 3dbf9c523..e83adc498 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -2210,6 +2210,7 @@ export class LoroDoc = Record(entries); for (const [id, diff] of diffs) { @@ -2250,10 +2251,15 @@ export class LoroDoc = Record = Record ({ diff --git a/loro-js/tests/revived-child-events.test.ts b/loro-js/tests/revived-child-events.test.ts new file mode 100644 index 000000000..0cec087ff --- /dev/null +++ b/loro-js/tests/revived-child-events.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc, LoroList, LoroText } from "../src/index"; +import type { ContainerID, Delta, LoroEventBatch } from "../src/index"; + +// A child container that an event attaches starts from empty for a listener, +// so, as in Rust's DocState::apply_diff, its event must carry its whole state +// ("revival") rather than a delta against the state it had while hidden. + +function peers(count: number): LoroDoc[] { + return Array.from({ length: count }, (_, index) => { + const doc = new LoroDoc(); + doc.setPeerId(index + 1); + return doc; + }); +} + +function merged(...docs: LoroDoc[]): LoroDoc { + const doc = new LoroDoc(); + for (const source of docs) doc.import(source.export({ mode: "update" })); + return doc; +} + +function eventsOf(batches: readonly LoroEventBatch[]): unknown[] { + return batches.flatMap((batch) => + batch.events.map((event) => [ + event.target, + JSON.parse(JSON.stringify(event.diff)) as unknown, + ]), + ); +} + +describe("events for child containers attached by a checkout", () => { + // The expected events are exactly what loro-crdt emits. + test("carry the full state of a map child set back from null", () => { + const [p1] = peers(1) as [LoroDoc]; + const text = p1.getMap("map").setContainer("key", new LoroText()); + text.insert(0, "πŸ˜€"); + p1.getMap("map").set("key", null); + p1.commit(); + const doc = merged(p1); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + doc.checkout([{ peer: "1", counter: 1 }]); + expect(doc.toJSON()).toEqual({ map: { key: "πŸ˜€" } }); + expect(eventsOf(batches)).toEqual([ + ["cid:root-map:Map", { type: "map", updated: { key: "πŸ˜€" } }], + [text.id, { type: "text", diff: [{ insert: "πŸ˜€" }] }], + ]); + }); + + test("carry the full state of a list child whose element is restored", () => { + const [p1] = peers(1) as [LoroDoc]; + const child = p1.getList("list").insertContainer(0, new LoroList()); + p1.commit(); + child.insert(0, 5); + p1.commit(); + p1.getList("list").delete(0, 1); + p1.commit(); + const doc = merged(p1); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + doc.checkout([{ peer: "1", counter: 1 }]); + expect(doc.toJSON()).toEqual({ list: [[5]] }); + expect(eventsOf(batches)).toEqual([ + ["cid:root-list:List", { type: "list", diff: [{ insert: [[5]] }] }], + [child.id, { type: "list", diff: [{ insert: [5] }] }], + ]); + }); + + test("replace a delta against the hidden state with the full state", () => { + const [p1, p2] = peers(2) as [LoroDoc, LoroDoc]; + const text = p1.getMap("map").setContainer("key", new LoroText()); + p1.commit(); + p2.import(p1.export({ mode: "update" })); + // Edited concurrently with the key being overwritten. + (p2.getContainerById(text.id) as LoroText).insert(0, "ab"); + p2.commit(); + p1.getMap("map").set("key", "scalar"); + p1.commit(); + const doc = merged(p1, p2); + expect(doc.toJSON()).toEqual({ map: { key: "scalar" } }); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + doc.checkout([ + { peer: "1", counter: 0 }, + { peer: "2", counter: 1 }, + ]); + expect(doc.toJSON()).toEqual({ map: { key: "ab" } }); + // Before, the text event deleted "ab" from a child the listener had just + // created empty. + expect(eventsOf(batches)).toEqual([ + ["cid:root-map:Map", { type: "map", updated: { key: "ab" } }], + [text.id, { type: "text", diff: [{ insert: "ab" }] }], + ]); + }); + + test("revive a list child that a whole-value list diff deletes and re-inserts", () => { + const [p1, p2, p3] = peers(3) as [LoroDoc, LoroDoc, LoroDoc]; + p1.getList("list").insert(0, "s"); + p1.commit(); + p2.import(p1.export({ mode: "update" })); + const text = p2.getList("list").insertContainer(1, new LoroText()); + text.insert(0, "a"); + p2.commit(); + p3.import(p1.export({ mode: "update" })); + p3.import(p2.export({ mode: "update" })); + const version = p3.frontiers(); + p1.getList("list").insert(1, true); + p1.commit(); + p3.import(p1.export({ mode: "update" })); + p3.checkout(version); + // Imported while detached, so checkoutToLatest replays and diffs values. + p2.getList("list").push(false); + p2.commit(); + p3.import(p2.export({ mode: "update", from: p3.oplogVersion() })); + + const batches: LoroEventBatch[] = []; + p3.subscribe((batch) => batches.push(batch)); + const model = new Map([ + ["cid:root-list:List", ["s", { child: text.id }]], + [text.id, "a"], + ]); + p3.checkoutToLatest(); + // Apply the events the way a listener does: an inserted child restarts + // from empty. + for (const event of batches.flatMap((batch) => batch.events)) { + if (event.diff.type === "text") { + model.set( + event.target, + applyText(model.get(event.target) as string, event.diff.diff), + ); + } else if (event.diff.type === "list") { + const next: unknown[] = []; + const current = (model.get(event.target) ?? []) as unknown[]; + let index = 0; + for (const item of event.diff.diff) { + if ("retain" in item) { + next.push(...current.slice(index, index + item.retain)); + index += item.retain; + } else if ("delete" in item) { + index += item.delete; + } else { + for (const value of item.insert) { + if (value instanceof LoroText) { + model.set(value.id, ""); + next.push({ child: value.id }); + } else { + next.push(value); + } + } + } + } + next.push(...current.slice(index)); + model.set(event.target, next); + } + } + const list = model.get("cid:root-list:List") as unknown[]; + const resolved = list.map((value) => + typeof value === "object" && value !== null && "child" in value + ? model.get((value as { child: ContainerID }).child) + : value, + ); + expect(p3.toJSON()).toEqual({ list: ["s", true, "a", false] }); + expect(resolved).toEqual(["s", true, "a", false]); + }); +}); + +function applyText(text: string, delta: readonly Delta[]): string { + let output = ""; + let cursor = 0; + for (const item of delta) { + if ("retain" in item) { + output += text.slice(cursor, cursor + item.retain); + cursor += item.retain; + } else if ("delete" in item) { + cursor += item.delete; + } else { + output += item.insert; + } + } + return output + text.slice(cursor); +} From 0ab8e3872ab0395f82a9bbcff5ba4e85c063415d Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:21:01 +0800 Subject: [PATCH 4/7] fix(loro.js): keep a checkout of the current latest version attached (#1143) Rust's checkout returns early when the target equals the state's frontiers, re-attaching when they are the latest frontiers, and also when the shrunk target equals them. loro.js always detached, so `doc.checkout(doc.frontiers())` stopped later imports from reaching the state and made loro.js peers diverge from loro-crdt ones that ran the same calls. The old detached-import test relied on this; it now uses detach(). Co-authored-by: Claude Opus 5.5 (1M context) --- .../loro-js-checkout-current-attached.md | 5 ++ loro-js/src/runtime/document.ts | 22 +++++++++ loro-js/tests/runtime.test.ts | 49 ++++++++++++++++++- 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 .changeset/loro-js-checkout-current-attached.md diff --git a/.changeset/loro-js-checkout-current-attached.md b/.changeset/loro-js-checkout-current-attached.md new file mode 100644 index 000000000..cac9db071 --- /dev/null +++ b/.changeset/loro-js-checkout-current-attached.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +`checkout(frontiers)` on a `loro.js` document now behaves like `loro-crdt` when `frontiers` is the current version: nothing changes, and the document stays (or becomes) attached when that version is the latest one. Before, `doc.checkout(doc.frontiers())` detached the document, so later imports were not applied to its state. Checking out the latest version from an older one still leaves the document detached, as in Rust. diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index e83adc498..c1fc58a78 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -3015,9 +3015,25 @@ export class LoroDoc = Record `${id.counter}@${id.peer}`)); + return right.every((id) => keys.has(`${id.counter}@${id.peer}`)); +} + interface StyleRedaction { /** Start IDs (`peer:counter`) of the pairs whose values were nulled. */ readonly redacted: Set; diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 832aeaef6..5d23e8e6b 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -349,8 +349,7 @@ describe("loro-wasm-compatible runtime", () => { const doc = new LoroDoc(); doc.setPeerId(2); doc.import(base); - const checkedOut = doc.frontiers(); - doc.checkout(checkedOut); + doc.detach(); source.getText("text").push(" latest"); doc.import(source.export({ mode: "update", from: doc.oplogVersion() })); @@ -361,6 +360,52 @@ describe("loro-wasm-compatible runtime", () => { expect(doc.getText("text").toString()).toBe("base latest"); }); + // Rust's checkout returns early when the target is the current version and + // re-attaches when that version is the latest one. + test("keeps a checkout of the current version attached like loro-crdt", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("text").insert(0, "x"); + doc.commit(); + doc.getText("text").insert(1, "y"); + doc.commit(); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + + doc.checkout(doc.frontiers()); + expect(doc.isDetached()).toBe(false); + expect(batches).toHaveLength(0); + const remote = new LoroDoc(); + remote.setPeerId(2); + remote.getText("text").insert(0, "B"); + remote.commit(); + doc.import(remote.export({ mode: "update" })); + expect(doc.getText("text").toString()).toBe("xyB"); + + // A dominated extra frontier still names the current version. + doc.checkout([...doc.frontiers(), { peer: "1", counter: 0 }]); + expect(doc.isDetached()).toBe(false); + + doc.checkout([{ peer: "1", counter: 0 }]); + doc.checkout([{ peer: "1", counter: 0 }]); + expect(doc.isDetached()).toBe(true); + expect(doc.getText("text").toString()).toBe("x"); + // Checking out the latest version from an older one stays detached. + doc.checkout(doc.oplogFrontiers()); + expect(doc.isDetached()).toBe(true); + expect(doc.getText("text").toString()).toBe("xyB"); + + // detach() at the latest version, then checkout of it, re-attaches. + doc.checkoutToLatest(); + doc.detach(); + doc.checkout(doc.frontiers()); + expect(doc.isDetached()).toBe(false); + doc.getText("text").insert(0, "!"); + doc.commit(); + expect(doc.getText("text").toString()).toBe("!xyB"); + expect(doc.version().compare(doc.oplogVersion())).toBe(0); + }); + test("holds causally incomplete updates pending until dependencies arrive", () => { const source = new LoroDoc(); source.setPeerId(1); From 53659920e4bcb30f9b3f4c4ed042eb938004d374 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:21:35 +0800 Subject: [PATCH 5/7] test(loro.js): add a multi-peer differential fuzz harness (#1141) * test(loro.js): add a multi-peer differential fuzz harness Random multi-peer edits, update exchanges in several orders, and checkouts, checked against replay, checkout-at-version, snapshot, shallow-snapshot, and event-mirror oracles, and optionally step by step against a loro-crdt WASM build. Failures are grouped by signature, minimized, and printed as loro.js calls. The default run is a fixed 120-seed Map/List/Text/Counter set that needs no WASM build (~2 s). Seeds that fail on main are listed with the fix that makes them pass. Usage and findings: context/loro-js-differential-fuzz.md. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(loro.js): record the fuzz findings and their PRs Co-Authored-By: Claude Opus 5.5 (1M context) * docs(loro.js): state which PR the lockstep measurement used Co-Authored-By: Claude Opus 5.5 * test(loro.js): run nested-container fuzz seeds in CI Seeds 0-119 with nested children and snapshot imports pass on main now that #1131 is in. See context/loro-js-differential-fuzz.md. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 + context/loro-js-differential-fuzz.md | 135 ++ loro-js/AGENTS.md | 3 + loro-js/tests/differential-fuzz.test.ts | 196 +++ loro-js/tests/fuzz/differential.ts | 1648 +++++++++++++++++++++++ 5 files changed, 1984 insertions(+) create mode 100644 context/loro-js-differential-fuzz.md create mode 100644 loro-js/tests/differential-fuzz.test.ts create mode 100644 loro-js/tests/fuzz/differential.ts diff --git a/AGENTS.md b/AGENTS.md index 523a73b39..5beac63ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,8 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. work around: [context/loro-js-rust-differential.md](context/loro-js-rust-differential.md). - loro.js rich-text style anchors, entity positions, and Rust's insert/delete/mark position rules: [context/loro-js-richtext-anchors.md](context/loro-js-richtext-anchors.md). +- loro.js multi-peer differential fuzzing against replay, checkout, event, and + Rust/WASM oracles: [context/loro-js-differential-fuzz.md](context/loro-js-differential-fuzz.md). - WASM panic/OOM reporting channels, `__wbindgen_start` glue invariant, and trap-testing recipes: [context/wasm-error-reporting.md](context/wasm-error-reporting.md). diff --git a/context/loro-js-differential-fuzz.md b/context/loro-js-differential-fuzz.md new file mode 100644 index 000000000..f29a9848c --- /dev/null +++ b/context/loro-js-differential-fuzz.md @@ -0,0 +1,135 @@ +# loro.js Differential Fuzzing + +Verified against code 2026-09-29. + +`loro-js/tests/fuzz/differential.ts` drives several `loro.js` peers through +random edits, update exchanges, and checkouts, and checks them against +independent oracles. `loro-js/tests/differential-fuzz.test.ts` runs a fixed, +CI-sized seed set by default and larger runs through environment variables. + +## Scenario model + +A scenario is a list of abstract actions: a local edit (`op`), an update +exchange (`sync`), or a checkout. Each action carries random 32-bit parameters +that are resolved against the acting document when it runs (container, position, +value). Removing actions therefore keeps the rest meaningful, which is what +`minimizeScenario` (delta debugging on the action list) relies on. + +- Containers: root Map/List/Text/Tree/Counter/MovableList, and child + containers in maps, lists, and tree metadata (`FuzzFeatures`). + MovableList and rich-text marks are off in `CORE_FEATURES`. +- Syncs: `update` from the receiver's oplog version, full updates, one blob per + missing change in a shuffled order (`import` or `importBatch`), and optionally + full snapshots into a non-empty document. +- Checkouts go to a random op or to the union of two ops, and either return to + the latest version or leave the peer detached until its next edit. + +## Oracles + +- Every peer, after a final full exchange, equals a document that imports all + updates at once, and one that imports each change separately in a shuffled + order. +- `checkout(F)` equals a fresh document that imports only the changes in `F` + (`updates-in-range` from counter 0), for every op of a small history. +- A full snapshot and three shallow snapshots equal the full-history document, + including checkouts into the retained range and back. +- `EventMirror` applies every event batch to a model keyed by container ID and + must reproduce `toJSON()` after each commit, import, and checkout. It follows + the Rust fuzzer's `ContainerTracker` (`crates/fuzz/src/value.rs`): attaching a + container value restarts that child from an empty state, a movable-list move + keeps the moved child's state, tree `delete` removes the subtree, and every + sequence index must be valid when the items are applied in order. + +## Rust/WASM reference + +With `FUZZ_WASM=1` (or a path to `loro_wasm.js`), each peer also has a +`loro-crdt` twin with the same peer ID. In lockstep mode (default when WASM is +loaded) every concrete edit, sync, and checkout runs on both, and the values +are compared after each action. The first mismatch names the action where the +implementations diverge. After the scenario, a Rust replay, Rust checkouts, and +shallow snapshots exchanged in both directions are compared too, including the +shallow root each side picks. `FUZZ_VALIDATE_HARNESS=1` checks only Rust's own +event mirror; use it after changing `EventMirror`. + +`FUZZ_WASM=1` loads the same reference as the other differential suites +(`tests/support/rust-reference.ts`, `LORO_WASM_NODEJS` overrides the path). +Lockstep requires both implementations to emit the same local ops for the same +API calls; an op-generation difference shows up as `lockstep` or +`lockstep-throws` on the first affected action. On `main` without #1145 and #1143, +most lockstep seeds diverge at the first delete of an absent map key, zero +counter increment, or checkout of the current version. With #1145 alone, 49 of +50 Map/List/Text/Counter seeds matched Rust step by step; the remaining one is +the #1143 case. + +## Running + +```sh +cd loro-js +pnpm vitest run tests/differential-fuzz.test.ts # fixed CI seeds +FUZZ_SEEDS=0-499 FUZZ_FEATURES=core FUZZ_WASM=1 FUZZ_MINIMIZE=1 \ + FUZZ_OUT=/tmp/report.json pnpm vitest run tests/differential-fuzz.test.ts +``` + +`FUZZ_CHECKS` narrows the oracles (`events,checkout,snapshot,shallow`), +`FUZZ_LOCKSTEP=0` compares with Rust only after the scenario, and `FUZZ_TRACE=1` +prints each action as it runs, which finds hangs. The report groups failures by +signature (oracle plus first differing root or normalized error) and stores the +minimized action list and a trace written as `loro.js` API calls. + +CI runs 120 fixed seeds of Map/List/Text/Counter, and 120 more with nested +child containers and snapshot imports, with the replay, checkout, +full-snapshot, and event oracles. A seed that fails on `main` for a known +reason goes into `pending` with that reason until its fix lands; both lists are +empty on 2026-09-29 (`main` at #1131). + +## Divergences found (2026-09-28/29) + +Fixed on `main`: Fugue insert order after the origin's last child, doubled +checkout events for concurrent deletes, and text delete run order (#1136); +deletes imported while detached (#1126). + +Open PRs: a `checkout(currentFrontiers)` that detached (#1143), shallow roots +that were not critical versions (#1144; without it shallow snapshots still +give wrong values after retained checkouts, sometimes also at the latest +version), absent-key map deletes and zero counter increments (#1145), +liveness of containers under deleted ancestors (#1147), local tree positions +and no-op moves (#1148), and a List child that a replay-fallback event +re-inserts (#1150). With nested children and snapshot imports, seeds 297, +315, 344, 422, 452, and 559 of 0–599 fail on `main` in the event oracle; +#1150 fixes all six. + +Still open, so `CI_RUNS` does not enable them yet: + +- Tree convergence: `loro.js` applies tree moves per node, last writer wins, + with no cycle check. Two peers that move `x` under `y` and `y` under `x` + converge to a cycle, so both nodes disappear (Rust keeps `y` as a root with + child `x`). Rust applies a tree's ops in (lamport, peer) order and skips a + move whose new parent descends from the target (`TreeCacheForDiff::apply`, + `diff_calc/tree.rs`). A port needs, per tree, the ops in that order with an + effective flag. A new op applies directly when it is the newest; otherwise + the suffix after it is retreated and reapplied. Version transitions replay + the suffix from the lowest changed op, and the shallow-root placement + fallback from #1127 applies when a node's retained ops are all retreated. +- Tree events: items are a before/after snapshot diff that lists final + indexes in op order. Importing two roots created at index 0 emits + `create 0@2 @1, create 1@2 @0` (Rust: `@0, @0`); creating a child and + deleting its parent in one commit emits the delete first; reviving a node + omits `create` for its children. Rust records an item per effective op at + the moment it is applied, plus a `create` for each child of a revived node + (`TreeState::apply_diff_and_convert`). This belongs with the port above. +- Events for containers that are unreachable after the batch: Rust drops them + (`DocState::get_path`); `loro.js` still sends them. Needs #1147's liveness. +- Shallow snapshots in the harness need #1144 (the root check fails first on + every seed without it). +- MovableList: `loro.js` skips a movable-list `set` to the current value, + which Rust records. + +## Rust-side findings + +Found while comparing; all four reproduce on the 1.16 release and are fixed +on `main` (checked on 2026-09-29 at `c0ff1255`, after #1135, #1152, and +#1153): an `Op/hint length mismatch` panic when deleting across an astral +character after `checkout` + `attach` with a subscriber; a `tree_state.rs` +panic in a subscribed checkout sequence; a tree node kept by a checkout path +between concurrent versions; and tree metadata left marked deleted after a +local move revived it. diff --git a/loro-js/AGENTS.md b/loro-js/AGENTS.md index 8b6be0101..3e46a5bb0 100644 --- a/loro-js/AGENTS.md +++ b/loro-js/AGENTS.md @@ -9,6 +9,9 @@ the same binary and JSON data as Rust (`loro-crdt`) and give the same results. [context/loro-js-performance.md](../context/loro-js-performance.md). - Randomized Text comparison with Rust (build the nodejs WASM package first): [context/loro-js-rust-differential.md](../context/loro-js-rust-differential.md). +- Multi-peer fuzzing of all containers against replay, checkout, event, and + shallow-snapshot oracles, optionally in lockstep with Rust: + [context/loro-js-differential-fuzz.md](../context/loro-js-differential-fuzz.md). - Rich text: style anchors live in the Text sequence and op positions count them. Read [context/loro-js-richtext-anchors.md](../context/loro-js-richtext-anchors.md) before changing Text positions, marks, or Text snapshot state. diff --git a/loro-js/tests/differential-fuzz.test.ts b/loro-js/tests/differential-fuzz.test.ts new file mode 100644 index 000000000..04ea92fa1 --- /dev/null +++ b/loro-js/tests/differential-fuzz.test.ts @@ -0,0 +1,196 @@ +import { writeFileSync, writeSync } from "node:fs"; +import { createRequire } from "node:module"; + +import { describe, expect, test } from "vitest"; + +import { + ALL_CHECKS, + ALL_FEATURES, + CORE_FEATURES, + describeScenario, + generateScenario, + minimizeScenario, + runScenario, +} from "./fuzz/differential"; +import type { FuzzChecks, FuzzFeatures, RustOracle } from "./fuzz/differential"; +import { loadRustReference } from "./support/rust-reference"; + +// See context/loro-js-differential-fuzz.md. Without FUZZ_SEEDS this file runs +// the fixed CI seeds below. Large runs, e.g.: +// +// FUZZ_SEEDS=0-999 FUZZ_FEATURES=core FUZZ_WASM=1 FUZZ_MINIMIZE=1 \ +// FUZZ_OUT=/tmp/report.json pnpm vitest run tests/differential-fuzz.test.ts +// +// FUZZ_FEATURES: core | all | comma list of FuzzFeatures keys. +// FUZZ_CHECKS: comma list of FuzzChecks keys (default: all). +// FUZZ_WASM: 1 for the reference in tests/support/rust-reference.ts (honors +// LORO_WASM_NODEJS), or a path to a loro-crdt nodejs build. +// FUZZ_LOCKSTEP: 0 to compare with Rust only after the scenario. +// FUZZ_ACTIONS: actions per scenario. FUZZ_TRACE: print actions as they run. +// FUZZ_VALIDATE_HARNESS: check only Rust's own event mirror. +const env = process.env; + +const MAP_LIST_TEXT_COUNTER: FuzzFeatures = { + ...CORE_FEATURES, + tree: false, + nested: false, +}; + +/** + * Fixed seeds that must pass on every change. They need no WASM build. + * `pending` seeds fail on main for the reasons listed; move each back into the + * run once its fix lands. + */ +const CI_RUNS: readonly { + readonly name: string; + readonly features: FuzzFeatures; + readonly checks: FuzzChecks; + readonly seeds: readonly number[]; + readonly pending: ReadonlyMap; +}[] = [ + { + name: "Map/List/Text/Counter: replay, checkout, snapshot, and event oracles", + features: MAP_LIST_TEXT_COUNTER, + checks: { events: true, checkout: true, snapshot: true, shallow: false }, + seeds: Array.from({ length: 120 }, (_, index) => index), + pending: new Map(), + }, + { + name: "nested child containers and snapshot imports: replay, checkout, snapshot, and event oracles", + features: { ...MAP_LIST_TEXT_COUNTER, nested: true, snapshotSync: true }, + checks: { events: true, checkout: true, snapshot: true, shallow: false }, + seeds: Array.from({ length: 120 }, (_, index) => index), + pending: new Map(), + }, +]; + +describe.skipIf(env.FUZZ_SEEDS !== undefined)("differential fuzz (fixed seeds)", () => { + test.each(CI_RUNS)( + "$name", + (run) => { + const failures: string[] = []; + for (const seed of run.seeds) { + if (run.pending.has(seed)) continue; + const scenario = generateScenario(seed, { features: run.features }); + const failure = runScenario(scenario, { checks: run.checks }); + if (failure !== undefined) { + failures.push( + `seed ${seed}: ${failure.message}\n${failure.trace.slice(-40).join("\n")}`, + ); + } + } + expect(failures).toEqual([]); + }, + 120_000, + ); + + test("generates the same scenario for a seed", () => { + expect(describeScenario(generateScenario(7))).toBe( + describeScenario(generateScenario(7)), + ); + expect(describeScenario(generateScenario(7))).not.toBe( + describeScenario(generateScenario(8)), + ); + }); +}); + +function parseSeeds(input: string): number[] { + const seeds: number[] = []; + for (const part of input.split(",")) { + const [start, end] = part.split("-").map(Number); + if (end === undefined) seeds.push(start!); + else for (let seed = start!; seed <= end; seed += 1) seeds.push(seed); + } + return seeds; +} + +function parseFeatures(input: string | undefined): FuzzFeatures { + if (input === undefined || input === "core") return CORE_FEATURES; + if (input === "all") return ALL_FEATURES; + const enabled = new Set(input.split(",")); + const features: Record = {}; + for (const key of Object.keys(ALL_FEATURES)) features[key] = enabled.has(key); + return features as unknown as FuzzFeatures; +} + +function parseChecks(input: string | undefined): FuzzChecks { + if (input === undefined) return ALL_CHECKS; + const enabled = new Set(input.split(",")); + return { + events: enabled.has("events"), + checkout: enabled.has("checkout"), + snapshot: enabled.has("snapshot"), + shallow: enabled.has("shallow"), + }; +} + +function loadRust(): RustOracle | undefined { + if (env.FUZZ_WASM === undefined) return undefined; + const wasm = ( + env.FUZZ_WASM === "1" + ? loadRustReference() + : createRequire(import.meta.url)(env.FUZZ_WASM) + ) as + | (Record & { + LoroDoc: new () => ReturnType; + callPendingEvents(): void; + }) + | undefined; + if (wasm === undefined) + throw new Error("FUZZ_WASM=1 needs the Rust WASM reference build"); + return { + newDoc: () => new wasm.LoroDoc(), + newContainer: (type) => new (wasm[`Loro${type}`] as new () => unknown)(), + flushEvents: () => wasm.callPendingEvents(), + }; +} + +describe.runIf(env.FUZZ_SEEDS !== undefined)("differential fuzz (manual)", () => { + test("runs the requested seeds", () => { + const seeds = parseSeeds(env.FUZZ_SEEDS!); + const features = parseFeatures(env.FUZZ_FEATURES); + const rust = loadRust(); + const checks = parseChecks(env.FUZZ_CHECKS); + const runOptions = { + rust, + checks, + lockstep: env.FUZZ_LOCKSTEP !== "0", + validateHarness: env.FUZZ_VALIDATE_HARNESS !== undefined, + onTrace: + env.FUZZ_TRACE === undefined + ? undefined + : (line: string) => writeSync(2, `${line}\n`), + }; + const actions = env.FUZZ_ACTIONS === undefined ? undefined : Number(env.FUZZ_ACTIONS); + const report: unknown[] = []; + const counts = new Map(); + for (const seed of seeds) { + const scenario = generateScenario(seed, { + features, + ...(actions === undefined ? {} : { actions }), + }); + const failure = runScenario(scenario, runOptions); + if (failure === undefined) continue; + counts.set(failure.signature, (counts.get(failure.signature) ?? 0) + 1); + let minimized = scenario; + let minimalFailure = failure; + if (env.FUZZ_MINIMIZE !== undefined) { + minimized = minimizeScenario(scenario, failure.signature, runOptions); + minimalFailure = runScenario(minimized, runOptions) ?? failure; + } + report.push({ + seed, + signature: failure.signature, + message: minimalFailure.message, + actions: minimized.actions.length, + trace: minimalFailure.trace, + scenario: JSON.parse(describeScenario(minimized)), + }); + } + const summary = [...counts.entries()].sort((a, b) => b[1] - a[1]); + if (env.FUZZ_OUT !== undefined) { + writeFileSync(env.FUZZ_OUT, JSON.stringify({ summary, report }, null, 2)); + } + expect({ failures: report.length, summary }).toEqual({ failures: 0, summary: [] }); + }, 3_600_000); +}); diff --git a/loro-js/tests/fuzz/differential.ts b/loro-js/tests/fuzz/differential.ts new file mode 100644 index 000000000..f95f548f8 --- /dev/null +++ b/loro-js/tests/fuzz/differential.ts @@ -0,0 +1,1648 @@ +// Differential fuzzing harness for the TypeScript runtime. +// +// A scenario is a list of abstract actions (local edits, update exchanges in +// several orders, checkouts). Each action carries random 32-bit parameters that +// are resolved against the acting document when it runs, so removing actions +// while minimizing keeps the remaining ones meaningful. +// +// Oracles: +// - every peer converges with a document that replays the same updates at once +// and with one that imports each change separately in a shuffled order; +// - events mirrored into a container-keyed model reproduce `toJSON()` after +// every local commit, import, and checkout; +// - `checkout(F)` equals a fresh document that imports only the changes in F; +// - a full snapshot and shallow snapshots equal the full-history document, +// including checkouts into the retained range; +// - optionally, a Rust/WASM `loro-crdt` build replays the same updates, checks +// out the same versions, and exchanges shallow snapshots with the runtime. +import { + LoroCounter, + LoroDoc, + LoroList, + LoroMap, + LoroMovableList, + LoroText, + LoroTree, +} from "../../src/runtime"; +import type { Container } from "../../src/runtime"; +import type { + ContainerID, + ContainerType, + Delta, + Frontiers, + LoroEventBatch, + OpId, + TreeDiffItem, + TreeID, +} from "../../src/runtime/types"; + +export interface FuzzFeatures { + readonly map: boolean; + readonly list: boolean; + readonly text: boolean; + readonly tree: boolean; + readonly counter: boolean; + readonly movableList: boolean; + /** Rich-text `mark`/`unmark`. */ + readonly textStyle: boolean; + /** Child containers inside maps, lists, and tree metadata. */ + readonly nested: boolean; + /** Import full snapshots into documents that already have history. */ + readonly snapshotSync: boolean; +} + +export const ALL_FEATURES: FuzzFeatures = { + map: true, + list: true, + text: true, + tree: true, + counter: true, + movableList: true, + textStyle: true, + nested: true, + snapshotSync: false, +}; + +export const CORE_FEATURES: FuzzFeatures = { + ...ALL_FEATURES, + movableList: false, + textStyle: false, +}; + +export interface FuzzChecks { + readonly events: boolean; + readonly checkout: boolean; + readonly snapshot: boolean; + readonly shallow: boolean; +} + +export const ALL_CHECKS: FuzzChecks = { + events: true, + checkout: true, + snapshot: true, + shallow: true, +}; + +export type SyncMode = "update" | "full" | "shuffled" | "batch" | "snapshot"; + +export type Action = + | { + readonly kind: "op"; + readonly peer: number; + readonly target: number; + readonly op: number; + readonly a: number; + readonly b: number; + readonly commit: boolean; + } + | { + readonly kind: "sync"; + readonly from: number; + readonly to: number; + readonly mode: SyncMode; + readonly r: number; + } + | { + readonly kind: "checkout"; + readonly peer: number; + readonly r: number; + readonly r2: number; + readonly stay: boolean; + }; + +export interface Scenario { + readonly seed: number; + readonly peers: number; + readonly features: FuzzFeatures; + readonly actions: readonly Action[]; +} + +/** A structural subset of the `loro-crdt` WASM `LoroDoc` used by the Rust oracle. */ +export interface RustDoc { + setPeerId(peer: bigint | number): void; + import(bytes: Uint8Array): unknown; + importBatch(blobs: Uint8Array[]): unknown; + export(mode: unknown): Uint8Array; + toJSON(): unknown; + commit(): void; + checkout(frontiers: Frontiers): void; + checkoutToLatest(): void; + attach(): void; + oplogVersion(): unknown; + shallowSinceFrontiers(): Frontiers; + configTextStyle(styles: unknown): void; + subscribe(listener: (batch: LoroEventBatch) => void): unknown; + getContainerById(id: ContainerID): unknown; +} + +export interface RustOracle { + newDoc(): RustDoc; + /** A detached container of the given type. */ + newContainer(type: ContainerType): unknown; + /** Delivers queued WASM events (`callPendingEvents`). */ + flushEvents(): void; +} + +export interface FuzzFailure { + readonly check: string; + readonly signature: string; + readonly message: string; + readonly trace: readonly string[]; +} + +export class FuzzMismatch extends Error { + constructor( + readonly check: string, + readonly key: string, + message: string, + ) { + super(message); + } +} + +// --------------------------------------------------------------------------- +// Random generation + +export class Random { + #state: number; + + constructor(seed: number) { + this.#state = (seed ^ 0x9e37_79b9) >>> 0 || 1; + for (let index = 0; index < 4; index += 1) this.next(); + } + + next(): number { + let x = this.#state; + x ^= x << 13; + x >>>= 0; + x ^= x >>> 17; + x ^= x << 5; + x >>>= 0; + this.#state = x; + return x; + } + + below(limit: number): number { + return limit <= 0 ? 0 : this.next() % limit; + } + + chance(numerator: number, denominator: number): boolean { + return this.below(denominator) < numerator; + } +} + +export interface GenerateOptions { + readonly peers?: number; + readonly actions?: number; + readonly features?: FuzzFeatures; +} + +export function generateScenario(seed: number, options: GenerateOptions = {}): Scenario { + const random = new Random(seed); + const peers = options.peers ?? 2 + random.below(3); + const count = options.actions ?? 30 + random.below(50); + const features = options.features ?? CORE_FEATURES; + const syncModes: SyncMode[] = ["update", "update", "full", "shuffled", "batch"]; + if (features.snapshotSync) syncModes.push("snapshot"); + const actions: Action[] = []; + for (let index = 0; index < count; index += 1) { + const roll = random.below(100); + if (roll < 66) { + actions.push({ + kind: "op", + peer: random.below(peers), + target: random.next(), + op: random.next(), + a: random.next(), + b: random.next(), + commit: random.chance(3, 4), + }); + } else if (roll < 91) { + const from = random.below(peers); + let to = random.below(peers - 1); + if (to >= from) to += 1; + actions.push({ + kind: "sync", + from, + to, + mode: syncModes[random.below(syncModes.length)]!, + r: random.next(), + }); + } else { + actions.push({ + kind: "checkout", + peer: random.below(peers), + r: random.next(), + r2: random.next(), + stay: random.chance(1, 3), + }); + } + } + return { seed, peers, features, actions }; +} + +// --------------------------------------------------------------------------- +// Value normalization + +const ROOTS: readonly { + name: string; + type: ContainerType; + feature: keyof FuzzFeatures; +}[] = [ + { name: "m", type: "Map", feature: "map" }, + { name: "l", type: "List", feature: "list" }, + { name: "t", type: "Text", feature: "text" }, + { name: "tr", type: "Tree", feature: "tree" }, + { name: "c", type: "Counter", feature: "counter" }, + { name: "ml", type: "MovableList", feature: "movableList" }, +]; + +const TEXT_STYLES = { + bold: { expand: "after" }, + link: { expand: "none" }, + comment: { expand: "both" }, + em: { expand: "before" }, +} as const; + +function sortValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortValue); + if (value instanceof Uint8Array) return Array.from(value); + if (value !== null && typeof value === "object") { + const output: Record = {}; + for (const key of Object.keys(value).sort()) { + const child = (value as Record)[key]; + if (child !== undefined) output[key] = sortValue(child); + } + return output; + } + return value; +} + +function isEmptyValue(value: unknown): boolean { + if (value === "" || value === 0 || value === null || value === undefined) return true; + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "object") return Object.keys(value as object).length === 0; + return false; +} + +/** Deep JSON with sorted keys and empty root containers removed. */ +export function normalizeJson(value: unknown): Record { + const sorted = sortValue(value) as Record; + const output: Record = {}; + for (const [key, child] of Object.entries(sorted)) { + if (!isEmptyValue(child)) output[key] = child; + } + return output; +} + +interface ContainerLike { + readonly id: ContainerID; + kind(): string; +} + +function isContainerLike(value: unknown): value is ContainerLike { + return ( + value !== null && + typeof value === "object" && + typeof (value as { kind?: unknown }).kind === "function" && + typeof (value as { id?: unknown }).id === "string" + ); +} + +/** Rich-text deltas of every text reachable through maps and lists, keyed by id. */ +function collectTextDeltas( + doc: { + getContainerById(id: ContainerID): unknown; + }, + json: unknown, +): Record { + const output: Record = {}; + const visit = (handle: unknown): void => { + if (!isContainerLike(handle)) return; + const kind = handle.kind(); + if (kind === "Text") { + output[handle.id] = sortValue((handle as unknown as LoroText).toDelta()); + } else if (kind === "Map") { + const map = handle as unknown as LoroMap; + for (const key of map.keys().sort()) visit(map.get(key)); + } else if (kind === "List" || kind === "MovableList") { + const list = handle as unknown as LoroList; + for (let index = 0; index < list.length; index += 1) visit(list.get(index)); + } + }; + for (const root of ROOTS) { + if (json !== null && typeof json === "object" && root.name in json) { + visit(doc.getContainerById(`cid:root-${root.name}:${root.type}`)); + } + } + return output; +} + +export interface NormalizedValue { + readonly json: Record; + readonly texts?: Record; +} + +export function docValue( + doc: { toJSON(): unknown; getContainerById(id: ContainerID): unknown }, + withStyles: boolean, +): NormalizedValue { + const raw = doc.toJSON(); + const json = normalizeJson(raw); + if (!withStyles) return { json }; + const texts = collectTextDeltas(doc, raw); + for (const key of Object.keys(texts)) { + const delta = texts[key] as { attributes?: unknown }[]; + if (!delta.some((item) => item.attributes !== undefined)) delete texts[key]; + } + return { json, texts }; +} + +function firstDifferenceKey(left: NormalizedValue, right: NormalizedValue): string { + const keys = new Set([...Object.keys(left.json), ...Object.keys(right.json)]); + for (const key of [...keys].sort()) { + if (JSON.stringify(left.json[key]) !== JSON.stringify(right.json[key])) return key; + } + if (JSON.stringify(left.texts) !== JSON.stringify(right.texts)) return "style"; + return "?"; +} + +function describeDifference(left: unknown, right: unknown): string { + const a = String(JSON.stringify(left)); + const b = String(JSON.stringify(right)); + if (a.length + b.length < 1600) return `\n left: ${a}\n right: ${b}`; + let start = 0; + while (start < a.length && a[start] === b[start]) start += 1; + const from = Math.max(0, start - 160); + return `\n left: …${a.slice(from, start + 400)}\n right: …${b.slice(from, start + 400)}`; +} + +export function assertSameValue( + check: string, + left: NormalizedValue, + right: NormalizedValue, + context = "", +): void { + if (JSON.stringify(left) === JSON.stringify(right)) return; + const key = firstDifferenceKey(left, right); + const leftPart = key === "style" ? left.texts : left.json[key]; + const rightPart = key === "style" ? right.texts : right.json[key]; + throw new FuzzMismatch( + check, + key, + `${check} at root "${key}"${context}${describeDifference(leftPart, rightPart)}`, + ); +} + +// --------------------------------------------------------------------------- +// Event mirror + +type MirrorValue = unknown; + +interface MirrorRef { + readonly __container: ContainerID; +} + +interface TreeMirror { + readonly kind: "Tree"; + readonly parent: Map; + readonly position: Map; + readonly children: Map; +} + +type MirrorState = + | { readonly kind: "Map"; readonly value: Map } + | { readonly kind: "List" | "MovableList"; value: MirrorValue[] } + | { readonly kind: "Text"; value: string } + | { readonly kind: "Counter"; value: number } + | TreeMirror; + +function containerTypeOf(id: ContainerID): ContainerType { + return id.slice(id.lastIndexOf(":") + 1) as ContainerType; +} + +function emptyMirror(id: ContainerID): MirrorState { + const kind = containerTypeOf(id); + switch (kind) { + case "Map": + return { kind, value: new Map() }; + case "List": + case "MovableList": + return { kind, value: [] }; + case "Text": + return { kind, value: "" }; + case "Counter": + return { kind, value: 0 }; + case "Tree": + return { kind, parent: new Map(), position: new Map(), children: new Map() }; + } +} + +function treeMetaId(node: TreeID): ContainerID { + return `cid:${node}:Map` as ContainerID; +} + +class MirrorError extends Error {} + +/** + * Applies event batches to a model keyed by container ID, with the same + * container-reset rules as the Rust fuzzer's `ContainerTracker`: attaching a + * container value starts that child from an empty state, while a movable-list + * move keeps the moved child's state. + */ +export class EventMirror { + readonly #states = new Map(); + + apply(batch: LoroEventBatch): void { + for (const event of batch.events) { + const state = this.#state(event.target); + const diff = event.diff; + switch (diff.type) { + case "map": + this.#applyMap(state, diff.updated); + break; + case "list": + this.#applyList(state, diff.diff as Delta[]); + break; + case "text": + this.#applyText(state, diff.diff); + break; + case "counter": + if (state.kind !== "Counter") + throw new MirrorError(`counter diff on ${state.kind}`); + state.value += diff.increment; + break; + case "tree": + this.#applyTree(state, diff.diff); + break; + } + } + } + + toJSON( + roots: readonly { name: string; type: ContainerType }[], + ): Record { + const output: Record = {}; + for (const root of roots) { + const id = `cid:root-${root.name}:${root.type}` as ContainerID; + if (this.#states.has(id)) output[root.name] = this.#json(id); + } + return output; + } + + #state(id: ContainerID): MirrorState { + let state = this.#states.get(id); + if (state === undefined) { + state = emptyMirror(id); + this.#states.set(id, state); + } + return state; + } + + #attach(value: unknown, moved: Set): MirrorValue { + if (!isContainerLike(value)) return value; + const id = value.id; + if (!moved.has(id)) this.#states.set(id, emptyMirror(id)); + return { __container: id } satisfies MirrorRef; + } + + #applyMap(state: MirrorState, updated: Readonly>): void { + if (state.kind !== "Map") throw new MirrorError(`map diff on ${state.kind}`); + for (const [key, value] of Object.entries(updated)) { + if (value === undefined) state.value.delete(key); + else state.value.set(key, this.#attach(value, new Set())); + } + } + + #applyList(state: MirrorState, delta: readonly Delta[]): void { + if (state.kind !== "List" && state.kind !== "MovableList") { + throw new MirrorError(`list diff on ${state.kind}`); + } + const removed = new Set(); + if (state.kind === "MovableList") { + let index = 0; + for (const item of delta) { + if ("retain" in item) index += item.retain; + else if ("delete" in item) { + for (const value of state.value.slice(index, index + item.delete)) { + if (value !== null && typeof value === "object" && "__container" in value) { + removed.add((value as MirrorRef).__container); + } + } + index += item.delete; + } + } + } + const next: MirrorValue[] = []; + let index = 0; + for (const item of delta) { + if ("retain" in item) { + if (index + item.retain > state.value.length) { + throw new MirrorError( + `list retain ${item.retain} beyond ${state.value.length}`, + ); + } + next.push(...state.value.slice(index, index + item.retain)); + index += item.retain; + } else if ("delete" in item) { + if (index + item.delete > state.value.length) { + throw new MirrorError( + `list delete ${item.delete} beyond ${state.value.length}`, + ); + } + index += item.delete; + } else { + for (const value of item.insert) next.push(this.#attach(value, removed)); + } + } + next.push(...state.value.slice(index)); + state.value = next; + } + + #applyText(state: MirrorState, delta: readonly Delta[]): void { + if (state.kind !== "Text") throw new MirrorError(`text diff on ${state.kind}`); + let output = ""; + const chars = Array.from(state.value); + // Event text positions are UTF-16 code units in the JS API. + let utf16 = 0; + let charIndex = 0; + const takeUtf16 = (count: number): string => { + let taken = ""; + const target = utf16 + count; + while (utf16 < target) { + const char = chars[charIndex]; + if (char === undefined) + throw new MirrorError(`text span beyond ${state.value.length}`); + taken += char; + utf16 += char.length; + charIndex += 1; + } + if (utf16 !== target) throw new MirrorError("text span splits a surrogate pair"); + return taken; + }; + for (const item of delta) { + if ("retain" in item) { + output += takeUtf16(item.retain); + } else if ("delete" in item) { + takeUtf16(item.delete); + } else { + output += item.insert; + } + } + output += chars.slice(charIndex).join(""); + state.value = output; + } + + #applyTree(state: MirrorState, diff: readonly TreeDiffItem[]): void { + if (state.kind !== "Tree") throw new MirrorError(`tree diff on ${state.kind}`); + const childrenOf = (parent: TreeID | undefined): TreeID[] => { + const key = parent ?? ""; + let list = state.children.get(key); + if (list === undefined) { + list = []; + state.children.set(key, list); + } + return list; + }; + const insert = (target: TreeID, parent: TreeID | undefined, index: number): void => { + if (parent !== undefined && !state.parent.has(parent)) { + throw new MirrorError(`tree parent ${parent} of ${target} does not exist`); + } + const siblings = childrenOf(parent); + if (index > siblings.length) { + throw new MirrorError( + `tree index ${index} beyond ${siblings.length} for ${target}`, + ); + } + siblings.splice(index, 0, target); + state.parent.set(target, parent); + }; + const detach = (target: TreeID): void => { + if (!state.parent.has(target)) + throw new MirrorError(`tree node ${target} is absent`); + const siblings = childrenOf(state.parent.get(target)); + const index = siblings.indexOf(target); + if (index < 0) throw new MirrorError(`tree node ${target} is not under its parent`); + siblings.splice(index, 1); + }; + for (const item of diff) { + if (item.action === "create") { + if (state.parent.has(item.target)) { + throw new MirrorError(`tree node ${item.target} already exists`); + } + insert(item.target, item.parent, item.index); + state.position.set(item.target, item.fractionalIndex); + this.#states.set(treeMetaId(item.target), emptyMirror(treeMetaId(item.target))); + } else if (item.action === "move") { + detach(item.target); + insert(item.target, item.parent, item.index); + state.position.set(item.target, item.fractionalIndex); + } else { + detach(item.target); + const stack = [item.target]; + while (stack.length > 0) { + const node = stack.pop()!; + stack.push(...(state.children.get(node) ?? [])); + state.children.delete(node); + state.parent.delete(node); + state.position.delete(node); + } + } + } + } + + #json(id: ContainerID): unknown { + const state = this.#states.get(id) ?? emptyMirror(id); + const value = (item: MirrorValue): unknown => + item !== null && typeof item === "object" && "__container" in item + ? this.#json((item as MirrorRef).__container) + : item; + switch (state.kind) { + case "Map": { + const output: Record = {}; + for (const [key, item] of state.value) output[key] = value(item); + return output; + } + case "List": + case "MovableList": + return state.value.map(value); + case "Text": + case "Counter": + return state.value; + case "Tree": { + const nodes = (parent: TreeID | undefined): unknown[] => + (state.children.get(parent ?? "") ?? []).map((node, index) => ({ + id: node, + parent: parent ?? null, + index, + fractional_index: state.position.get(node), + meta: this.#json(treeMetaId(node)), + children: nodes(node), + })); + return nodes(undefined); + } + } + } +} + +// --------------------------------------------------------------------------- +// Scenario execution + +interface Mirrored { + readonly doc: D; + readonly mirror: EventMirror; + mirrorError: unknown; +} + +interface Peer extends Mirrored { + readonly name: string; + readonly rust: Mirrored | undefined; +} + +/** A local edit resolved against the acting document. */ +interface ConcreteOp { + readonly target: ContainerID; + readonly method: string; + readonly args: readonly unknown[]; +} + +interface NewContainer { + readonly __new: ContainerType; +} + +function isNewContainer(value: unknown): value is NewContainer { + return value !== null && typeof value === "object" && "__new" in value; +} + +function enabledRoots(features: FuzzFeatures): typeof ROOTS { + return ROOTS.filter((root) => features[root.feature]); +} + +function scalar(seed: number): unknown { + switch (seed % 5) { + case 0: + return seed % 97; + case 1: + return `s${seed % 31}`; + case 2: + return (seed & 8) !== 0; + case 3: + return null; + default: + return { n: seed % 7 }; + } +} + +const TEXT_PIECES = ["a", "bc", "XYZ", "δΈ­", "πŸ˜€", "eπŸ˜€f", "\n", "hello"]; + +function textPiece(seed: number): string { + return TEXT_PIECES[seed % TEXT_PIECES.length]!; +} + +function childContainerTypes(features: FuzzFeatures): ContainerType[] { + const types: ContainerType[] = []; + if (features.map) types.push("Map", "Map"); + if (features.list) types.push("List"); + if (features.text) types.push("Text"); + if (features.counter) types.push("Counter"); + if (features.tree) types.push("Tree"); + if (features.movableList) types.push("MovableList"); + return types; +} + +function newDetached(type: ContainerType): Container { + switch (type) { + case "Map": + return new LoroMap(); + case "List": + return new LoroList(); + case "Text": + return new LoroText(); + case "Counter": + return new LoroCounter(); + case "Tree": + return new LoroTree(); + case "MovableList": + return new LoroMovableList(); + } +} + +function frontiersLiteral(frontiers: Frontiers): string { + return JSON.stringify(frontiers.map((id) => ({ peer: id.peer, counter: id.counter }))); +} + +const ROOT_GETTERS: Record = { + Map: "getMap", + List: "getList", + Text: "getText", + Tree: "getTree", + Counter: "getCounter", + MovableList: "getMovableList", +}; + +function containerExpression(doc: string, id: ContainerID): string { + const root = /^cid:root-(.*):(\w+)$/.exec(id); + if (root !== null) { + return `${doc}.${ROOT_GETTERS[root[2] as ContainerType]}(${JSON.stringify(root[1])})`; + } + return `(${doc}.getContainerById(${JSON.stringify(id)}) as any)`; +} + +function renderArgument(value: unknown): string { + if (isNewContainer(value)) return `new Loro${value.__new}()`; + if (value === undefined) return "undefined"; + return JSON.stringify(value); +} + +function renderOp(doc: string, op: ConcreteOp): string { + return `${containerExpression(doc, op.target)}.${op.method}(${op.args.map(renderArgument).join(", ")});`; +} + +function executeOp( + doc: { getContainerById(id: ContainerID): unknown }, + op: ConcreteOp, + create: (type: ContainerType) => unknown, +): string | undefined { + try { + const container = doc.getContainerById(op.target) as Record< + string, + (...args: unknown[]) => unknown + >; + const args = op.args.map((arg) => (isNewContainer(arg) ? create(arg.__new) : arg)); + container[op.method]!(...args); + return undefined; + } catch (error) { + return String((error as Error)?.message ?? error); + } +} + +function sortedChanges(doc: LoroDoc): [string, { counter: number; length: number }[]][] { + return [...doc.getAllChanges().entries()].sort(([a], [b]) => (a < b ? -1 : 1)); +} + +export interface RunOptions { + readonly checks?: FuzzChecks; + readonly rust?: RustOracle | undefined; + /** Run every action on Rust peers too and compare after each action. */ + readonly lockstep?: boolean; + /** Upper bound on versions checked out after the scenario. */ + readonly maxCheckoutVersions?: number; + readonly onTrace?: ((line: string) => void) | undefined; + /** Check only the Rust event mirrors and values; validates the harness itself. */ + readonly validateHarness?: boolean; +} + +class Runner { + readonly peers: Peer[] = []; + readonly trace: string[] = []; + readonly roots: typeof ROOTS; + readonly checks: FuzzChecks; + readonly rust: RustOracle | undefined; + readonly lockstep: boolean; + rustReplay: Mirrored | undefined; + + constructor( + readonly scenario: Scenario, + readonly options: RunOptions, + ) { + this.roots = enabledRoots(scenario.features); + this.checks = options.checks ?? ALL_CHECKS; + this.rust = options.rust; + this.lockstep = options.lockstep === true && options.rust !== undefined; + for (let index = 0; index < scenario.peers; index += 1) { + const doc = this.newDoc(index + 1); + let rust: Mirrored | undefined; + if (this.lockstep) { + const rustDoc = this.newRustDoc()!; + rustDoc.setPeerId(BigInt(index + 1)); + rust = { doc: rustDoc, mirror: new EventMirror(), mirrorError: undefined }; + if (this.checks.events) this.subscribe(rust); + } + const peer: Peer = { + name: `p${index + 1}`, + doc, + mirror: new EventMirror(), + mirrorError: undefined, + rust, + }; + if (this.checks.events) this.subscribe(peer); + this.peers.push(peer); + } + } + + get withStyles(): boolean { + return this.scenario.features.textStyle; + } + + log(line: string): void { + this.trace.push(line); + this.options.onTrace?.(line); + } + + newDoc(peer?: number): LoroDoc { + const doc = new LoroDoc(); + if (peer !== undefined) doc.setPeerId(peer); + if (this.scenario.features.textStyle) doc.configTextStyle(TEXT_STYLES); + return doc; + } + + newRustDoc(): RustDoc | undefined { + const doc = this.rust?.newDoc(); + if (doc !== undefined && this.scenario.features.textStyle) { + doc.configTextStyle(TEXT_STYLES); + } + return doc; + } + + subscribe( + target: Mirrored<{ subscribe(listener: (batch: LoroEventBatch) => void): unknown }>, + ): void { + target.doc.subscribe((batch) => { + if (target.mirrorError !== undefined) return; + try { + target.mirror.apply(batch); + } catch (error) { + target.mirrorError = error; + } + }); + } + + value(doc: LoroDoc): NormalizedValue { + return docValue(doc, this.withStyles); + } + + rustValue(doc: RustDoc): NormalizedValue { + return docValue(doc, this.withStyles); + } + + checkMirror( + target: Mirrored<{ toJSON(): unknown }>, + label: string, + check = "event", + ): void { + if (!this.checks.events) return; + if (this.options.validateHarness === true && !check.startsWith("rust")) return; + if (target.mirrorError !== undefined) { + const message = String((target.mirrorError as Error).message ?? target.mirrorError); + throw new FuzzMismatch( + `${check}-invalid`, + message.replace(/\d+(@\d+)?/g, "N"), + `${label}: ${message}`, + ); + } + const mirrored = normalizeJson(sortValue(target.mirror.toJSON(this.roots))); + const actual = normalizeJson(target.doc.toJSON()); + assertSameValue( + `${check}-mirror`, + { json: mirrored }, + { json: actual }, + ` (${label})`, + ); + } + + /** Compares a peer with its Rust twin after an action in lockstep mode. */ + checkPeer(peer: Peer, label: string): void { + this.checkMirror(peer, label); + if (peer.rust === undefined) return; + this.rust!.flushEvents(); + assertSameValue( + "lockstep", + this.value(peer.doc), + this.rustValue(peer.rust.doc), + ` (${peer.name} vs Rust after ${label})`, + ); + this.checkMirror(peer.rust, `Rust ${label}`, "rust-event"); + } + + // -- local edits ----------------------------------------------------------- + + containers(doc: LoroDoc): ContainerLike[] { + const output: ContainerLike[] = []; + const queue: ContainerLike[] = []; + for (const root of this.roots) { + queue.push(doc.getContainerById(`cid:root-${root.name}:${root.type}`) as never); + } + while (queue.length > 0 && output.length < 32) { + const container = queue.shift()!; + output.push(container); + const kind = container.kind(); + if (kind === "Map") { + const map = container as unknown as LoroMap; + for (const key of map.keys().sort()) { + const child = map.get(key); + if (isContainerLike(child)) queue.push(child); + } + } else if (kind === "List" || kind === "MovableList") { + const list = container as unknown as LoroList; + for (let index = 0; index < list.length; index += 1) { + const child = list.get(index); + if (isContainerLike(child)) queue.push(child); + } + } else if (kind === "Tree" && this.scenario.features.nested) { + const tree = container as unknown as LoroTree; + const nodes = tree.getNodes().sort((a, b) => (a.id < b.id ? -1 : 1)); + for (const node of nodes) queue.push(node.data as unknown as ContainerLike); + } + } + return output; + } + + /** Resolves an abstract edit against `doc`; undefined when it is a no-op. */ + concreteOp( + doc: LoroDoc, + action: Extract, + ): ConcreteOp | undefined { + const containers = this.containers(doc); + const container = containers[action.target % containers.length]!; + const target = container.id; + const features = this.scenario.features; + const types = childContainerTypes(features); + const childType: NewContainer = { __new: types[action.b % types.length]! }; + const { a, b } = action; + switch (container.kind()) { + case "Map": { + const key = `k${a % 5}`; + const choice = action.op % 5; + if (choice === 0) return { target, method: "delete", args: [key] }; + if (choice === 1 && features.nested) { + return { target, method: "setContainer", args: [key, childType] }; + } + return { target, method: "set", args: [key, scalar(b)] }; + } + case "List": + case "MovableList": { + const list = container as unknown as LoroMovableList; + const movable = container.kind() === "MovableList"; + const length = list.length; + const choice = action.op % (movable ? 7 : 5); + if (length > 0 && (choice === 0 || choice === 1)) { + const pos = a % length; + const len = 1 + (b % Math.min(3, length - pos)); + return { target, method: "delete", args: [pos, len] }; + } + if (choice === 2 && features.nested) { + return { + target, + method: "insertContainer", + args: [a % (length + 1), childType], + }; + } + if (movable && length > 0 && choice === 5) { + return { target, method: "move", args: [a % length, b % length] }; + } + if (movable && length > 0 && choice === 6) { + return { target, method: "set", args: [a % length, scalar(b)] }; + } + return { target, method: "insert", args: [a % (length + 1), scalar(b)] }; + } + case "Text": { + const text = container as unknown as LoroText; + const length = text.length; + const choice = action.op % (features.textStyle ? 6 : 4); + if (length > 0 && choice === 0) { + const pos = safeTextPos(text, a % length); + const end = safeTextPos(text, Math.min(length, pos + 1 + (b % 4))); + return end > pos + ? { target, method: "delete", args: [pos, end - pos] } + : undefined; + } + if (length > 0 && choice >= 4) { + const start = safeTextPos(text, a % length); + const end = safeTextPos(text, Math.min(length, start + 1 + (b % 5))); + if (end <= start) return undefined; + const keys = Object.keys(TEXT_STYLES); + const key = keys[(b >>> 8) % keys.length]!; + if (choice === 4 || (b & 1) === 0) { + const value = (b >>> 12) % 3 === 0 ? null : (b >>> 12) % 5; + return { target, method: "mark", args: [{ start, end }, key, value] }; + } + return { target, method: "unmark", args: [{ start, end }, key] }; + } + return { + target, + method: "insert", + args: [safeTextPos(text, a % (length + 1)), textPiece(b)], + }; + } + case "Counter": + return { target, method: "increment", args: [(a % 9) - 4 || 5] }; + case "Tree": { + const tree = container as unknown as LoroTree; + const nodes = tree.getNodes().sort((x, y) => (x.id < y.id ? -1 : 1)); + const choice = action.op % 6; + const childCount = (parent: TreeID | undefined): number => + parent === undefined + ? tree.roots().length + : tree.getNodeByID(parent)!.children().length; + if (nodes.length > 0 && choice === 0) { + return { target, method: "delete", args: [nodes[a % nodes.length]!.id] }; + } + const parentIndex = b % (nodes.length + 1); + const parent = parentIndex === nodes.length ? undefined : nodes[parentIndex]!.id; + if (nodes.length > 0 && (choice === 1 || choice === 2)) { + const node = nodes[a % nodes.length]!.id; + const siblings = childCount(parent); + const current = tree.getNodeByID(node)!; + const sameParent = current.parent()?.id === parent; + const position = (b >>> 8) % (sameParent ? siblings : siblings + 1); + return { target, method: "move", args: [node, parent, position] }; + } + return { + target, + method: "createNode", + args: [parent, (a >>> 4) % (childCount(parent) + 1)], + }; + } + } + return undefined; + } + + applyOp(index: number, action: Extract): void { + const peer = this.peers[action.peer]!; + if (peer.doc.isDetached()) { + this.log(`${peer.name}.attach();`); + peer.doc.attach(); + peer.rust?.doc.attach(); + this.checkPeer(peer, `attach before action ${index}`); + } + const op = this.concreteOp(peer.doc, action); + if (op !== undefined) { + this.log(renderOp(peer.name, op)); + const error = executeOp(peer.doc, op, newDetached); + if (error !== undefined) this.log(`// throws: ${error}`); + if (peer.rust !== undefined) { + const rustError = executeOp(peer.rust.doc, op, (type) => + this.rust!.newContainer(type), + ); + if ((error === undefined) !== (rustError === undefined)) { + throw new FuzzMismatch( + "lockstep-throws", + `${op.method}`, + `${renderOp(peer.name, op)} ${error === undefined ? "succeeds" : `throws "${error}"`} in loro-js but ${rustError === undefined ? "succeeds" : `throws "${rustError}"`} in Rust`, + ); + } + } + } + if (action.commit) { + this.log(`${peer.name}.commit();`); + peer.doc.commit(); + peer.rust?.doc.commit(); + this.checkPeer(peer, `commit after action ${index}`); + } + } + + // -- synchronization ----------------------------------------------------- + + /** ID spans of every change in `source` that is not covered by `known`. */ + missingSpans( + source: LoroDoc, + known: ReturnType, + ): { id: OpId; len: number }[] { + const spans: { id: OpId; len: number }[] = []; + for (const [peer, list] of sortedChanges(source)) { + const seen = known.get(peer) ?? 0; + for (const change of list) { + const start = Math.max(change.counter, seen); + const end = change.counter + change.length; + if (start < end) { + spans.push({ + id: { peer: peer as OpId["peer"], counter: start }, + len: end - start, + }); + } + } + } + return spans; + } + + sync(index: number, action: Extract): void { + const from = this.peers[action.from]!; + const to = this.peers[action.to]!; + for (const peer of [from, to]) { + peer.doc.commit(); + peer.rust?.doc.commit(); + } + const rustFrom = from.rust?.doc; + const rustTo = to.rust?.doc; + switch (action.mode) { + case "update": + this.log( + `${to.name}.import(${from.name}.export({ mode: "update", from: ${to.name}.oplogVersion() }));`, + ); + to.doc.import(from.doc.export({ mode: "update", from: to.doc.oplogVersion() })); + rustTo?.import(rustFrom!.export({ mode: "update", from: rustTo.oplogVersion() })); + break; + case "full": + this.log(`${to.name}.import(${from.name}.export({ mode: "update" }));`); + to.doc.import(from.doc.export({ mode: "update" })); + rustTo?.import(rustFrom!.export({ mode: "update" })); + break; + case "snapshot": + this.log(`${to.name}.import(${from.name}.export({ mode: "snapshot" }));`); + to.doc.import(from.doc.export({ mode: "snapshot" })); + rustTo?.import(rustFrom!.export({ mode: "snapshot" })); + break; + case "shuffled": + case "batch": { + const spans = this.missingSpans(from.doc, to.doc.oplogVersion()); + shuffle(spans, new Random(action.r)); + const exportSpan = (span: { id: OpId; len: number }): string => + `${from.name}.export({ mode: "updates-in-range", spans: [${JSON.stringify(span)}] })`; + if (action.mode === "batch") { + this.log(`${to.name}.importBatch([${spans.map(exportSpan).join(", ")}]);`); + } else { + for (const span of spans) this.log(`${to.name}.import(${exportSpan(span)});`); + } + const blobs = (doc: { export(mode: unknown): Uint8Array }): Uint8Array[] => + spans.map((span) => doc.export({ mode: "updates-in-range", spans: [span] })); + if (action.mode === "batch") { + to.doc.importBatch(blobs(from.doc)); + rustTo?.importBatch(blobs(rustFrom!)); + } else { + for (const blob of blobs(from.doc)) to.doc.import(blob); + if (rustTo !== undefined) + for (const blob of blobs(rustFrom!)) rustTo.import(blob); + } + break; + } + } + this.checkPeer(to, `import in action ${index}`); + } + + // -- versions ------------------------------------------------------------ + + /** Every operation ID in `doc`'s history, sorted deterministically. */ + allOpIds(doc: LoroDoc): OpId[] { + const ids: OpId[] = []; + for (const [peer, list] of sortedChanges(doc)) { + for (const change of list) { + for (let offset = 0; offset < change.length; offset += 1) { + ids.push({ peer: peer as OpId["peer"], counter: change.counter + offset }); + } + } + } + return ids; + } + + pickFrontiers(doc: LoroDoc, r: number, r2: number): Frontiers | undefined { + const ids = this.allOpIds(doc); + if (ids.length === 0) return undefined; + const first = ids[r % ids.length]!; + if (r2 % 4 !== 0) return [first]; + const second = ids[r2 % ids.length]!; + const version = doc.frontiersToVV([first]); + version.merge(doc.frontiersToVV([second])); + return doc.vvToFrontiers(version); + } + + /** A fresh document containing exactly the changes included in `frontiers`. */ + replayAt(source: LoroDoc, frontiers: Frontiers): LoroDoc { + const version = source.frontiersToVV(frontiers); + const spans = [...version.toJSON().entries()] + .filter(([, end]) => end > 0) + .map(([peer, end]) => ({ id: { peer, counter: 0 }, len: end })); + const doc = this.newDoc(); + if (spans.length > 0) doc.import(source.export({ mode: "updates-in-range", spans })); + return doc; + } + + checkoutAction(index: number, action: Extract): void { + const peer = this.peers[action.peer]!; + peer.doc.commit(); + peer.rust?.doc.commit(); + const frontiers = this.pickFrontiers(peer.doc, action.r, action.r2); + if (frontiers === undefined) return; + this.log(`${peer.name}.checkout(${frontiersLiteral(frontiers)});`); + peer.doc.checkout(frontiers); + peer.rust?.doc.checkout(frontiers); + this.checkPeer(peer, `checkout in action ${index}`); + if (this.checks.checkout) { + assertSameValue( + "checkout-vs-replay", + this.value(peer.doc), + this.value(this.replayAt(peer.doc, frontiers)), + ` (${peer.name} at ${frontiersLiteral(frontiers)})`, + ); + } + if (!action.stay) { + this.log(`${peer.name}.checkoutToLatest();`); + peer.doc.checkoutToLatest(); + peer.rust?.doc.checkoutToLatest(); + this.checkPeer(peer, `checkoutToLatest in action ${index}`); + } + } + + // -- final checks ---------------------------------------------------------- + + finish(): void { + for (const peer of this.peers) { + peer.doc.commit(); + peer.rust?.doc.commit(); + if (peer.doc.isDetached()) { + this.log(`${peer.name}.checkoutToLatest();`); + peer.doc.checkoutToLatest(); + peer.rust?.doc.checkoutToLatest(); + this.checkPeer(peer, "final attach"); + } + } + this.log("// final: every peer imports every other peer's updates"); + for (let round = 0; round < 2; round += 1) { + for (const from of this.peers) { + for (const to of this.peers) { + if (from === to) continue; + to.doc.import(from.doc.export({ mode: "update", from: to.doc.oplogVersion() })); + to.rust?.doc.import( + from.rust!.doc.export({ mode: "update", from: to.rust.doc.oplogVersion() }), + ); + } + } + } + for (const peer of this.peers) this.checkPeer(peer, `final sync of ${peer.name}`); + const base = this.peers[0]!.doc; + const baseValue = this.value(base); + for (const peer of this.peers.slice(1)) { + assertSameValue( + "peer-divergence", + this.value(peer.doc), + baseValue, + ` (${peer.name} vs p1)`, + ); + } + + const updates = base.export({ mode: "update" }); + const replay: Mirrored = { + doc: this.newDoc(), + mirror: new EventMirror(), + mirrorError: undefined, + }; + if (this.checks.events) this.subscribe(replay); + replay.doc.import(updates); + this.checkMirror(replay, "replay import"); + const replayValue = this.value(replay.doc); + const rust = this.newRustDoc(); + if (rust !== undefined) { + if (this.checks.events) { + this.rustReplay = { + doc: rust, + mirror: new EventMirror(), + mirrorError: undefined, + }; + this.subscribe(this.rustReplay); + } + rust.import(updates); + this.rust!.flushEvents(); + assertSameValue( + "rust-replay", + replayValue, + this.rustValue(rust), + " (loro-js replay vs Rust replay)", + ); + } + assertSameValue( + "replay-vs-peers", + baseValue, + replayValue, + " (p1 vs one-shot replay)", + ); + + const shuffled = this.newDoc(); + const spans = this.missingSpans(base, shuffled.oplogVersion()); + shuffle(spans, new Random(this.scenario.seed ^ 0x5bd1_e995)); + for (const span of spans) { + shuffled.import(base.export({ mode: "updates-in-range", spans: [span] })); + } + assertSameValue( + "shuffled-replay", + this.value(shuffled), + replayValue, + " (per-change shuffled replay vs one-shot replay)", + ); + + const versions = this.checkoutVersions(base); + if (this.checks.checkout) { + this.checkoutEverywhere("replay", replay, versions, rust); + this.checkoutEverywhere("p1", this.peers[0]!, versions, undefined); + } + if (this.checks.snapshot) { + // Events are not mirrored here: the mirror would need the imported state as its base. + const snapshot: Mirrored = { + doc: this.newDoc(), + mirror: new EventMirror(), + mirrorError: undefined, + }; + snapshot.doc.import(base.export({ mode: "snapshot" })); + assertSameValue( + "snapshot-latest", + this.value(snapshot.doc), + replayValue, + " (snapshot import)", + ); + if (this.checks.checkout) { + this.checkoutEverywhere("snapshot", snapshot, versions, undefined, false); + } + } + if (this.checks.shallow) this.checkShallow(base, versions); + } + + checkoutVersions(doc: LoroDoc): Frontiers[] { + const ids = this.allOpIds(doc); + const limit = this.options.maxCheckoutVersions ?? 48; + const random = new Random(this.scenario.seed ^ 0x2545_f491); + const versions: Frontiers[] = []; + if (ids.length <= limit) { + for (const id of ids) versions.push([id]); + } else { + for (let index = 0; index < limit; index += 1) { + versions.push([ids[random.below(ids.length)]!]); + } + } + for (let index = 0; index < Math.min(8, ids.length); index += 1) { + const frontiers = this.pickFrontiers(doc, random.next(), 0); + if (frontiers !== undefined) versions.push(frontiers); + } + return versions; + } + + checkoutEverywhere( + label: string, + target: Mirrored, + versions: readonly Frontiers[], + rust: RustDoc | undefined, + mirrored = true, + ): void { + const base = this.peers[0]!.doc; + const order = [...versions]; + shuffle(order, new Random(this.scenario.seed ^ 0x68e3_1da4)); + for (const frontiers of order) { + const context = ` (${label}.checkout(${frontiersLiteral(frontiers)}))`; + this.log(`${label}.checkout(${frontiersLiteral(frontiers)});`); + try { + target.doc.checkout(frontiers); + } catch (error) { + const message = String((error as Error).message); + throw new FuzzMismatch( + `checkout-throws:${label}`, + normalizeMessage(message), + `${label}.checkout(${frontiersLiteral(frontiers)}) threw: ${message}`, + ); + } + if (mirrored) + this.checkMirror(target, `${label} checkout ${frontiersLiteral(frontiers)}`); + const expected = this.value(this.replayAt(base, frontiers)); + assertSameValue(`checkout:${label}`, this.value(target.doc), expected, context); + if (rust !== undefined) { + rust.checkout(frontiers); + assertSameValue("rust-checkout", expected, this.rustValue(rust), context); + if (this.rustReplay?.doc === rust) { + this.rust!.flushEvents(); + this.checkMirror(this.rustReplay, `Rust ${label} checkout`, "rust-event"); + } + } + } + target.doc.checkoutToLatest(); + if (mirrored) this.checkMirror(target, `${label} checkoutToLatest`); + rust?.checkoutToLatest(); + } + + checkShallow(base: LoroDoc, versions: readonly Frontiers[]): void { + const random = new Random(this.scenario.seed ^ 0x1b87_3593); + if (this.allOpIds(base).length === 0) return; + const latest = this.value(base); + const rustFull = this.newRustDoc(); + rustFull?.import(base.export({ mode: "update" })); + for (let attempt = 0; attempt < 3; attempt += 1) { + const root = this.pickFrontiers(base, random.next(), random.next())!; + const context = ` (shallow at ${frontiersLiteral(root)})`; + this.log( + `const shallow = LoroDoc.fromSnapshot(p1.export({ mode: "shallow-snapshot", frontiers: ${frontiersLiteral(root)} }));`, + ); + const blob = base.export({ mode: "shallow-snapshot", frontiers: root }); + const shallow = this.newDoc(); + shallow.import(blob); + assertSameValue("shallow-latest", this.value(shallow), latest, context); + if (rustFull !== undefined) { + const rustShallow = this.newRustDoc()!; + rustShallow.import(blob); + const rustRoot = this.newRustDoc()!; + rustRoot.import(rustFull.export({ mode: "shallow-snapshot", frontiers: root })); + const roots = [shallow, rustRoot].map((doc) => + frontiersLiteral( + [...doc.shallowSinceFrontiers()].sort((a, b) => + a.peer === b.peer ? a.counter - b.counter : a.peer < b.peer ? -1 : 1, + ), + ), + ); + if (roots[0] !== roots[1]) { + throw new FuzzMismatch( + "shallow-root", + "root", + `shallow root${context}: loro-js ${roots[0]}, Rust ${roots[1]}`, + ); + } + assertSameValue( + "rust-imports-ts-shallow", + this.rustValue(rustShallow), + latest, + context, + ); + const fromRust = this.newDoc(); + fromRust.import(rustFull.export({ mode: "shallow-snapshot", frontiers: root })); + assertSameValue("ts-imports-rust-shallow", this.value(fromRust), latest, context); + } + if (!this.checks.checkout) continue; + const since = base.frontiersToVV(shallow.shallowSinceFrontiers()); + for (const frontiers of versions) { + const order = since.compare(base.frontiersToVV(frontiers)); + if (order !== -1 && order !== 0) continue; + this.log(`shallow.checkout(${frontiersLiteral(frontiers)});`); + let actual: NormalizedValue; + try { + shallow.checkout(frontiers); + actual = this.value(shallow); + } catch (error) { + const message = String((error as Error).message); + throw new FuzzMismatch( + "shallow-checkout-throws", + normalizeMessage(message), + `shallow.checkout(${frontiersLiteral(frontiers)}) threw${context}: ${message}`, + ); + } + assertSameValue( + "shallow-checkout", + actual, + this.value(this.replayAt(base, frontiers)), + `${context} checkout ${frontiersLiteral(frontiers)}`, + ); + } + shallow.checkoutToLatest(); + assertSameValue( + "shallow-latest", + this.value(shallow), + latest, + `${context} after checkouts`, + ); + } + } +} + +function safeTextPos(text: LoroText, pos: number): number { + // Never split a surrogate pair. + const value = text.toString(); + let clamped = Math.min(pos, value.length); + const code = value.charCodeAt(clamped - 1); + if (clamped > 0 && code >= 0xd800 && code <= 0xdbff) clamped -= 1; + return clamped; +} + +function shuffle(items: T[], random: Random): void { + for (let index = items.length - 1; index > 0; index -= 1) { + const other = random.below(index + 1); + [items[index], items[other]] = [items[other]!, items[index]!]; + } +} + +function normalizeMessage(message: string): string { + return message + .split("\n")[0]! + .replace(/cid:[^\s)"]+/g, "CID") + .replace(/\d+/g, "N") + .slice(0, 120); +} + +/** Runs a scenario; returns undefined on success or the first failure. */ +export function runScenario( + scenario: Scenario, + options: RunOptions = {}, +): FuzzFailure | undefined { + const runner = new Runner(scenario, options); + let phase = "setup"; + try { + for (const [index, action] of scenario.actions.entries()) { + phase = `${action.kind}#${index}`; + if (action.kind === "op") runner.applyOp(index, action); + else if (action.kind === "sync") runner.sync(index, action); + else runner.checkoutAction(index, action); + } + phase = "final"; + runner.finish(); + return undefined; + } catch (error) { + if (error instanceof FuzzMismatch) { + return { + check: error.check, + signature: `${error.check}:${error.key}`, + message: error.message, + trace: runner.trace, + }; + } + const message = (error as Error)?.stack ?? String(error); + const kind = phase.replace(/#\d+$/, ""); + return { + check: `throws:${kind}`, + signature: `throws:${kind}:${normalizeMessage((error as Error)?.message ?? String(error))}`, + message: `${phase} threw: ${message}`, + trace: runner.trace, + }; + } +} + +/** + * Delta-debugging over the action list. Keeps a reduction only when the + * failure keeps the same signature. + */ +export function minimizeScenario( + scenario: Scenario, + signature: string, + options: RunOptions = {}, + budget = 4_000, +): Scenario { + let current = scenario; + let runs = 0; + const fails = (candidate: Scenario): boolean => { + runs += 1; + return ( + runScenario(candidate, { ...options, onTrace: undefined })?.signature === signature + ); + }; + let chunk = Math.max(1, Math.floor(current.actions.length / 2)); + while (chunk >= 1 && runs < budget) { + let removed = false; + for (let start = 0; start < current.actions.length && runs < budget; ) { + const actions = [ + ...current.actions.slice(0, start), + ...current.actions.slice(start + chunk), + ]; + const candidate = { ...current, actions }; + if (actions.length < current.actions.length && fails(candidate)) { + current = candidate; + removed = true; + } else { + start += chunk; + } + } + if (!removed) chunk = Math.floor(chunk / 2); + } + // Fewer peers make the reproduction easier to read. + for (let peers = 2; peers < current.peers && runs < budget; peers += 1) { + const actions = current.actions.filter((action) => + action.kind === "sync" + ? action.from < peers && action.to < peers + : action.peer < peers, + ); + const candidate = { ...current, peers, actions }; + if (fails(candidate)) { + current = candidate; + break; + } + } + return current; +} + +export function describeScenario(scenario: Scenario): string { + return JSON.stringify({ + seed: scenario.seed, + peers: scenario.peers, + features: scenario.features, + actions: scenario.actions, + }); +} From 489088c0b10053fd767a8264287726b2ea44eb55 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:45:25 +0800 Subject: [PATCH 6/7] fix(loro.js): match Rust's local tree create and move positions (#1148) * fix(loro.js): match Rust's local tree create and move positions - Port generate_fi_at's rearrangement: when both neighbors share a fractional index, the right neighbor and following siblings with that index get new positions (generate_n_evenly) through extra move ops. loro.js threw "fractional index bounds are not ordered" instead. - Like mov_with_txn, a move to the node's current index, including an omitted index when it is already last, records nothing. - Like mov_after/mov_before, count the target index without the moved node when it shares the parent. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): stop the local tree cycle check at deleted nodes Rust's is_ancestor_of treats a deleted node's parent as the deleted root. loro.js kept walking a deleted node's old parent, so moving a node under its own deleted child threw. The walk is also bounded by the node count: a cycle left by concurrent moves made it loop forever. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): count siblings in O(1) when checking a same-parent tree move The no-op check listed every sibling to count them, so each move within a parent was linear in the sibling count. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .changeset/loro-js-tree-local-positions.md | 10 ++ loro-js/src/runtime/containers.ts | 60 +++++++-- loro-js/src/runtime/document.ts | 41 +++++- loro-js/src/runtime/fractional-index.ts | 30 +++++ loro-js/tests/tree-local-positions.test.ts | 146 +++++++++++++++++++++ 5 files changed, 274 insertions(+), 13 deletions(-) create mode 100644 .changeset/loro-js-tree-local-positions.md create mode 100644 loro-js/tests/tree-local-positions.test.ts diff --git a/.changeset/loro-js-tree-local-positions.md b/.changeset/loro-js-tree-local-positions.md new file mode 100644 index 000000000..594857ff2 --- /dev/null +++ b/.changeset/loro-js-tree-local-positions.md @@ -0,0 +1,10 @@ +--- +"loro.js": patch +--- + +Fix `loro.js` local tree moves and creates to match `loro-crdt`: + +- Creating or moving a node between two siblings that share a fractional index (for example two roots created concurrently at index 0) threw `fractional index bounds are not ordered`. It now gives the following siblings with that index new positions in the same transaction, as Rust does. +- Moving a node to the position it already has, including `move(node, parent)` without an index when it is already last, `node.move(parent)`, and `moveAfter`/`moveBefore` its current neighbor, no longer records an op. +- `moveAfter` and `moveBefore` within the same parent counted the moved node itself, so moving a node after a later sibling threw `tree index … is out of range` or landed one slot off. +- Moving a node under one of its own deleted descendants threw `cannot move a tree node below itself or its descendant`. A deleted node's parent is the deleted root, as in Rust, so the move is allowed (and the node becomes deleted). diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index d7e9c46df..2ffbdc5b2 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -1,4 +1,4 @@ -import { bytesToHex } from "../codec/bytes"; +import { bytesEqual, bytesToHex } from "../codec/bytes"; import { containerTypeFromHistoricalByte, containerTypeToHistoricalByte, @@ -7,7 +7,7 @@ import { PostcardReader, PostcardWriter } from "../codec/postcard"; import type { ContainerId as CodecContainerId, Id as CodecId } from "../codec/types"; import { formatContainerId, formatTreeId, parseContainerId } from "./ids"; import type { LoroDoc } from "./document"; -import { fractionalIndexBetween } from "./fractional-index"; +import { fractionalIndexBetween, fractionalIndexesBetween } from "./fractional-index"; import { OrderedIndex } from "./ordered-index"; import { SequenceIndex, SequenceSpan } from "./sequence-index"; import type { SequenceIdRun } from "./sequence-index"; @@ -3028,11 +3028,18 @@ export class LoroTree< ); } + /** + * Port of Rust's `generate_fi_at`: the position for a node placed at + * `index` among `parent`'s children, not counting `exclude`. When both + * neighbors share a position there is no index between them, so the right + * neighbor and every following sibling with that position get new + * positions in `rearranged`, in order. + */ _positionFor( parent: CodecId | undefined, index?: number, exclude?: CodecId, - ): Uint8Array { + ): { position: Uint8Array; rearranged: { id: CodecId; position: Uint8Array }[] } { const children = this._children.get(treeParentKey(parent)); const excluded = exclude === undefined ? undefined : this._nodes.get(formatTreeId(exclude)); @@ -3053,10 +3060,29 @@ export class LoroTree< ? siblingIndex + 1 : siblingIndex, ); - return fractionalIndexBetween( - position === 0 ? undefined : siblingAt(position - 1)!.position, - position === length ? undefined : siblingAt(position)!.position, - ); + const left = position === 0 ? undefined : siblingAt(position - 1)!.position; + const right = position === length ? undefined : siblingAt(position)!; + if (left === undefined || right === undefined || !bytesEqual(left, right.position)) { + return { position: fractionalIndexBetween(left, right?.position), rearranged: [] }; + } + const reset = [right]; + let nextRight: Uint8Array | undefined; + for (let sibling = position + 1; sibling < length; sibling += 1) { + const record = siblingAt(sibling)!; + if (!bytesEqual(record.position, left)) { + nextRight = record.position; + break; + } + reset.push(record); + } + const positions = fractionalIndexesBetween(left, nextRight, reset.length + 1); + return { + position: positions[0]!, + rearranged: reset.map((record, offset) => ({ + id: record.id, + position: positions[offset + 1]!, + })), + }; } _setRecord(record: TreeNodeRecord): void { @@ -3198,12 +3224,28 @@ export class LoroTreeNode = Record): void { + // Rust's mov_after: the index is counted without this node. const parent = target.parent(); - this.move(parent, target.index() + 1); + let index = target.index() + 1; + if (this.#hasParent(parent) && this.index() < index) index -= 1; + this.move(parent, index); } moveBefore(target: LoroTreeNode): void { - this.move(target.parent(), target.index()); + const parent = target.parent(); + let index = target.index(); + if (this.#hasParent(parent) && index >= 1 && this.index() < index) index -= 1; + this.move(parent, index); + } + + #hasParent(parent: LoroTreeNode | undefined): boolean { + const record = this.#record(); + return ( + !record.deleted && + (parent === undefined + ? record.parent === undefined + : record.parent !== undefined && formatTreeId(record.parent) === parent.id) + ); } parent(): LoroTreeNode | undefined { diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index c1fc58a78..ced47c814 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -4229,15 +4229,27 @@ export class LoroDoc = Record = Record `${id.counter}@${id.peer}`)); diff --git a/loro-js/src/runtime/fractional-index.ts b/loro-js/src/runtime/fractional-index.ts index 652ac8ecd..3066f8c11 100644 --- a/loro-js/src/runtime/fractional-index.ts +++ b/loro-js/src/runtime/fractional-index.ts @@ -25,6 +25,36 @@ export function fractionalIndexBetween( return terminated(between); } +/** + * `n` increasing indexes between the bounds, spread by bisection. Port of + * Rust's `FractionalIndex::generate_n_evenly`. + */ +export function fractionalIndexesBetween( + lower: Uint8Array | undefined, + upper: Uint8Array | undefined, + n: number, +): Uint8Array[] { + const output: Uint8Array[] = []; + const generate = ( + low: Uint8Array | undefined, + high: Uint8Array | undefined, + count: number, + ): void => { + if (count === 0) return; + const middle = fractionalIndexBetween(low, high); + if (count === 1) { + output.push(middle); + return; + } + const half = Math.floor(count / 2); + generate(low, middle, half); + output.push(middle); + generate(middle, high, count - half - 1); + }; + generate(lower, upper, n); + return output; +} + function terminated(bytes: number[]): Uint8Array { return Uint8Array.from([...bytes, TERMINATOR]); } diff --git a/loro-js/tests/tree-local-positions.test.ts b/loro-js/tests/tree-local-positions.test.ts new file mode 100644 index 000000000..e3164cf93 --- /dev/null +++ b/loro-js/tests/tree-local-positions.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc } from "../src/index"; +import type { LoroTree } from "../src/index"; + +// Expected values are what loro-crdt returns for the same calls. + +function shape(tree: LoroTree): string[] { + return tree.toJSON().map((node) => `${node.id}:${node.fractional_index}`); +} + +/** Two roots created concurrently at index 0 share the position 80. */ +function equalPositions(): LoroDoc { + const p1 = new LoroDoc(); + p1.setPeerId(1); + const p2 = new LoroDoc(); + p2.setPeerId(2); + p1.getTree("tree").createNode(); + p1.commit(); + p2.getTree("tree").createNode(); + p2.commit(); + p1.import(p2.export({ mode: "update" })); + expect(shape(p1.getTree("tree"))).toEqual(["0@1:80", "0@2:80"]); + return p1; +} + +describe("local tree positions", () => { + // Rust's generate_fi_at: there is no index between two equal positions, so + // the right neighbor is moved to a new position in the same transaction. + test("rearranges equal neighbors when creating a node between them", () => { + const doc = equalPositions(); + const tree = doc.getTree("tree"); + const before = doc.opCount(); + const node = tree.createNode(undefined, 1); + doc.commit(); + expect(node.id).toBe("1@1"); + expect(doc.opCount() - before).toBe(2); + expect(shape(tree)).toEqual(["0@1:80", "1@1:817F80", "0@2:8180"]); + + const replay = new LoroDoc(); + replay.import(doc.export({ mode: "update" })); + expect(shape(replay.getTree("tree"))).toEqual(shape(tree)); + }); + + test("rearranges equal neighbors when moving a node between them", () => { + const doc = equalPositions(); + const tree = doc.getTree("tree"); + const extra = tree.createNode(); + doc.commit(); + const before = doc.opCount(); + tree.move(extra.id, undefined, 1); + doc.commit(); + expect(doc.opCount() - before).toBe(2); + expect(shape(tree)).toEqual(["0@1:80", "1@1:817F80", "0@2:8180"]); + }); + + test("records nothing for a move to the current position", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const a = tree.createNode(); + const b = tree.createNode(); + const c = tree.createNode(); + doc.commit(); + const count = (edit: () => void): number => { + const before = doc.opCount(); + edit(); + doc.commit(); + return doc.opCount() - before; + }; + + expect(count(() => tree.move(a.id, undefined, 0))).toBe(0); + // Without an index the node goes last; it already is. + expect(count(() => tree.move(c.id, undefined))).toBe(0); + expect(count(() => c.move(undefined))).toBe(0); + expect(count(() => b.moveAfter(a))).toBe(0); + expect(count(() => a.moveBefore(b))).toBe(0); + expect(tree.toJSON().map((node) => node.id)).toEqual([a.id, b.id, c.id]); + }); + + // Rust's mov_after/mov_before count the index without the moved node. + test("moves after and before a sibling in the same parent", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const a = tree.createNode(); + const b = tree.createNode(); + const c = tree.createNode(); + doc.commit(); + + a.moveAfter(c); + doc.commit(); + expect(tree.toJSON().map((node) => node.id)).toEqual([b.id, c.id, a.id]); + a.moveBefore(b); + doc.commit(); + expect(tree.toJSON().map((node) => node.id)).toEqual([a.id, b.id, c.id]); + b.moveAfter(c); + doc.commit(); + expect(tree.toJSON().map((node) => node.id)).toEqual([a.id, c.id, b.id]); + }); + + test("moves within a parent in time independent of the sibling count", () => { + /** Best of three runs of 1,000 same-parent moves, half of them no-ops. */ + function movesMs(siblings: number): number { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const nodes = Array.from({ length: siblings }, () => tree.createNode()); + doc.commit(); + let best = Infinity; + for (let run = 0; run < 3; run += 1) { + const started = performance.now(); + for (let step = 0; step < 500; step += 1) { + const node = nodes[(step * 7_919) % siblings]!; + tree.move(node.id, undefined, node.index()!); + tree.move(node.id, undefined, 0); + } + doc.commit(); + best = Math.min(best, performance.now() - started); + } + return best; + } + // Counting the siblings by listing them made each move linear in their + // number (16x the siblings took about 16x the time). + const ratio = movesMs(16_000) / movesMs(1_000); + expect(ratio).toBeLessThan(4); + }, 60_000); + + // Rust's is_ancestor_of treats a deleted node's parent as the deleted root. + test("moves a node under its own deleted child", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const parent = tree.createNode(); + const child = tree.createNode(parent.id); + doc.commit(); + tree.delete(child.id); + doc.commit(); + const before = doc.opCount(); + tree.move(parent.id, child.id, 0); + doc.commit(); + expect(doc.opCount() - before).toBe(1); + expect(tree.toJSON()).toEqual([]); + expect(() => tree.move(child.id, child.id, 0)).toThrow(/below itself/); + }); +}); From 4b0493418f70539f21dd38dd5f86c0dfa150538e Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Tue, 29 Sep 2026 21:45:58 +0800 Subject: [PATCH 7/7] fix(loro.js): treat containers under deleted ancestors as deleted (#1147) * fix(loro.js): treat containers under deleted ancestors as deleted isDeleted() only looked at a container's own parent binding, and a tree node only at its own deleted flag. A child of a removed map value, or a node under a deleted tree node, therefore counted as alive: it kept a path, getNodes() listed it, and local edits to it recorded ops on invisible state. Rust's DocState::is_deleted follows the whole chain and rejects such edits with ContainerDeleted. Walk the parent chain (and tree ancestors) for isDeleted, getPathToContainer, and a new guard in #appendAndApply that throws Rust's message. getContainerById keeps its one-level check, since Rust still resolves those IDs. nodes() now returns every node and getNodes() lists alive nodes breadth-first, as Rust does. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(loro.js): keep deleted-container checks off #1131's reachability path Keep main's one-level _isContainerDeleted for getContainerById and the diff/event reachability walk, and use the ancestor walk only in isDeleted(), getPathToContainer() and getCursorPos(). Drop the guard on local edits to deleted containers: it threw on a redo that main allows. Co-Authored-By: Claude Opus 5.5 * fix(loro.js): list tree nodes without spreading child lists into push() getNodes() spread each child list, and with withDeleted the deleted subtrees, into one push() call, which throws a RangeError past V8's argument limit (about 125k on Node's main thread). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .changeset/loro-js-deleted-ancestors.md | 5 + loro-js/src/runtime/containers.ts | 46 +++++++-- loro-js/src/runtime/document.ts | 32 +++++- loro-js/tests/deleted-containers.test.ts | 123 +++++++++++++++++++++++ 4 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 .changeset/loro-js-deleted-ancestors.md create mode 100644 loro-js/tests/deleted-containers.test.ts diff --git a/.changeset/loro-js-deleted-ancestors.md b/.changeset/loro-js-deleted-ancestors.md new file mode 100644 index 000000000..37a053c43 --- /dev/null +++ b/.changeset/loro-js-deleted-ancestors.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +`loro.js` now treats a container as deleted when any ancestor is deleted, as `loro-crdt` does: `isDeleted()`, `getPathToContainer()`, and `getCursorPos()` account for deleted ancestors. Descendants of a deleted tree node are deleted: `getNodes()` omits them, and `isNodeDeleted()`/`LoroTreeNode.isDeleted()` return `true`. `LoroTree.nodes()` returns every node, including deleted ones, and `getNodes()` lists alive nodes breadth-first from the roots, matching Rust. diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 2ffbdc5b2..7bf15d5d6 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -149,7 +149,7 @@ export abstract class LoroContainer { } isDeleted(): boolean { - return this._doc?._isContainerDeleted(this) ?? false; + return this._doc?._isContainerUnreachable(this) ?? false; } subscribe(listener: (event: LoroEventBatch) => void): Subscription { @@ -2916,7 +2916,20 @@ export class LoroTree< isNodeDeleted(target: TreeID): boolean { this._ensureHydrated(); - return this._nodes.get(target)?.deleted ?? false; + const record = this._nodes.get(target); + return record !== undefined && this._isNodeHidden(record); + } + + /** Whether the node or one of its ancestors is deleted. */ + _isNodeHidden(record: TreeNodeRecord): boolean { + let current: TreeNodeRecord | undefined = record; + // A parent chain longer than the node count can only be a cycle. + for (let steps = 0; steps <= this._nodes.size; steps += 1) { + if (current === undefined || current.deleted) return true; + if (current.parent === undefined) return false; + current = this._nodes.get(formatTreeId(current.parent)); + } + return true; } enableFractionalIndex(jitter = 0): void { @@ -2940,15 +2953,34 @@ export class LoroTree< return record === undefined ? undefined : new LoroTreeNode(this, record.id); } + /** + * Alive nodes in breadth-first order from the roots, like Rust. With + * `withDeleted`, the deleted nodes and their subtrees follow. + */ getNodes(options: { withDeleted?: boolean } = {}): LoroTreeNode[] { this._ensureHydrated(); - return [...this._nodes.values()] - .filter((record) => options.withDeleted === true || !record.deleted) - .map((record) => new LoroTreeNode(this, record.id)); + const records = this._subtreeRecords(this._childrenOf(undefined)); + if (options.withDeleted === true) { + const deleted = [...this._nodes.values()].filter((record) => record.deleted); + for (const record of this._subtreeRecords(deleted)) records.push(record); + } + return records.map((record) => new LoroTreeNode(this, record.id)); } + /** Every node, including deleted ones, as in Rust. */ nodes(): LoroTreeNode[] { - return this.getNodes(); + this._ensureHydrated(); + return [...this._nodes.values()].map( + (record) => new LoroTreeNode(this, record.id), + ); + } + + _subtreeRecords(starts: readonly TreeNodeRecord[]): TreeNodeRecord[] { + const output = [...starts]; + for (let index = 0; index < output.length; index += 1) { + for (const child of this._childrenOf(output[index]!.id)) output.push(child); + } + return output; } roots(): LoroTreeNode[] { @@ -3275,7 +3307,7 @@ export class LoroTreeNode = Record = Record = Record = Record { + test("treat the subtree of a deleted tree node as deleted", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const parent = tree.createNode(); + const child = tree.createNode(parent.id); + const grandchild = tree.createNode(child.id); + const other = tree.createNode(); + doc.commit(); + tree.delete(parent.id); + doc.commit(); + + expect(tree.getNodes().map((node) => node.id)).toEqual([other.id]); + expect( + tree + .getNodes({ withDeleted: true }) + .map((node) => node.id) + .sort(), + ).toEqual([parent.id, child.id, grandchild.id, other.id].sort()); + expect( + tree + .nodes() + .map((node) => node.id) + .sort(), + ).toEqual([parent.id, child.id, grandchild.id, other.id].sort()); + expect(tree.isNodeDeleted(parent.id)).toBe(true); + expect(tree.isNodeDeleted(child.id)).toBe(true); + expect(tree.getNodeByID(grandchild.id)!.isDeleted()).toBe(true); + expect(tree.getNodeByID(other.id)!.isDeleted()).toBe(false); + expect(grandchild.data.isDeleted()).toBe(true); + expect(doc.getPathToContainer(grandchild.data.id)).toBeUndefined(); + + // Moving a node out of the deleted subtree revives it and its children. + // (loro-crdt 1.16 keeps the metadata of a node it already reported as + // deleted marked deleted after such a local move; a reload clears it.) + tree.move(child.id, undefined, 0); + doc.commit(); + expect(tree.isNodeDeleted(grandchild.id)).toBe(false); + grandchild.data.set("key", 1); + doc.commit(); + expect(tree.getNodes().map((node) => node.id)).toEqual([ + child.id, + other.id, + grandchild.id, + ]); + }); + + test("treat children of a removed map value as deleted", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const root = doc.getMap("root"); + const map = root.setContainer("map", new LoroMap()); + const empty = map.setContainer("empty", new LoroText()); + const text = map.setContainer("text", new LoroText()); + text.insert(0, "x"); + doc.commit(); + root.delete("map"); + doc.commit(); + + for (const container of [map, empty, text]) { + expect(container.isDeleted()).toBe(true); + expect(doc.getPathToContainer(container.id)).toBeUndefined(); + expect(doc.getContainerById(container.id)).toBeDefined(); + } + }); + + test("treats a list child removed from its list as deleted", () => { + const doc = new LoroDoc(); + const list = doc.getList("list"); + const child = list.insertContainer(0, new LoroMap()); + const grandchild = child.setContainer("text", new LoroText()); + doc.commit(); + list.delete(0, 1); + doc.commit(); + expect(child.isDeleted()).toBe(true); + expect(grandchild.isDeleted()).toBe(true); + expect(doc.getPathToContainer(grandchild.id)).toBeUndefined(); + }); + + // V8 caps the arguments of one call (about 125k on Node's main thread), so + // spreading a child list into push() threw a RangeError for a node with that + // many children, or that many deleted nodes with `withDeleted`. Building + // such a tree is too slow for CI, so check the widest push() at 2,000. + test("lists nodes without passing a whole child list to one call", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const parent = tree.createNode(); + for (let index = 0; index < 2_000; index += 1) tree.createNode(parent.id); + doc.commit(); + const widestPush = (list: () => unknown[]): number => { + const push = Array.prototype.push; + let widest = 0; + Array.prototype.push = function (this: unknown[], ...items: unknown[]) { + widest = Math.max(widest, items.length); + return push.apply(this, items); + }; + try { + list(); + } finally { + Array.prototype.push = push; + } + return widest; + }; + + expect(tree.getNodes()).toHaveLength(2_001); + expect(widestPush(() => tree.getNodes())).toBeLessThan(100); + tree.delete(parent.id); + doc.commit(); + expect(tree.getNodes({ withDeleted: true })).toHaveLength(2_001); + expect(widestPush(() => tree.getNodes({ withDeleted: true }))).toBeLessThan(100); + }); +});