diff --git a/.changeset/loro-js-checkout-current-attached.md b/.changeset/loro-js-checkout-current-attached.md new file mode 100644 index 000000000..cac9db071 --- /dev/null +++ b/.changeset/loro-js-checkout-current-attached.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +`checkout(frontiers)` on a `loro.js` document now behaves like `loro-crdt` when `frontiers` is the current version: nothing changes, and the document stays (or becomes) attached when that version is the latest one. Before, `doc.checkout(doc.frontiers())` detached the document, so later imports were not applied to its state. Checking out the latest version from an older one still leaves the document detached, as in Rust. diff --git a/.changeset/loro-js-deleted-ancestors.md b/.changeset/loro-js-deleted-ancestors.md new file mode 100644 index 000000000..37a053c43 --- /dev/null +++ b/.changeset/loro-js-deleted-ancestors.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +`loro.js` now treats a container as deleted when any ancestor is deleted, as `loro-crdt` does: `isDeleted()`, `getPathToContainer()`, and `getCursorPos()` account for deleted ancestors. Descendants of a deleted tree node are deleted: `getNodes()` omits them, and `isNodeDeleted()`/`LoroTreeNode.isDeleted()` return `true`. `LoroTree.nodes()` returns every node, including deleted ones, and `getNodes()` lists alive nodes breadth-first from the roots, matching Rust. diff --git a/.changeset/loro-js-local-op-parity.md b/.changeset/loro-js-local-op-parity.md new file mode 100644 index 000000000..8cb0bf580 --- /dev/null +++ b/.changeset/loro-js-local-op-parity.md @@ -0,0 +1,8 @@ +--- +"loro.js": patch +--- + +`loro.js` now records the same local ops as `loro-crdt` for these API calls, so a document edited through either package gets the same op IDs and the same concurrent outcomes: + +- `LoroMap.delete(key)` records a delete even when the key is absent. Like in Rust, that delete can win against a concurrent `set` of the same key. +- `LoroCounter.increment(0)` records an increment, and attaching a detached counter records an increment by its value, even 0. diff --git a/.changeset/loro-js-revived-child-events.md b/.changeset/loro-js-revived-child-events.md new file mode 100644 index 000000000..8e2cfcce9 --- /dev/null +++ b/.changeset/loro-js-revived-child-events.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +Fix a `loro.js` checkout event for a child container in a List when the checkout falls back to replaying history, for example after updates were imported while the document was detached. The List diff could delete and re-insert an unchanged child container without sending its content, so a listener that starts an inserted child from empty lost that child's content. The child's whole state is now sent, as for any other re-attached child. diff --git a/.changeset/loro-js-shallow-critical-root.md b/.changeset/loro-js-shallow-critical-root.md new file mode 100644 index 000000000..7a7fe3aa8 --- /dev/null +++ b/.changeset/loro-js-shallow-critical-root.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +`loro.js` shallow snapshots now pick the same root as `loro-crdt`: the latest single-head critical version of the requested frontiers and the latest version (loro-dev/loro#1095). Before, `loro.js` used the requested version (or the meet of its heads) even when a retained op was concurrent with it. Checking out retained versions of such a snapshot and returning to the latest version could reorder concurrent Text/List elements, and the snapshot disagreed with a Rust export of the same document. diff --git a/.changeset/loro-js-tree-local-positions.md b/.changeset/loro-js-tree-local-positions.md new file mode 100644 index 000000000..594857ff2 --- /dev/null +++ b/.changeset/loro-js-tree-local-positions.md @@ -0,0 +1,10 @@ +--- +"loro.js": patch +--- + +Fix `loro.js` local tree moves and creates to match `loro-crdt`: + +- Creating or moving a node between two siblings that share a fractional index (for example two roots created concurrently at index 0) threw `fractional index bounds are not ordered`. It now gives the following siblings with that index new positions in the same transaction, as Rust does. +- Moving a node to the position it already has, including `move(node, parent)` without an index when it is already last, `node.move(parent)`, and `moveAfter`/`moveBefore` its current neighbor, no longer records an op. +- `moveAfter` and `moveBefore` within the same parent counted the moved node itself, so moving a node after a later sibling threw `tree index … is out of range` or landed one slot off. +- Moving a node under one of its own deleted descendants threw `cannot move a tree node below itself or its descendant`. A deleted node's parent is the deleted root, as in Rust, so the move is allowed (and the node becomes deleted). diff --git a/AGENTS.md b/AGENTS.md index 523a73b39..5beac63ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,8 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. work around: [context/loro-js-rust-differential.md](context/loro-js-rust-differential.md). - loro.js rich-text style anchors, entity positions, and Rust's insert/delete/mark position rules: [context/loro-js-richtext-anchors.md](context/loro-js-richtext-anchors.md). +- loro.js multi-peer differential fuzzing against replay, checkout, event, and + Rust/WASM oracles: [context/loro-js-differential-fuzz.md](context/loro-js-differential-fuzz.md). - WASM panic/OOM reporting channels, `__wbindgen_start` glue invariant, and trap-testing recipes: [context/wasm-error-reporting.md](context/wasm-error-reporting.md). diff --git a/context/loro-js-differential-fuzz.md b/context/loro-js-differential-fuzz.md new file mode 100644 index 000000000..f29a9848c --- /dev/null +++ b/context/loro-js-differential-fuzz.md @@ -0,0 +1,135 @@ +# loro.js Differential Fuzzing + +Verified against code 2026-09-29. + +`loro-js/tests/fuzz/differential.ts` drives several `loro.js` peers through +random edits, update exchanges, and checkouts, and checks them against +independent oracles. `loro-js/tests/differential-fuzz.test.ts` runs a fixed, +CI-sized seed set by default and larger runs through environment variables. + +## Scenario model + +A scenario is a list of abstract actions: a local edit (`op`), an update +exchange (`sync`), or a checkout. Each action carries random 32-bit parameters +that are resolved against the acting document when it runs (container, position, +value). Removing actions therefore keeps the rest meaningful, which is what +`minimizeScenario` (delta debugging on the action list) relies on. + +- Containers: root Map/List/Text/Tree/Counter/MovableList, and child + containers in maps, lists, and tree metadata (`FuzzFeatures`). + MovableList and rich-text marks are off in `CORE_FEATURES`. +- Syncs: `update` from the receiver's oplog version, full updates, one blob per + missing change in a shuffled order (`import` or `importBatch`), and optionally + full snapshots into a non-empty document. +- Checkouts go to a random op or to the union of two ops, and either return to + the latest version or leave the peer detached until its next edit. + +## Oracles + +- Every peer, after a final full exchange, equals a document that imports all + updates at once, and one that imports each change separately in a shuffled + order. +- `checkout(F)` equals a fresh document that imports only the changes in `F` + (`updates-in-range` from counter 0), for every op of a small history. +- A full snapshot and three shallow snapshots equal the full-history document, + including checkouts into the retained range and back. +- `EventMirror` applies every event batch to a model keyed by container ID and + must reproduce `toJSON()` after each commit, import, and checkout. It follows + the Rust fuzzer's `ContainerTracker` (`crates/fuzz/src/value.rs`): attaching a + container value restarts that child from an empty state, a movable-list move + keeps the moved child's state, tree `delete` removes the subtree, and every + sequence index must be valid when the items are applied in order. + +## Rust/WASM reference + +With `FUZZ_WASM=1` (or a path to `loro_wasm.js`), each peer also has a +`loro-crdt` twin with the same peer ID. In lockstep mode (default when WASM is +loaded) every concrete edit, sync, and checkout runs on both, and the values +are compared after each action. The first mismatch names the action where the +implementations diverge. After the scenario, a Rust replay, Rust checkouts, and +shallow snapshots exchanged in both directions are compared too, including the +shallow root each side picks. `FUZZ_VALIDATE_HARNESS=1` checks only Rust's own +event mirror; use it after changing `EventMirror`. + +`FUZZ_WASM=1` loads the same reference as the other differential suites +(`tests/support/rust-reference.ts`, `LORO_WASM_NODEJS` overrides the path). +Lockstep requires both implementations to emit the same local ops for the same +API calls; an op-generation difference shows up as `lockstep` or +`lockstep-throws` on the first affected action. On `main` without #1145 and #1143, +most lockstep seeds diverge at the first delete of an absent map key, zero +counter increment, or checkout of the current version. With #1145 alone, 49 of +50 Map/List/Text/Counter seeds matched Rust step by step; the remaining one is +the #1143 case. + +## Running + +```sh +cd loro-js +pnpm vitest run tests/differential-fuzz.test.ts # fixed CI seeds +FUZZ_SEEDS=0-499 FUZZ_FEATURES=core FUZZ_WASM=1 FUZZ_MINIMIZE=1 \ + FUZZ_OUT=/tmp/report.json pnpm vitest run tests/differential-fuzz.test.ts +``` + +`FUZZ_CHECKS` narrows the oracles (`events,checkout,snapshot,shallow`), +`FUZZ_LOCKSTEP=0` compares with Rust only after the scenario, and `FUZZ_TRACE=1` +prints each action as it runs, which finds hangs. The report groups failures by +signature (oracle plus first differing root or normalized error) and stores the +minimized action list and a trace written as `loro.js` API calls. + +CI runs 120 fixed seeds of Map/List/Text/Counter, and 120 more with nested +child containers and snapshot imports, with the replay, checkout, +full-snapshot, and event oracles. A seed that fails on `main` for a known +reason goes into `pending` with that reason until its fix lands; both lists are +empty on 2026-09-29 (`main` at #1131). + +## Divergences found (2026-09-28/29) + +Fixed on `main`: Fugue insert order after the origin's last child, doubled +checkout events for concurrent deletes, and text delete run order (#1136); +deletes imported while detached (#1126). + +Open PRs: a `checkout(currentFrontiers)` that detached (#1143), shallow roots +that were not critical versions (#1144; without it shallow snapshots still +give wrong values after retained checkouts, sometimes also at the latest +version), absent-key map deletes and zero counter increments (#1145), +liveness of containers under deleted ancestors (#1147), local tree positions +and no-op moves (#1148), and a List child that a replay-fallback event +re-inserts (#1150). With nested children and snapshot imports, seeds 297, +315, 344, 422, 452, and 559 of 0–599 fail on `main` in the event oracle; +#1150 fixes all six. + +Still open, so `CI_RUNS` does not enable them yet: + +- Tree convergence: `loro.js` applies tree moves per node, last writer wins, + with no cycle check. Two peers that move `x` under `y` and `y` under `x` + converge to a cycle, so both nodes disappear (Rust keeps `y` as a root with + child `x`). Rust applies a tree's ops in (lamport, peer) order and skips a + move whose new parent descends from the target (`TreeCacheForDiff::apply`, + `diff_calc/tree.rs`). A port needs, per tree, the ops in that order with an + effective flag. A new op applies directly when it is the newest; otherwise + the suffix after it is retreated and reapplied. Version transitions replay + the suffix from the lowest changed op, and the shallow-root placement + fallback from #1127 applies when a node's retained ops are all retreated. +- Tree events: items are a before/after snapshot diff that lists final + indexes in op order. Importing two roots created at index 0 emits + `create 0@2 @1, create 1@2 @0` (Rust: `@0, @0`); creating a child and + deleting its parent in one commit emits the delete first; reviving a node + omits `create` for its children. Rust records an item per effective op at + the moment it is applied, plus a `create` for each child of a revived node + (`TreeState::apply_diff_and_convert`). This belongs with the port above. +- Events for containers that are unreachable after the batch: Rust drops them + (`DocState::get_path`); `loro.js` still sends them. Needs #1147's liveness. +- Shallow snapshots in the harness need #1144 (the root check fails first on + every seed without it). +- MovableList: `loro.js` skips a movable-list `set` to the current value, + which Rust records. + +## Rust-side findings + +Found while comparing; all four reproduce on the 1.16 release and are fixed +on `main` (checked on 2026-09-29 at `c0ff1255`, after #1135, #1152, and +#1153): an `Op/hint length mismatch` panic when deleting across an astral +character after `checkout` + `attach` with a subscriber; a `tree_state.rs` +panic in a subscribed checkout sequence; a tree node kept by a checkout path +between concurrent versions; and tree metadata left marked deleted after a +local move revived it. diff --git a/loro-js/AGENTS.md b/loro-js/AGENTS.md index 8b6be0101..3e46a5bb0 100644 --- a/loro-js/AGENTS.md +++ b/loro-js/AGENTS.md @@ -9,6 +9,9 @@ the same binary and JSON data as Rust (`loro-crdt`) and give the same results. [context/loro-js-performance.md](../context/loro-js-performance.md). - Randomized Text comparison with Rust (build the nodejs WASM package first): [context/loro-js-rust-differential.md](../context/loro-js-rust-differential.md). +- Multi-peer fuzzing of all containers against replay, checkout, event, and + shallow-snapshot oracles, optionally in lockstep with Rust: + [context/loro-js-differential-fuzz.md](../context/loro-js-differential-fuzz.md). - Rich text: style anchors live in the Text sequence and op positions count them. Read [context/loro-js-richtext-anchors.md](../context/loro-js-richtext-anchors.md) before changing Text positions, marks, or Text snapshot state. diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index d7e9c46df..7bf15d5d6 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -1,4 +1,4 @@ -import { bytesToHex } from "../codec/bytes"; +import { bytesEqual, bytesToHex } from "../codec/bytes"; import { containerTypeFromHistoricalByte, containerTypeToHistoricalByte, @@ -7,7 +7,7 @@ import { PostcardReader, PostcardWriter } from "../codec/postcard"; import type { ContainerId as CodecContainerId, Id as CodecId } from "../codec/types"; import { formatContainerId, formatTreeId, parseContainerId } from "./ids"; import type { LoroDoc } from "./document"; -import { fractionalIndexBetween } from "./fractional-index"; +import { fractionalIndexBetween, fractionalIndexesBetween } from "./fractional-index"; import { OrderedIndex } from "./ordered-index"; import { SequenceIndex, SequenceSpan } from "./sequence-index"; import type { SequenceIdRun } from "./sequence-index"; @@ -149,7 +149,7 @@ export abstract class LoroContainer { } isDeleted(): boolean { - return this._doc?._isContainerDeleted(this) ?? false; + return this._doc?._isContainerUnreachable(this) ?? false; } subscribe(listener: (event: LoroEventBatch) => void): Subscription { @@ -2916,7 +2916,20 @@ export class LoroTree< isNodeDeleted(target: TreeID): boolean { this._ensureHydrated(); - return this._nodes.get(target)?.deleted ?? false; + const record = this._nodes.get(target); + return record !== undefined && this._isNodeHidden(record); + } + + /** Whether the node or one of its ancestors is deleted. */ + _isNodeHidden(record: TreeNodeRecord): boolean { + let current: TreeNodeRecord | undefined = record; + // A parent chain longer than the node count can only be a cycle. + for (let steps = 0; steps <= this._nodes.size; steps += 1) { + if (current === undefined || current.deleted) return true; + if (current.parent === undefined) return false; + current = this._nodes.get(formatTreeId(current.parent)); + } + return true; } enableFractionalIndex(jitter = 0): void { @@ -2940,15 +2953,34 @@ export class LoroTree< return record === undefined ? undefined : new LoroTreeNode(this, record.id); } + /** + * Alive nodes in breadth-first order from the roots, like Rust. With + * `withDeleted`, the deleted nodes and their subtrees follow. + */ getNodes(options: { withDeleted?: boolean } = {}): LoroTreeNode[] { this._ensureHydrated(); - return [...this._nodes.values()] - .filter((record) => options.withDeleted === true || !record.deleted) - .map((record) => new LoroTreeNode(this, record.id)); + const records = this._subtreeRecords(this._childrenOf(undefined)); + if (options.withDeleted === true) { + const deleted = [...this._nodes.values()].filter((record) => record.deleted); + for (const record of this._subtreeRecords(deleted)) records.push(record); + } + return records.map((record) => new LoroTreeNode(this, record.id)); } + /** Every node, including deleted ones, as in Rust. */ nodes(): LoroTreeNode[] { - return this.getNodes(); + this._ensureHydrated(); + return [...this._nodes.values()].map( + (record) => new LoroTreeNode(this, record.id), + ); + } + + _subtreeRecords(starts: readonly TreeNodeRecord[]): TreeNodeRecord[] { + const output = [...starts]; + for (let index = 0; index < output.length; index += 1) { + for (const child of this._childrenOf(output[index]!.id)) output.push(child); + } + return output; } roots(): LoroTreeNode[] { @@ -3028,11 +3060,18 @@ export class LoroTree< ); } + /** + * Port of Rust's `generate_fi_at`: the position for a node placed at + * `index` among `parent`'s children, not counting `exclude`. When both + * neighbors share a position there is no index between them, so the right + * neighbor and every following sibling with that position get new + * positions in `rearranged`, in order. + */ _positionFor( parent: CodecId | undefined, index?: number, exclude?: CodecId, - ): Uint8Array { + ): { position: Uint8Array; rearranged: { id: CodecId; position: Uint8Array }[] } { const children = this._children.get(treeParentKey(parent)); const excluded = exclude === undefined ? undefined : this._nodes.get(formatTreeId(exclude)); @@ -3053,10 +3092,29 @@ export class LoroTree< ? siblingIndex + 1 : siblingIndex, ); - return fractionalIndexBetween( - position === 0 ? undefined : siblingAt(position - 1)!.position, - position === length ? undefined : siblingAt(position)!.position, - ); + const left = position === 0 ? undefined : siblingAt(position - 1)!.position; + const right = position === length ? undefined : siblingAt(position)!; + if (left === undefined || right === undefined || !bytesEqual(left, right.position)) { + return { position: fractionalIndexBetween(left, right?.position), rearranged: [] }; + } + const reset = [right]; + let nextRight: Uint8Array | undefined; + for (let sibling = position + 1; sibling < length; sibling += 1) { + const record = siblingAt(sibling)!; + if (!bytesEqual(record.position, left)) { + nextRight = record.position; + break; + } + reset.push(record); + } + const positions = fractionalIndexesBetween(left, nextRight, reset.length + 1); + return { + position: positions[0]!, + rearranged: reset.map((record, offset) => ({ + id: record.id, + position: positions[offset + 1]!, + })), + }; } _setRecord(record: TreeNodeRecord): void { @@ -3198,12 +3256,28 @@ export class LoroTreeNode = Record): void { + // Rust's mov_after: the index is counted without this node. const parent = target.parent(); - this.move(parent, target.index() + 1); + let index = target.index() + 1; + if (this.#hasParent(parent) && this.index() < index) index -= 1; + this.move(parent, index); } moveBefore(target: LoroTreeNode): void { - this.move(target.parent(), target.index()); + const parent = target.parent(); + let index = target.index(); + if (this.#hasParent(parent) && index >= 1 && this.index() < index) index -= 1; + this.move(parent, index); + } + + #hasParent(parent: LoroTreeNode | undefined): boolean { + const record = this.#record(); + return ( + !record.deleted && + (parent === undefined + ? record.parent === undefined + : record.parent !== undefined && formatTreeId(record.parent) === parent.id) + ); } parent(): LoroTreeNode | undefined { @@ -3233,7 +3307,7 @@ export class LoroTreeNode = Record = Record(entries); for (const [id, diff] of diffs) { @@ -2250,10 +2251,15 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record = Record = Record = Record = Record 1) { - const versions = start.map((frontier) => this.#causalVersionAt([frontier])); - const peers = new Set(versions.flatMap((version) => [...version.keys()])); - const common = new VersionVector(); - for (const peer of peers) { - const counter = Math.min(...versions.map((version) => version.get(peer) ?? 0)); - if (counter > 0) common.set(peer, counter); - } - const commonFrontiers = this.#frontiersForVersion(common); - start = commonFrontiers.length === 1 ? commonFrontiers : []; - } + // Every retained op must be causally before or after the root, so a + // branch merged after `requested` that forked below it moves the root + // down (loro-dev/loro#1095). + let start = + parsed.length === 0 + ? [] + : this.#latestSingleHeadCriticalVersion(parsed, [ + ...this.#historyFrontiers.values(), + ]); if (start.length === 1) { const operation = this.#operationAt(start[0]!); @@ -7743,6 +7813,130 @@ export class LoroDoc = Record { + const record = this.#recordContaining(id); + if (record === undefined) return undefined; + const change = record.change; + return { + peer: change.id.peer, + counter: change.id.counter, + lamport: change.lamport, + len: id.counter - change.id.counter + 1, + deps: change.dependencies, + }; + }; + const lastCounter = (span: Span): number => span.counter + span.len - 1; + const lastLamport = (span: Span): number => span.lamport + span.len - 1; + const contains = (span: Span, peer: bigint, counter: number): boolean => + span.peer === peer && span.counter <= counter && counter <= lastCounter(span); + // Max-heap order: last lamport, then peer; a shorter span with the same + // last op is greater. + const greater = (a: Span, b: Span): boolean => + lastLamport(a) !== lastLamport(b) + ? lastLamport(a) > lastLamport(b) + : a.peer !== b.peer + ? a.peer > b.peer + : a.len < b.len; + const heap: Span[] = []; + const push = (span: Span): void => { + heap.push(span); + for (let index = heap.length - 1; index > 0; ) { + const parent = (index - 1) >> 1; + if (!greater(heap[index]!, heap[parent]!)) break; + [heap[index], heap[parent]] = [heap[parent]!, heap[index]!]; + index = parent; + } + }; + const pop = (): Span => { + const top = heap[0]!; + const last = heap.pop()!; + if (heap.length > 0) { + heap[0] = last; + for (let index = 0; ; ) { + const leftChild = index * 2 + 1; + const rightChild = leftChild + 1; + let largest = index; + if (leftChild < heap.length && greater(heap[leftChild]!, heap[largest]!)) { + largest = leftChild; + } + if (rightChild < heap.length && greater(heap[rightChild]!, heap[largest]!)) { + largest = rightChild; + } + if (largest === index) break; + [heap[index], heap[largest]] = [heap[largest]!, heap[index]!]; + index = largest; + } + } + return top; + }; + for (const id of [...left, ...right]) { + const span = spanOf(id); + if (span === undefined) return []; + push(span); + } + while (heap.length > 0) { + const node = pop(); + while ( + heap.length > 0 && + heap[0]!.peer === node.peer && + lastCounter(heap[0]!) === lastCounter(node) + ) { + pop(); + } + if (heap.length === 0) return [{ peer: node.peer, counter: lastCounter(node) }]; + const other = heap[0]!; + if (contains(node, other.peer, lastCounter(other))) { + node.len = lastCounter(other) - node.counter + 1; + push(node); + continue; + } + if (node.len > 1) { + node.len = + lastLamport(other) >= node.lamport + ? Math.min(lastLamport(other) - node.lamport + 1, node.len - 1) + : 1; + push(node); + continue; + } + const deps: Span[] = []; + for (const dependency of node.deps) { + const span = spanOf(dependency); + if (span === undefined) return []; + deps.push(span); + } + if (node.counter > 0) { + const previous = spanOf({ peer: node.peer, counter: node.counter - 1 }); + if ( + previous !== undefined && + !deps.some((dep) => contains(dep, previous.peer, lastCounter(previous))) + ) { + deps.push(previous); + } + } + if (deps.length === 0) return []; + for (const dep of deps) push(dep); + } + return []; + } + #causalVersionForKnownFrontiers(frontiers: readonly CodecId[]): VersionVector { const version = new VersionVector(); for (const [peer, counter] of this.#causalVersionAt(frontiers)) { @@ -8431,6 +8625,7 @@ export class LoroDoc = Record ({ @@ -10223,7 +10418,8 @@ function restoreBlueprint(container: Container, blueprint: ContainerBlueprint): } else if (container instanceof LoroText) { container.applyDelta(blueprint.value as never); } else if (container instanceof LoroCounter) { - if ((blueprint.value as number) !== 0) container.increment(blueprint.value as number); + // Rust's CounterHandler::attach increments by the detached value, even 0. + container.increment(blueprint.value as number); } else if (container instanceof LoroList) { for (const value of blueprint.value as unknown[]) { if (isContainer(value)) container.pushContainer(value); @@ -11172,6 +11368,21 @@ function treeNodeAtPath( return index === undefined ? undefined : tree._nodeAt(undefined, index); } +function sameOptionalCodecId( + left: CodecId | undefined, + right: CodecId | undefined, +): boolean { + return left === undefined || right === undefined + ? left === right + : idsEqual(left, right); +} + +function sameFrontierSet(left: readonly OpId[], right: readonly OpId[]): boolean { + if (left.length !== right.length) return false; + const keys = new Set(left.map((id) => `${id.counter}@${id.peer}`)); + return right.every((id) => keys.has(`${id.counter}@${id.peer}`)); +} + interface StyleRedaction { /** Start IDs (`peer:counter`) of the pairs whose values were nulled. */ readonly redacted: Set; diff --git a/loro-js/src/runtime/fractional-index.ts b/loro-js/src/runtime/fractional-index.ts index 652ac8ecd..3066f8c11 100644 --- a/loro-js/src/runtime/fractional-index.ts +++ b/loro-js/src/runtime/fractional-index.ts @@ -25,6 +25,36 @@ export function fractionalIndexBetween( return terminated(between); } +/** + * `n` increasing indexes between the bounds, spread by bisection. Port of + * Rust's `FractionalIndex::generate_n_evenly`. + */ +export function fractionalIndexesBetween( + lower: Uint8Array | undefined, + upper: Uint8Array | undefined, + n: number, +): Uint8Array[] { + const output: Uint8Array[] = []; + const generate = ( + low: Uint8Array | undefined, + high: Uint8Array | undefined, + count: number, + ): void => { + if (count === 0) return; + const middle = fractionalIndexBetween(low, high); + if (count === 1) { + output.push(middle); + return; + } + const half = Math.floor(count / 2); + generate(low, middle, half); + output.push(middle); + generate(middle, high, count - half - 1); + }; + generate(lower, upper, n); + return output; +} + function terminated(bytes: number[]): Uint8Array { return Uint8Array.from([...bytes, TERMINATOR]); } diff --git a/loro-js/tests/deleted-containers.test.ts b/loro-js/tests/deleted-containers.test.ts new file mode 100644 index 000000000..12f226711 --- /dev/null +++ b/loro-js/tests/deleted-containers.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc, LoroMap, LoroText } from "../src/index"; + +// A container is deleted when any ancestor was removed from its parent, and a +// tree node is deleted when any ancestor node is. Expected values are what +// loro-crdt returns for the same calls. (loro-crdt also rejects local edits to +// a deleted container; loro.js still accepts them.) + +describe("deleted containers and tree nodes", () => { + test("treat the subtree of a deleted tree node as deleted", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const parent = tree.createNode(); + const child = tree.createNode(parent.id); + const grandchild = tree.createNode(child.id); + const other = tree.createNode(); + doc.commit(); + tree.delete(parent.id); + doc.commit(); + + expect(tree.getNodes().map((node) => node.id)).toEqual([other.id]); + expect( + tree + .getNodes({ withDeleted: true }) + .map((node) => node.id) + .sort(), + ).toEqual([parent.id, child.id, grandchild.id, other.id].sort()); + expect( + tree + .nodes() + .map((node) => node.id) + .sort(), + ).toEqual([parent.id, child.id, grandchild.id, other.id].sort()); + expect(tree.isNodeDeleted(parent.id)).toBe(true); + expect(tree.isNodeDeleted(child.id)).toBe(true); + expect(tree.getNodeByID(grandchild.id)!.isDeleted()).toBe(true); + expect(tree.getNodeByID(other.id)!.isDeleted()).toBe(false); + expect(grandchild.data.isDeleted()).toBe(true); + expect(doc.getPathToContainer(grandchild.data.id)).toBeUndefined(); + + // Moving a node out of the deleted subtree revives it and its children. + // (loro-crdt 1.16 keeps the metadata of a node it already reported as + // deleted marked deleted after such a local move; a reload clears it.) + tree.move(child.id, undefined, 0); + doc.commit(); + expect(tree.isNodeDeleted(grandchild.id)).toBe(false); + grandchild.data.set("key", 1); + doc.commit(); + expect(tree.getNodes().map((node) => node.id)).toEqual([ + child.id, + other.id, + grandchild.id, + ]); + }); + + test("treat children of a removed map value as deleted", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const root = doc.getMap("root"); + const map = root.setContainer("map", new LoroMap()); + const empty = map.setContainer("empty", new LoroText()); + const text = map.setContainer("text", new LoroText()); + text.insert(0, "x"); + doc.commit(); + root.delete("map"); + doc.commit(); + + for (const container of [map, empty, text]) { + expect(container.isDeleted()).toBe(true); + expect(doc.getPathToContainer(container.id)).toBeUndefined(); + expect(doc.getContainerById(container.id)).toBeDefined(); + } + }); + + test("treats a list child removed from its list as deleted", () => { + const doc = new LoroDoc(); + const list = doc.getList("list"); + const child = list.insertContainer(0, new LoroMap()); + const grandchild = child.setContainer("text", new LoroText()); + doc.commit(); + list.delete(0, 1); + doc.commit(); + expect(child.isDeleted()).toBe(true); + expect(grandchild.isDeleted()).toBe(true); + expect(doc.getPathToContainer(grandchild.id)).toBeUndefined(); + }); + + // V8 caps the arguments of one call (about 125k on Node's main thread), so + // spreading a child list into push() threw a RangeError for a node with that + // many children, or that many deleted nodes with `withDeleted`. Building + // such a tree is too slow for CI, so check the widest push() at 2,000. + test("lists nodes without passing a whole child list to one call", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const parent = tree.createNode(); + for (let index = 0; index < 2_000; index += 1) tree.createNode(parent.id); + doc.commit(); + const widestPush = (list: () => unknown[]): number => { + const push = Array.prototype.push; + let widest = 0; + Array.prototype.push = function (this: unknown[], ...items: unknown[]) { + widest = Math.max(widest, items.length); + return push.apply(this, items); + }; + try { + list(); + } finally { + Array.prototype.push = push; + } + return widest; + }; + + expect(tree.getNodes()).toHaveLength(2_001); + expect(widestPush(() => tree.getNodes())).toBeLessThan(100); + tree.delete(parent.id); + doc.commit(); + expect(tree.getNodes({ withDeleted: true })).toHaveLength(2_001); + expect(widestPush(() => tree.getNodes({ withDeleted: true }))).toBeLessThan(100); + }); +}); diff --git a/loro-js/tests/differential-fuzz.test.ts b/loro-js/tests/differential-fuzz.test.ts new file mode 100644 index 000000000..04ea92fa1 --- /dev/null +++ b/loro-js/tests/differential-fuzz.test.ts @@ -0,0 +1,196 @@ +import { writeFileSync, writeSync } from "node:fs"; +import { createRequire } from "node:module"; + +import { describe, expect, test } from "vitest"; + +import { + ALL_CHECKS, + ALL_FEATURES, + CORE_FEATURES, + describeScenario, + generateScenario, + minimizeScenario, + runScenario, +} from "./fuzz/differential"; +import type { FuzzChecks, FuzzFeatures, RustOracle } from "./fuzz/differential"; +import { loadRustReference } from "./support/rust-reference"; + +// See context/loro-js-differential-fuzz.md. Without FUZZ_SEEDS this file runs +// the fixed CI seeds below. Large runs, e.g.: +// +// FUZZ_SEEDS=0-999 FUZZ_FEATURES=core FUZZ_WASM=1 FUZZ_MINIMIZE=1 \ +// FUZZ_OUT=/tmp/report.json pnpm vitest run tests/differential-fuzz.test.ts +// +// FUZZ_FEATURES: core | all | comma list of FuzzFeatures keys. +// FUZZ_CHECKS: comma list of FuzzChecks keys (default: all). +// FUZZ_WASM: 1 for the reference in tests/support/rust-reference.ts (honors +// LORO_WASM_NODEJS), or a path to a loro-crdt nodejs build. +// FUZZ_LOCKSTEP: 0 to compare with Rust only after the scenario. +// FUZZ_ACTIONS: actions per scenario. FUZZ_TRACE: print actions as they run. +// FUZZ_VALIDATE_HARNESS: check only Rust's own event mirror. +const env = process.env; + +const MAP_LIST_TEXT_COUNTER: FuzzFeatures = { + ...CORE_FEATURES, + tree: false, + nested: false, +}; + +/** + * Fixed seeds that must pass on every change. They need no WASM build. + * `pending` seeds fail on main for the reasons listed; move each back into the + * run once its fix lands. + */ +const CI_RUNS: readonly { + readonly name: string; + readonly features: FuzzFeatures; + readonly checks: FuzzChecks; + readonly seeds: readonly number[]; + readonly pending: ReadonlyMap; +}[] = [ + { + name: "Map/List/Text/Counter: replay, checkout, snapshot, and event oracles", + features: MAP_LIST_TEXT_COUNTER, + checks: { events: true, checkout: true, snapshot: true, shallow: false }, + seeds: Array.from({ length: 120 }, (_, index) => index), + pending: new Map(), + }, + { + name: "nested child containers and snapshot imports: replay, checkout, snapshot, and event oracles", + features: { ...MAP_LIST_TEXT_COUNTER, nested: true, snapshotSync: true }, + checks: { events: true, checkout: true, snapshot: true, shallow: false }, + seeds: Array.from({ length: 120 }, (_, index) => index), + pending: new Map(), + }, +]; + +describe.skipIf(env.FUZZ_SEEDS !== undefined)("differential fuzz (fixed seeds)", () => { + test.each(CI_RUNS)( + "$name", + (run) => { + const failures: string[] = []; + for (const seed of run.seeds) { + if (run.pending.has(seed)) continue; + const scenario = generateScenario(seed, { features: run.features }); + const failure = runScenario(scenario, { checks: run.checks }); + if (failure !== undefined) { + failures.push( + `seed ${seed}: ${failure.message}\n${failure.trace.slice(-40).join("\n")}`, + ); + } + } + expect(failures).toEqual([]); + }, + 120_000, + ); + + test("generates the same scenario for a seed", () => { + expect(describeScenario(generateScenario(7))).toBe( + describeScenario(generateScenario(7)), + ); + expect(describeScenario(generateScenario(7))).not.toBe( + describeScenario(generateScenario(8)), + ); + }); +}); + +function parseSeeds(input: string): number[] { + const seeds: number[] = []; + for (const part of input.split(",")) { + const [start, end] = part.split("-").map(Number); + if (end === undefined) seeds.push(start!); + else for (let seed = start!; seed <= end; seed += 1) seeds.push(seed); + } + return seeds; +} + +function parseFeatures(input: string | undefined): FuzzFeatures { + if (input === undefined || input === "core") return CORE_FEATURES; + if (input === "all") return ALL_FEATURES; + const enabled = new Set(input.split(",")); + const features: Record = {}; + for (const key of Object.keys(ALL_FEATURES)) features[key] = enabled.has(key); + return features as unknown as FuzzFeatures; +} + +function parseChecks(input: string | undefined): FuzzChecks { + if (input === undefined) return ALL_CHECKS; + const enabled = new Set(input.split(",")); + return { + events: enabled.has("events"), + checkout: enabled.has("checkout"), + snapshot: enabled.has("snapshot"), + shallow: enabled.has("shallow"), + }; +} + +function loadRust(): RustOracle | undefined { + if (env.FUZZ_WASM === undefined) return undefined; + const wasm = ( + env.FUZZ_WASM === "1" + ? loadRustReference() + : createRequire(import.meta.url)(env.FUZZ_WASM) + ) as + | (Record & { + LoroDoc: new () => ReturnType; + callPendingEvents(): void; + }) + | undefined; + if (wasm === undefined) + throw new Error("FUZZ_WASM=1 needs the Rust WASM reference build"); + return { + newDoc: () => new wasm.LoroDoc(), + newContainer: (type) => new (wasm[`Loro${type}`] as new () => unknown)(), + flushEvents: () => wasm.callPendingEvents(), + }; +} + +describe.runIf(env.FUZZ_SEEDS !== undefined)("differential fuzz (manual)", () => { + test("runs the requested seeds", () => { + const seeds = parseSeeds(env.FUZZ_SEEDS!); + const features = parseFeatures(env.FUZZ_FEATURES); + const rust = loadRust(); + const checks = parseChecks(env.FUZZ_CHECKS); + const runOptions = { + rust, + checks, + lockstep: env.FUZZ_LOCKSTEP !== "0", + validateHarness: env.FUZZ_VALIDATE_HARNESS !== undefined, + onTrace: + env.FUZZ_TRACE === undefined + ? undefined + : (line: string) => writeSync(2, `${line}\n`), + }; + const actions = env.FUZZ_ACTIONS === undefined ? undefined : Number(env.FUZZ_ACTIONS); + const report: unknown[] = []; + const counts = new Map(); + for (const seed of seeds) { + const scenario = generateScenario(seed, { + features, + ...(actions === undefined ? {} : { actions }), + }); + const failure = runScenario(scenario, runOptions); + if (failure === undefined) continue; + counts.set(failure.signature, (counts.get(failure.signature) ?? 0) + 1); + let minimized = scenario; + let minimalFailure = failure; + if (env.FUZZ_MINIMIZE !== undefined) { + minimized = minimizeScenario(scenario, failure.signature, runOptions); + minimalFailure = runScenario(minimized, runOptions) ?? failure; + } + report.push({ + seed, + signature: failure.signature, + message: minimalFailure.message, + actions: minimized.actions.length, + trace: minimalFailure.trace, + scenario: JSON.parse(describeScenario(minimized)), + }); + } + const summary = [...counts.entries()].sort((a, b) => b[1] - a[1]); + if (env.FUZZ_OUT !== undefined) { + writeFileSync(env.FUZZ_OUT, JSON.stringify({ summary, report }, null, 2)); + } + expect({ failures: report.length, summary }).toEqual({ failures: 0, summary: [] }); + }, 3_600_000); +}); diff --git a/loro-js/tests/fuzz/differential.ts b/loro-js/tests/fuzz/differential.ts new file mode 100644 index 000000000..f95f548f8 --- /dev/null +++ b/loro-js/tests/fuzz/differential.ts @@ -0,0 +1,1648 @@ +// Differential fuzzing harness for the TypeScript runtime. +// +// A scenario is a list of abstract actions (local edits, update exchanges in +// several orders, checkouts). Each action carries random 32-bit parameters that +// are resolved against the acting document when it runs, so removing actions +// while minimizing keeps the remaining ones meaningful. +// +// Oracles: +// - every peer converges with a document that replays the same updates at once +// and with one that imports each change separately in a shuffled order; +// - events mirrored into a container-keyed model reproduce `toJSON()` after +// every local commit, import, and checkout; +// - `checkout(F)` equals a fresh document that imports only the changes in F; +// - a full snapshot and shallow snapshots equal the full-history document, +// including checkouts into the retained range; +// - optionally, a Rust/WASM `loro-crdt` build replays the same updates, checks +// out the same versions, and exchanges shallow snapshots with the runtime. +import { + LoroCounter, + LoroDoc, + LoroList, + LoroMap, + LoroMovableList, + LoroText, + LoroTree, +} from "../../src/runtime"; +import type { Container } from "../../src/runtime"; +import type { + ContainerID, + ContainerType, + Delta, + Frontiers, + LoroEventBatch, + OpId, + TreeDiffItem, + TreeID, +} from "../../src/runtime/types"; + +export interface FuzzFeatures { + readonly map: boolean; + readonly list: boolean; + readonly text: boolean; + readonly tree: boolean; + readonly counter: boolean; + readonly movableList: boolean; + /** Rich-text `mark`/`unmark`. */ + readonly textStyle: boolean; + /** Child containers inside maps, lists, and tree metadata. */ + readonly nested: boolean; + /** Import full snapshots into documents that already have history. */ + readonly snapshotSync: boolean; +} + +export const ALL_FEATURES: FuzzFeatures = { + map: true, + list: true, + text: true, + tree: true, + counter: true, + movableList: true, + textStyle: true, + nested: true, + snapshotSync: false, +}; + +export const CORE_FEATURES: FuzzFeatures = { + ...ALL_FEATURES, + movableList: false, + textStyle: false, +}; + +export interface FuzzChecks { + readonly events: boolean; + readonly checkout: boolean; + readonly snapshot: boolean; + readonly shallow: boolean; +} + +export const ALL_CHECKS: FuzzChecks = { + events: true, + checkout: true, + snapshot: true, + shallow: true, +}; + +export type SyncMode = "update" | "full" | "shuffled" | "batch" | "snapshot"; + +export type Action = + | { + readonly kind: "op"; + readonly peer: number; + readonly target: number; + readonly op: number; + readonly a: number; + readonly b: number; + readonly commit: boolean; + } + | { + readonly kind: "sync"; + readonly from: number; + readonly to: number; + readonly mode: SyncMode; + readonly r: number; + } + | { + readonly kind: "checkout"; + readonly peer: number; + readonly r: number; + readonly r2: number; + readonly stay: boolean; + }; + +export interface Scenario { + readonly seed: number; + readonly peers: number; + readonly features: FuzzFeatures; + readonly actions: readonly Action[]; +} + +/** A structural subset of the `loro-crdt` WASM `LoroDoc` used by the Rust oracle. */ +export interface RustDoc { + setPeerId(peer: bigint | number): void; + import(bytes: Uint8Array): unknown; + importBatch(blobs: Uint8Array[]): unknown; + export(mode: unknown): Uint8Array; + toJSON(): unknown; + commit(): void; + checkout(frontiers: Frontiers): void; + checkoutToLatest(): void; + attach(): void; + oplogVersion(): unknown; + shallowSinceFrontiers(): Frontiers; + configTextStyle(styles: unknown): void; + subscribe(listener: (batch: LoroEventBatch) => void): unknown; + getContainerById(id: ContainerID): unknown; +} + +export interface RustOracle { + newDoc(): RustDoc; + /** A detached container of the given type. */ + newContainer(type: ContainerType): unknown; + /** Delivers queued WASM events (`callPendingEvents`). */ + flushEvents(): void; +} + +export interface FuzzFailure { + readonly check: string; + readonly signature: string; + readonly message: string; + readonly trace: readonly string[]; +} + +export class FuzzMismatch extends Error { + constructor( + readonly check: string, + readonly key: string, + message: string, + ) { + super(message); + } +} + +// --------------------------------------------------------------------------- +// Random generation + +export class Random { + #state: number; + + constructor(seed: number) { + this.#state = (seed ^ 0x9e37_79b9) >>> 0 || 1; + for (let index = 0; index < 4; index += 1) this.next(); + } + + next(): number { + let x = this.#state; + x ^= x << 13; + x >>>= 0; + x ^= x >>> 17; + x ^= x << 5; + x >>>= 0; + this.#state = x; + return x; + } + + below(limit: number): number { + return limit <= 0 ? 0 : this.next() % limit; + } + + chance(numerator: number, denominator: number): boolean { + return this.below(denominator) < numerator; + } +} + +export interface GenerateOptions { + readonly peers?: number; + readonly actions?: number; + readonly features?: FuzzFeatures; +} + +export function generateScenario(seed: number, options: GenerateOptions = {}): Scenario { + const random = new Random(seed); + const peers = options.peers ?? 2 + random.below(3); + const count = options.actions ?? 30 + random.below(50); + const features = options.features ?? CORE_FEATURES; + const syncModes: SyncMode[] = ["update", "update", "full", "shuffled", "batch"]; + if (features.snapshotSync) syncModes.push("snapshot"); + const actions: Action[] = []; + for (let index = 0; index < count; index += 1) { + const roll = random.below(100); + if (roll < 66) { + actions.push({ + kind: "op", + peer: random.below(peers), + target: random.next(), + op: random.next(), + a: random.next(), + b: random.next(), + commit: random.chance(3, 4), + }); + } else if (roll < 91) { + const from = random.below(peers); + let to = random.below(peers - 1); + if (to >= from) to += 1; + actions.push({ + kind: "sync", + from, + to, + mode: syncModes[random.below(syncModes.length)]!, + r: random.next(), + }); + } else { + actions.push({ + kind: "checkout", + peer: random.below(peers), + r: random.next(), + r2: random.next(), + stay: random.chance(1, 3), + }); + } + } + return { seed, peers, features, actions }; +} + +// --------------------------------------------------------------------------- +// Value normalization + +const ROOTS: readonly { + name: string; + type: ContainerType; + feature: keyof FuzzFeatures; +}[] = [ + { name: "m", type: "Map", feature: "map" }, + { name: "l", type: "List", feature: "list" }, + { name: "t", type: "Text", feature: "text" }, + { name: "tr", type: "Tree", feature: "tree" }, + { name: "c", type: "Counter", feature: "counter" }, + { name: "ml", type: "MovableList", feature: "movableList" }, +]; + +const TEXT_STYLES = { + bold: { expand: "after" }, + link: { expand: "none" }, + comment: { expand: "both" }, + em: { expand: "before" }, +} as const; + +function sortValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortValue); + if (value instanceof Uint8Array) return Array.from(value); + if (value !== null && typeof value === "object") { + const output: Record = {}; + for (const key of Object.keys(value).sort()) { + const child = (value as Record)[key]; + if (child !== undefined) output[key] = sortValue(child); + } + return output; + } + return value; +} + +function isEmptyValue(value: unknown): boolean { + if (value === "" || value === 0 || value === null || value === undefined) return true; + if (Array.isArray(value)) return value.length === 0; + if (typeof value === "object") return Object.keys(value as object).length === 0; + return false; +} + +/** Deep JSON with sorted keys and empty root containers removed. */ +export function normalizeJson(value: unknown): Record { + const sorted = sortValue(value) as Record; + const output: Record = {}; + for (const [key, child] of Object.entries(sorted)) { + if (!isEmptyValue(child)) output[key] = child; + } + return output; +} + +interface ContainerLike { + readonly id: ContainerID; + kind(): string; +} + +function isContainerLike(value: unknown): value is ContainerLike { + return ( + value !== null && + typeof value === "object" && + typeof (value as { kind?: unknown }).kind === "function" && + typeof (value as { id?: unknown }).id === "string" + ); +} + +/** Rich-text deltas of every text reachable through maps and lists, keyed by id. */ +function collectTextDeltas( + doc: { + getContainerById(id: ContainerID): unknown; + }, + json: unknown, +): Record { + const output: Record = {}; + const visit = (handle: unknown): void => { + if (!isContainerLike(handle)) return; + const kind = handle.kind(); + if (kind === "Text") { + output[handle.id] = sortValue((handle as unknown as LoroText).toDelta()); + } else if (kind === "Map") { + const map = handle as unknown as LoroMap; + for (const key of map.keys().sort()) visit(map.get(key)); + } else if (kind === "List" || kind === "MovableList") { + const list = handle as unknown as LoroList; + for (let index = 0; index < list.length; index += 1) visit(list.get(index)); + } + }; + for (const root of ROOTS) { + if (json !== null && typeof json === "object" && root.name in json) { + visit(doc.getContainerById(`cid:root-${root.name}:${root.type}`)); + } + } + return output; +} + +export interface NormalizedValue { + readonly json: Record; + readonly texts?: Record; +} + +export function docValue( + doc: { toJSON(): unknown; getContainerById(id: ContainerID): unknown }, + withStyles: boolean, +): NormalizedValue { + const raw = doc.toJSON(); + const json = normalizeJson(raw); + if (!withStyles) return { json }; + const texts = collectTextDeltas(doc, raw); + for (const key of Object.keys(texts)) { + const delta = texts[key] as { attributes?: unknown }[]; + if (!delta.some((item) => item.attributes !== undefined)) delete texts[key]; + } + return { json, texts }; +} + +function firstDifferenceKey(left: NormalizedValue, right: NormalizedValue): string { + const keys = new Set([...Object.keys(left.json), ...Object.keys(right.json)]); + for (const key of [...keys].sort()) { + if (JSON.stringify(left.json[key]) !== JSON.stringify(right.json[key])) return key; + } + if (JSON.stringify(left.texts) !== JSON.stringify(right.texts)) return "style"; + return "?"; +} + +function describeDifference(left: unknown, right: unknown): string { + const a = String(JSON.stringify(left)); + const b = String(JSON.stringify(right)); + if (a.length + b.length < 1600) return `\n left: ${a}\n right: ${b}`; + let start = 0; + while (start < a.length && a[start] === b[start]) start += 1; + const from = Math.max(0, start - 160); + return `\n left: …${a.slice(from, start + 400)}\n right: …${b.slice(from, start + 400)}`; +} + +export function assertSameValue( + check: string, + left: NormalizedValue, + right: NormalizedValue, + context = "", +): void { + if (JSON.stringify(left) === JSON.stringify(right)) return; + const key = firstDifferenceKey(left, right); + const leftPart = key === "style" ? left.texts : left.json[key]; + const rightPart = key === "style" ? right.texts : right.json[key]; + throw new FuzzMismatch( + check, + key, + `${check} at root "${key}"${context}${describeDifference(leftPart, rightPart)}`, + ); +} + +// --------------------------------------------------------------------------- +// Event mirror + +type MirrorValue = unknown; + +interface MirrorRef { + readonly __container: ContainerID; +} + +interface TreeMirror { + readonly kind: "Tree"; + readonly parent: Map; + readonly position: Map; + readonly children: Map; +} + +type MirrorState = + | { readonly kind: "Map"; readonly value: Map } + | { readonly kind: "List" | "MovableList"; value: MirrorValue[] } + | { readonly kind: "Text"; value: string } + | { readonly kind: "Counter"; value: number } + | TreeMirror; + +function containerTypeOf(id: ContainerID): ContainerType { + return id.slice(id.lastIndexOf(":") + 1) as ContainerType; +} + +function emptyMirror(id: ContainerID): MirrorState { + const kind = containerTypeOf(id); + switch (kind) { + case "Map": + return { kind, value: new Map() }; + case "List": + case "MovableList": + return { kind, value: [] }; + case "Text": + return { kind, value: "" }; + case "Counter": + return { kind, value: 0 }; + case "Tree": + return { kind, parent: new Map(), position: new Map(), children: new Map() }; + } +} + +function treeMetaId(node: TreeID): ContainerID { + return `cid:${node}:Map` as ContainerID; +} + +class MirrorError extends Error {} + +/** + * Applies event batches to a model keyed by container ID, with the same + * container-reset rules as the Rust fuzzer's `ContainerTracker`: attaching a + * container value starts that child from an empty state, while a movable-list + * move keeps the moved child's state. + */ +export class EventMirror { + readonly #states = new Map(); + + apply(batch: LoroEventBatch): void { + for (const event of batch.events) { + const state = this.#state(event.target); + const diff = event.diff; + switch (diff.type) { + case "map": + this.#applyMap(state, diff.updated); + break; + case "list": + this.#applyList(state, diff.diff as Delta[]); + break; + case "text": + this.#applyText(state, diff.diff); + break; + case "counter": + if (state.kind !== "Counter") + throw new MirrorError(`counter diff on ${state.kind}`); + state.value += diff.increment; + break; + case "tree": + this.#applyTree(state, diff.diff); + break; + } + } + } + + toJSON( + roots: readonly { name: string; type: ContainerType }[], + ): Record { + const output: Record = {}; + for (const root of roots) { + const id = `cid:root-${root.name}:${root.type}` as ContainerID; + if (this.#states.has(id)) output[root.name] = this.#json(id); + } + return output; + } + + #state(id: ContainerID): MirrorState { + let state = this.#states.get(id); + if (state === undefined) { + state = emptyMirror(id); + this.#states.set(id, state); + } + return state; + } + + #attach(value: unknown, moved: Set): MirrorValue { + if (!isContainerLike(value)) return value; + const id = value.id; + if (!moved.has(id)) this.#states.set(id, emptyMirror(id)); + return { __container: id } satisfies MirrorRef; + } + + #applyMap(state: MirrorState, updated: Readonly>): void { + if (state.kind !== "Map") throw new MirrorError(`map diff on ${state.kind}`); + for (const [key, value] of Object.entries(updated)) { + if (value === undefined) state.value.delete(key); + else state.value.set(key, this.#attach(value, new Set())); + } + } + + #applyList(state: MirrorState, delta: readonly Delta[]): void { + if (state.kind !== "List" && state.kind !== "MovableList") { + throw new MirrorError(`list diff on ${state.kind}`); + } + const removed = new Set(); + if (state.kind === "MovableList") { + let index = 0; + for (const item of delta) { + if ("retain" in item) index += item.retain; + else if ("delete" in item) { + for (const value of state.value.slice(index, index + item.delete)) { + if (value !== null && typeof value === "object" && "__container" in value) { + removed.add((value as MirrorRef).__container); + } + } + index += item.delete; + } + } + } + const next: MirrorValue[] = []; + let index = 0; + for (const item of delta) { + if ("retain" in item) { + if (index + item.retain > state.value.length) { + throw new MirrorError( + `list retain ${item.retain} beyond ${state.value.length}`, + ); + } + next.push(...state.value.slice(index, index + item.retain)); + index += item.retain; + } else if ("delete" in item) { + if (index + item.delete > state.value.length) { + throw new MirrorError( + `list delete ${item.delete} beyond ${state.value.length}`, + ); + } + index += item.delete; + } else { + for (const value of item.insert) next.push(this.#attach(value, removed)); + } + } + next.push(...state.value.slice(index)); + state.value = next; + } + + #applyText(state: MirrorState, delta: readonly Delta[]): void { + if (state.kind !== "Text") throw new MirrorError(`text diff on ${state.kind}`); + let output = ""; + const chars = Array.from(state.value); + // Event text positions are UTF-16 code units in the JS API. + let utf16 = 0; + let charIndex = 0; + const takeUtf16 = (count: number): string => { + let taken = ""; + const target = utf16 + count; + while (utf16 < target) { + const char = chars[charIndex]; + if (char === undefined) + throw new MirrorError(`text span beyond ${state.value.length}`); + taken += char; + utf16 += char.length; + charIndex += 1; + } + if (utf16 !== target) throw new MirrorError("text span splits a surrogate pair"); + return taken; + }; + for (const item of delta) { + if ("retain" in item) { + output += takeUtf16(item.retain); + } else if ("delete" in item) { + takeUtf16(item.delete); + } else { + output += item.insert; + } + } + output += chars.slice(charIndex).join(""); + state.value = output; + } + + #applyTree(state: MirrorState, diff: readonly TreeDiffItem[]): void { + if (state.kind !== "Tree") throw new MirrorError(`tree diff on ${state.kind}`); + const childrenOf = (parent: TreeID | undefined): TreeID[] => { + const key = parent ?? ""; + let list = state.children.get(key); + if (list === undefined) { + list = []; + state.children.set(key, list); + } + return list; + }; + const insert = (target: TreeID, parent: TreeID | undefined, index: number): void => { + if (parent !== undefined && !state.parent.has(parent)) { + throw new MirrorError(`tree parent ${parent} of ${target} does not exist`); + } + const siblings = childrenOf(parent); + if (index > siblings.length) { + throw new MirrorError( + `tree index ${index} beyond ${siblings.length} for ${target}`, + ); + } + siblings.splice(index, 0, target); + state.parent.set(target, parent); + }; + const detach = (target: TreeID): void => { + if (!state.parent.has(target)) + throw new MirrorError(`tree node ${target} is absent`); + const siblings = childrenOf(state.parent.get(target)); + const index = siblings.indexOf(target); + if (index < 0) throw new MirrorError(`tree node ${target} is not under its parent`); + siblings.splice(index, 1); + }; + for (const item of diff) { + if (item.action === "create") { + if (state.parent.has(item.target)) { + throw new MirrorError(`tree node ${item.target} already exists`); + } + insert(item.target, item.parent, item.index); + state.position.set(item.target, item.fractionalIndex); + this.#states.set(treeMetaId(item.target), emptyMirror(treeMetaId(item.target))); + } else if (item.action === "move") { + detach(item.target); + insert(item.target, item.parent, item.index); + state.position.set(item.target, item.fractionalIndex); + } else { + detach(item.target); + const stack = [item.target]; + while (stack.length > 0) { + const node = stack.pop()!; + stack.push(...(state.children.get(node) ?? [])); + state.children.delete(node); + state.parent.delete(node); + state.position.delete(node); + } + } + } + } + + #json(id: ContainerID): unknown { + const state = this.#states.get(id) ?? emptyMirror(id); + const value = (item: MirrorValue): unknown => + item !== null && typeof item === "object" && "__container" in item + ? this.#json((item as MirrorRef).__container) + : item; + switch (state.kind) { + case "Map": { + const output: Record = {}; + for (const [key, item] of state.value) output[key] = value(item); + return output; + } + case "List": + case "MovableList": + return state.value.map(value); + case "Text": + case "Counter": + return state.value; + case "Tree": { + const nodes = (parent: TreeID | undefined): unknown[] => + (state.children.get(parent ?? "") ?? []).map((node, index) => ({ + id: node, + parent: parent ?? null, + index, + fractional_index: state.position.get(node), + meta: this.#json(treeMetaId(node)), + children: nodes(node), + })); + return nodes(undefined); + } + } + } +} + +// --------------------------------------------------------------------------- +// Scenario execution + +interface Mirrored { + readonly doc: D; + readonly mirror: EventMirror; + mirrorError: unknown; +} + +interface Peer extends Mirrored { + readonly name: string; + readonly rust: Mirrored | undefined; +} + +/** A local edit resolved against the acting document. */ +interface ConcreteOp { + readonly target: ContainerID; + readonly method: string; + readonly args: readonly unknown[]; +} + +interface NewContainer { + readonly __new: ContainerType; +} + +function isNewContainer(value: unknown): value is NewContainer { + return value !== null && typeof value === "object" && "__new" in value; +} + +function enabledRoots(features: FuzzFeatures): typeof ROOTS { + return ROOTS.filter((root) => features[root.feature]); +} + +function scalar(seed: number): unknown { + switch (seed % 5) { + case 0: + return seed % 97; + case 1: + return `s${seed % 31}`; + case 2: + return (seed & 8) !== 0; + case 3: + return null; + default: + return { n: seed % 7 }; + } +} + +const TEXT_PIECES = ["a", "bc", "XYZ", "中", "😀", "e😀f", "\n", "hello"]; + +function textPiece(seed: number): string { + return TEXT_PIECES[seed % TEXT_PIECES.length]!; +} + +function childContainerTypes(features: FuzzFeatures): ContainerType[] { + const types: ContainerType[] = []; + if (features.map) types.push("Map", "Map"); + if (features.list) types.push("List"); + if (features.text) types.push("Text"); + if (features.counter) types.push("Counter"); + if (features.tree) types.push("Tree"); + if (features.movableList) types.push("MovableList"); + return types; +} + +function newDetached(type: ContainerType): Container { + switch (type) { + case "Map": + return new LoroMap(); + case "List": + return new LoroList(); + case "Text": + return new LoroText(); + case "Counter": + return new LoroCounter(); + case "Tree": + return new LoroTree(); + case "MovableList": + return new LoroMovableList(); + } +} + +function frontiersLiteral(frontiers: Frontiers): string { + return JSON.stringify(frontiers.map((id) => ({ peer: id.peer, counter: id.counter }))); +} + +const ROOT_GETTERS: Record = { + Map: "getMap", + List: "getList", + Text: "getText", + Tree: "getTree", + Counter: "getCounter", + MovableList: "getMovableList", +}; + +function containerExpression(doc: string, id: ContainerID): string { + const root = /^cid:root-(.*):(\w+)$/.exec(id); + if (root !== null) { + return `${doc}.${ROOT_GETTERS[root[2] as ContainerType]}(${JSON.stringify(root[1])})`; + } + return `(${doc}.getContainerById(${JSON.stringify(id)}) as any)`; +} + +function renderArgument(value: unknown): string { + if (isNewContainer(value)) return `new Loro${value.__new}()`; + if (value === undefined) return "undefined"; + return JSON.stringify(value); +} + +function renderOp(doc: string, op: ConcreteOp): string { + return `${containerExpression(doc, op.target)}.${op.method}(${op.args.map(renderArgument).join(", ")});`; +} + +function executeOp( + doc: { getContainerById(id: ContainerID): unknown }, + op: ConcreteOp, + create: (type: ContainerType) => unknown, +): string | undefined { + try { + const container = doc.getContainerById(op.target) as Record< + string, + (...args: unknown[]) => unknown + >; + const args = op.args.map((arg) => (isNewContainer(arg) ? create(arg.__new) : arg)); + container[op.method]!(...args); + return undefined; + } catch (error) { + return String((error as Error)?.message ?? error); + } +} + +function sortedChanges(doc: LoroDoc): [string, { counter: number; length: number }[]][] { + return [...doc.getAllChanges().entries()].sort(([a], [b]) => (a < b ? -1 : 1)); +} + +export interface RunOptions { + readonly checks?: FuzzChecks; + readonly rust?: RustOracle | undefined; + /** Run every action on Rust peers too and compare after each action. */ + readonly lockstep?: boolean; + /** Upper bound on versions checked out after the scenario. */ + readonly maxCheckoutVersions?: number; + readonly onTrace?: ((line: string) => void) | undefined; + /** Check only the Rust event mirrors and values; validates the harness itself. */ + readonly validateHarness?: boolean; +} + +class Runner { + readonly peers: Peer[] = []; + readonly trace: string[] = []; + readonly roots: typeof ROOTS; + readonly checks: FuzzChecks; + readonly rust: RustOracle | undefined; + readonly lockstep: boolean; + rustReplay: Mirrored | undefined; + + constructor( + readonly scenario: Scenario, + readonly options: RunOptions, + ) { + this.roots = enabledRoots(scenario.features); + this.checks = options.checks ?? ALL_CHECKS; + this.rust = options.rust; + this.lockstep = options.lockstep === true && options.rust !== undefined; + for (let index = 0; index < scenario.peers; index += 1) { + const doc = this.newDoc(index + 1); + let rust: Mirrored | undefined; + if (this.lockstep) { + const rustDoc = this.newRustDoc()!; + rustDoc.setPeerId(BigInt(index + 1)); + rust = { doc: rustDoc, mirror: new EventMirror(), mirrorError: undefined }; + if (this.checks.events) this.subscribe(rust); + } + const peer: Peer = { + name: `p${index + 1}`, + doc, + mirror: new EventMirror(), + mirrorError: undefined, + rust, + }; + if (this.checks.events) this.subscribe(peer); + this.peers.push(peer); + } + } + + get withStyles(): boolean { + return this.scenario.features.textStyle; + } + + log(line: string): void { + this.trace.push(line); + this.options.onTrace?.(line); + } + + newDoc(peer?: number): LoroDoc { + const doc = new LoroDoc(); + if (peer !== undefined) doc.setPeerId(peer); + if (this.scenario.features.textStyle) doc.configTextStyle(TEXT_STYLES); + return doc; + } + + newRustDoc(): RustDoc | undefined { + const doc = this.rust?.newDoc(); + if (doc !== undefined && this.scenario.features.textStyle) { + doc.configTextStyle(TEXT_STYLES); + } + return doc; + } + + subscribe( + target: Mirrored<{ subscribe(listener: (batch: LoroEventBatch) => void): unknown }>, + ): void { + target.doc.subscribe((batch) => { + if (target.mirrorError !== undefined) return; + try { + target.mirror.apply(batch); + } catch (error) { + target.mirrorError = error; + } + }); + } + + value(doc: LoroDoc): NormalizedValue { + return docValue(doc, this.withStyles); + } + + rustValue(doc: RustDoc): NormalizedValue { + return docValue(doc, this.withStyles); + } + + checkMirror( + target: Mirrored<{ toJSON(): unknown }>, + label: string, + check = "event", + ): void { + if (!this.checks.events) return; + if (this.options.validateHarness === true && !check.startsWith("rust")) return; + if (target.mirrorError !== undefined) { + const message = String((target.mirrorError as Error).message ?? target.mirrorError); + throw new FuzzMismatch( + `${check}-invalid`, + message.replace(/\d+(@\d+)?/g, "N"), + `${label}: ${message}`, + ); + } + const mirrored = normalizeJson(sortValue(target.mirror.toJSON(this.roots))); + const actual = normalizeJson(target.doc.toJSON()); + assertSameValue( + `${check}-mirror`, + { json: mirrored }, + { json: actual }, + ` (${label})`, + ); + } + + /** Compares a peer with its Rust twin after an action in lockstep mode. */ + checkPeer(peer: Peer, label: string): void { + this.checkMirror(peer, label); + if (peer.rust === undefined) return; + this.rust!.flushEvents(); + assertSameValue( + "lockstep", + this.value(peer.doc), + this.rustValue(peer.rust.doc), + ` (${peer.name} vs Rust after ${label})`, + ); + this.checkMirror(peer.rust, `Rust ${label}`, "rust-event"); + } + + // -- local edits ----------------------------------------------------------- + + containers(doc: LoroDoc): ContainerLike[] { + const output: ContainerLike[] = []; + const queue: ContainerLike[] = []; + for (const root of this.roots) { + queue.push(doc.getContainerById(`cid:root-${root.name}:${root.type}`) as never); + } + while (queue.length > 0 && output.length < 32) { + const container = queue.shift()!; + output.push(container); + const kind = container.kind(); + if (kind === "Map") { + const map = container as unknown as LoroMap; + for (const key of map.keys().sort()) { + const child = map.get(key); + if (isContainerLike(child)) queue.push(child); + } + } else if (kind === "List" || kind === "MovableList") { + const list = container as unknown as LoroList; + for (let index = 0; index < list.length; index += 1) { + const child = list.get(index); + if (isContainerLike(child)) queue.push(child); + } + } else if (kind === "Tree" && this.scenario.features.nested) { + const tree = container as unknown as LoroTree; + const nodes = tree.getNodes().sort((a, b) => (a.id < b.id ? -1 : 1)); + for (const node of nodes) queue.push(node.data as unknown as ContainerLike); + } + } + return output; + } + + /** Resolves an abstract edit against `doc`; undefined when it is a no-op. */ + concreteOp( + doc: LoroDoc, + action: Extract, + ): ConcreteOp | undefined { + const containers = this.containers(doc); + const container = containers[action.target % containers.length]!; + const target = container.id; + const features = this.scenario.features; + const types = childContainerTypes(features); + const childType: NewContainer = { __new: types[action.b % types.length]! }; + const { a, b } = action; + switch (container.kind()) { + case "Map": { + const key = `k${a % 5}`; + const choice = action.op % 5; + if (choice === 0) return { target, method: "delete", args: [key] }; + if (choice === 1 && features.nested) { + return { target, method: "setContainer", args: [key, childType] }; + } + return { target, method: "set", args: [key, scalar(b)] }; + } + case "List": + case "MovableList": { + const list = container as unknown as LoroMovableList; + const movable = container.kind() === "MovableList"; + const length = list.length; + const choice = action.op % (movable ? 7 : 5); + if (length > 0 && (choice === 0 || choice === 1)) { + const pos = a % length; + const len = 1 + (b % Math.min(3, length - pos)); + return { target, method: "delete", args: [pos, len] }; + } + if (choice === 2 && features.nested) { + return { + target, + method: "insertContainer", + args: [a % (length + 1), childType], + }; + } + if (movable && length > 0 && choice === 5) { + return { target, method: "move", args: [a % length, b % length] }; + } + if (movable && length > 0 && choice === 6) { + return { target, method: "set", args: [a % length, scalar(b)] }; + } + return { target, method: "insert", args: [a % (length + 1), scalar(b)] }; + } + case "Text": { + const text = container as unknown as LoroText; + const length = text.length; + const choice = action.op % (features.textStyle ? 6 : 4); + if (length > 0 && choice === 0) { + const pos = safeTextPos(text, a % length); + const end = safeTextPos(text, Math.min(length, pos + 1 + (b % 4))); + return end > pos + ? { target, method: "delete", args: [pos, end - pos] } + : undefined; + } + if (length > 0 && choice >= 4) { + const start = safeTextPos(text, a % length); + const end = safeTextPos(text, Math.min(length, start + 1 + (b % 5))); + if (end <= start) return undefined; + const keys = Object.keys(TEXT_STYLES); + const key = keys[(b >>> 8) % keys.length]!; + if (choice === 4 || (b & 1) === 0) { + const value = (b >>> 12) % 3 === 0 ? null : (b >>> 12) % 5; + return { target, method: "mark", args: [{ start, end }, key, value] }; + } + return { target, method: "unmark", args: [{ start, end }, key] }; + } + return { + target, + method: "insert", + args: [safeTextPos(text, a % (length + 1)), textPiece(b)], + }; + } + case "Counter": + return { target, method: "increment", args: [(a % 9) - 4 || 5] }; + case "Tree": { + const tree = container as unknown as LoroTree; + const nodes = tree.getNodes().sort((x, y) => (x.id < y.id ? -1 : 1)); + const choice = action.op % 6; + const childCount = (parent: TreeID | undefined): number => + parent === undefined + ? tree.roots().length + : tree.getNodeByID(parent)!.children().length; + if (nodes.length > 0 && choice === 0) { + return { target, method: "delete", args: [nodes[a % nodes.length]!.id] }; + } + const parentIndex = b % (nodes.length + 1); + const parent = parentIndex === nodes.length ? undefined : nodes[parentIndex]!.id; + if (nodes.length > 0 && (choice === 1 || choice === 2)) { + const node = nodes[a % nodes.length]!.id; + const siblings = childCount(parent); + const current = tree.getNodeByID(node)!; + const sameParent = current.parent()?.id === parent; + const position = (b >>> 8) % (sameParent ? siblings : siblings + 1); + return { target, method: "move", args: [node, parent, position] }; + } + return { + target, + method: "createNode", + args: [parent, (a >>> 4) % (childCount(parent) + 1)], + }; + } + } + return undefined; + } + + applyOp(index: number, action: Extract): void { + const peer = this.peers[action.peer]!; + if (peer.doc.isDetached()) { + this.log(`${peer.name}.attach();`); + peer.doc.attach(); + peer.rust?.doc.attach(); + this.checkPeer(peer, `attach before action ${index}`); + } + const op = this.concreteOp(peer.doc, action); + if (op !== undefined) { + this.log(renderOp(peer.name, op)); + const error = executeOp(peer.doc, op, newDetached); + if (error !== undefined) this.log(`// throws: ${error}`); + if (peer.rust !== undefined) { + const rustError = executeOp(peer.rust.doc, op, (type) => + this.rust!.newContainer(type), + ); + if ((error === undefined) !== (rustError === undefined)) { + throw new FuzzMismatch( + "lockstep-throws", + `${op.method}`, + `${renderOp(peer.name, op)} ${error === undefined ? "succeeds" : `throws "${error}"`} in loro-js but ${rustError === undefined ? "succeeds" : `throws "${rustError}"`} in Rust`, + ); + } + } + } + if (action.commit) { + this.log(`${peer.name}.commit();`); + peer.doc.commit(); + peer.rust?.doc.commit(); + this.checkPeer(peer, `commit after action ${index}`); + } + } + + // -- synchronization ----------------------------------------------------- + + /** ID spans of every change in `source` that is not covered by `known`. */ + missingSpans( + source: LoroDoc, + known: ReturnType, + ): { id: OpId; len: number }[] { + const spans: { id: OpId; len: number }[] = []; + for (const [peer, list] of sortedChanges(source)) { + const seen = known.get(peer) ?? 0; + for (const change of list) { + const start = Math.max(change.counter, seen); + const end = change.counter + change.length; + if (start < end) { + spans.push({ + id: { peer: peer as OpId["peer"], counter: start }, + len: end - start, + }); + } + } + } + return spans; + } + + sync(index: number, action: Extract): void { + const from = this.peers[action.from]!; + const to = this.peers[action.to]!; + for (const peer of [from, to]) { + peer.doc.commit(); + peer.rust?.doc.commit(); + } + const rustFrom = from.rust?.doc; + const rustTo = to.rust?.doc; + switch (action.mode) { + case "update": + this.log( + `${to.name}.import(${from.name}.export({ mode: "update", from: ${to.name}.oplogVersion() }));`, + ); + to.doc.import(from.doc.export({ mode: "update", from: to.doc.oplogVersion() })); + rustTo?.import(rustFrom!.export({ mode: "update", from: rustTo.oplogVersion() })); + break; + case "full": + this.log(`${to.name}.import(${from.name}.export({ mode: "update" }));`); + to.doc.import(from.doc.export({ mode: "update" })); + rustTo?.import(rustFrom!.export({ mode: "update" })); + break; + case "snapshot": + this.log(`${to.name}.import(${from.name}.export({ mode: "snapshot" }));`); + to.doc.import(from.doc.export({ mode: "snapshot" })); + rustTo?.import(rustFrom!.export({ mode: "snapshot" })); + break; + case "shuffled": + case "batch": { + const spans = this.missingSpans(from.doc, to.doc.oplogVersion()); + shuffle(spans, new Random(action.r)); + const exportSpan = (span: { id: OpId; len: number }): string => + `${from.name}.export({ mode: "updates-in-range", spans: [${JSON.stringify(span)}] })`; + if (action.mode === "batch") { + this.log(`${to.name}.importBatch([${spans.map(exportSpan).join(", ")}]);`); + } else { + for (const span of spans) this.log(`${to.name}.import(${exportSpan(span)});`); + } + const blobs = (doc: { export(mode: unknown): Uint8Array }): Uint8Array[] => + spans.map((span) => doc.export({ mode: "updates-in-range", spans: [span] })); + if (action.mode === "batch") { + to.doc.importBatch(blobs(from.doc)); + rustTo?.importBatch(blobs(rustFrom!)); + } else { + for (const blob of blobs(from.doc)) to.doc.import(blob); + if (rustTo !== undefined) + for (const blob of blobs(rustFrom!)) rustTo.import(blob); + } + break; + } + } + this.checkPeer(to, `import in action ${index}`); + } + + // -- versions ------------------------------------------------------------ + + /** Every operation ID in `doc`'s history, sorted deterministically. */ + allOpIds(doc: LoroDoc): OpId[] { + const ids: OpId[] = []; + for (const [peer, list] of sortedChanges(doc)) { + for (const change of list) { + for (let offset = 0; offset < change.length; offset += 1) { + ids.push({ peer: peer as OpId["peer"], counter: change.counter + offset }); + } + } + } + return ids; + } + + pickFrontiers(doc: LoroDoc, r: number, r2: number): Frontiers | undefined { + const ids = this.allOpIds(doc); + if (ids.length === 0) return undefined; + const first = ids[r % ids.length]!; + if (r2 % 4 !== 0) return [first]; + const second = ids[r2 % ids.length]!; + const version = doc.frontiersToVV([first]); + version.merge(doc.frontiersToVV([second])); + return doc.vvToFrontiers(version); + } + + /** A fresh document containing exactly the changes included in `frontiers`. */ + replayAt(source: LoroDoc, frontiers: Frontiers): LoroDoc { + const version = source.frontiersToVV(frontiers); + const spans = [...version.toJSON().entries()] + .filter(([, end]) => end > 0) + .map(([peer, end]) => ({ id: { peer, counter: 0 }, len: end })); + const doc = this.newDoc(); + if (spans.length > 0) doc.import(source.export({ mode: "updates-in-range", spans })); + return doc; + } + + checkoutAction(index: number, action: Extract): void { + const peer = this.peers[action.peer]!; + peer.doc.commit(); + peer.rust?.doc.commit(); + const frontiers = this.pickFrontiers(peer.doc, action.r, action.r2); + if (frontiers === undefined) return; + this.log(`${peer.name}.checkout(${frontiersLiteral(frontiers)});`); + peer.doc.checkout(frontiers); + peer.rust?.doc.checkout(frontiers); + this.checkPeer(peer, `checkout in action ${index}`); + if (this.checks.checkout) { + assertSameValue( + "checkout-vs-replay", + this.value(peer.doc), + this.value(this.replayAt(peer.doc, frontiers)), + ` (${peer.name} at ${frontiersLiteral(frontiers)})`, + ); + } + if (!action.stay) { + this.log(`${peer.name}.checkoutToLatest();`); + peer.doc.checkoutToLatest(); + peer.rust?.doc.checkoutToLatest(); + this.checkPeer(peer, `checkoutToLatest in action ${index}`); + } + } + + // -- final checks ---------------------------------------------------------- + + finish(): void { + for (const peer of this.peers) { + peer.doc.commit(); + peer.rust?.doc.commit(); + if (peer.doc.isDetached()) { + this.log(`${peer.name}.checkoutToLatest();`); + peer.doc.checkoutToLatest(); + peer.rust?.doc.checkoutToLatest(); + this.checkPeer(peer, "final attach"); + } + } + this.log("// final: every peer imports every other peer's updates"); + for (let round = 0; round < 2; round += 1) { + for (const from of this.peers) { + for (const to of this.peers) { + if (from === to) continue; + to.doc.import(from.doc.export({ mode: "update", from: to.doc.oplogVersion() })); + to.rust?.doc.import( + from.rust!.doc.export({ mode: "update", from: to.rust.doc.oplogVersion() }), + ); + } + } + } + for (const peer of this.peers) this.checkPeer(peer, `final sync of ${peer.name}`); + const base = this.peers[0]!.doc; + const baseValue = this.value(base); + for (const peer of this.peers.slice(1)) { + assertSameValue( + "peer-divergence", + this.value(peer.doc), + baseValue, + ` (${peer.name} vs p1)`, + ); + } + + const updates = base.export({ mode: "update" }); + const replay: Mirrored = { + doc: this.newDoc(), + mirror: new EventMirror(), + mirrorError: undefined, + }; + if (this.checks.events) this.subscribe(replay); + replay.doc.import(updates); + this.checkMirror(replay, "replay import"); + const replayValue = this.value(replay.doc); + const rust = this.newRustDoc(); + if (rust !== undefined) { + if (this.checks.events) { + this.rustReplay = { + doc: rust, + mirror: new EventMirror(), + mirrorError: undefined, + }; + this.subscribe(this.rustReplay); + } + rust.import(updates); + this.rust!.flushEvents(); + assertSameValue( + "rust-replay", + replayValue, + this.rustValue(rust), + " (loro-js replay vs Rust replay)", + ); + } + assertSameValue( + "replay-vs-peers", + baseValue, + replayValue, + " (p1 vs one-shot replay)", + ); + + const shuffled = this.newDoc(); + const spans = this.missingSpans(base, shuffled.oplogVersion()); + shuffle(spans, new Random(this.scenario.seed ^ 0x5bd1_e995)); + for (const span of spans) { + shuffled.import(base.export({ mode: "updates-in-range", spans: [span] })); + } + assertSameValue( + "shuffled-replay", + this.value(shuffled), + replayValue, + " (per-change shuffled replay vs one-shot replay)", + ); + + const versions = this.checkoutVersions(base); + if (this.checks.checkout) { + this.checkoutEverywhere("replay", replay, versions, rust); + this.checkoutEverywhere("p1", this.peers[0]!, versions, undefined); + } + if (this.checks.snapshot) { + // Events are not mirrored here: the mirror would need the imported state as its base. + const snapshot: Mirrored = { + doc: this.newDoc(), + mirror: new EventMirror(), + mirrorError: undefined, + }; + snapshot.doc.import(base.export({ mode: "snapshot" })); + assertSameValue( + "snapshot-latest", + this.value(snapshot.doc), + replayValue, + " (snapshot import)", + ); + if (this.checks.checkout) { + this.checkoutEverywhere("snapshot", snapshot, versions, undefined, false); + } + } + if (this.checks.shallow) this.checkShallow(base, versions); + } + + checkoutVersions(doc: LoroDoc): Frontiers[] { + const ids = this.allOpIds(doc); + const limit = this.options.maxCheckoutVersions ?? 48; + const random = new Random(this.scenario.seed ^ 0x2545_f491); + const versions: Frontiers[] = []; + if (ids.length <= limit) { + for (const id of ids) versions.push([id]); + } else { + for (let index = 0; index < limit; index += 1) { + versions.push([ids[random.below(ids.length)]!]); + } + } + for (let index = 0; index < Math.min(8, ids.length); index += 1) { + const frontiers = this.pickFrontiers(doc, random.next(), 0); + if (frontiers !== undefined) versions.push(frontiers); + } + return versions; + } + + checkoutEverywhere( + label: string, + target: Mirrored, + versions: readonly Frontiers[], + rust: RustDoc | undefined, + mirrored = true, + ): void { + const base = this.peers[0]!.doc; + const order = [...versions]; + shuffle(order, new Random(this.scenario.seed ^ 0x68e3_1da4)); + for (const frontiers of order) { + const context = ` (${label}.checkout(${frontiersLiteral(frontiers)}))`; + this.log(`${label}.checkout(${frontiersLiteral(frontiers)});`); + try { + target.doc.checkout(frontiers); + } catch (error) { + const message = String((error as Error).message); + throw new FuzzMismatch( + `checkout-throws:${label}`, + normalizeMessage(message), + `${label}.checkout(${frontiersLiteral(frontiers)}) threw: ${message}`, + ); + } + if (mirrored) + this.checkMirror(target, `${label} checkout ${frontiersLiteral(frontiers)}`); + const expected = this.value(this.replayAt(base, frontiers)); + assertSameValue(`checkout:${label}`, this.value(target.doc), expected, context); + if (rust !== undefined) { + rust.checkout(frontiers); + assertSameValue("rust-checkout", expected, this.rustValue(rust), context); + if (this.rustReplay?.doc === rust) { + this.rust!.flushEvents(); + this.checkMirror(this.rustReplay, `Rust ${label} checkout`, "rust-event"); + } + } + } + target.doc.checkoutToLatest(); + if (mirrored) this.checkMirror(target, `${label} checkoutToLatest`); + rust?.checkoutToLatest(); + } + + checkShallow(base: LoroDoc, versions: readonly Frontiers[]): void { + const random = new Random(this.scenario.seed ^ 0x1b87_3593); + if (this.allOpIds(base).length === 0) return; + const latest = this.value(base); + const rustFull = this.newRustDoc(); + rustFull?.import(base.export({ mode: "update" })); + for (let attempt = 0; attempt < 3; attempt += 1) { + const root = this.pickFrontiers(base, random.next(), random.next())!; + const context = ` (shallow at ${frontiersLiteral(root)})`; + this.log( + `const shallow = LoroDoc.fromSnapshot(p1.export({ mode: "shallow-snapshot", frontiers: ${frontiersLiteral(root)} }));`, + ); + const blob = base.export({ mode: "shallow-snapshot", frontiers: root }); + const shallow = this.newDoc(); + shallow.import(blob); + assertSameValue("shallow-latest", this.value(shallow), latest, context); + if (rustFull !== undefined) { + const rustShallow = this.newRustDoc()!; + rustShallow.import(blob); + const rustRoot = this.newRustDoc()!; + rustRoot.import(rustFull.export({ mode: "shallow-snapshot", frontiers: root })); + const roots = [shallow, rustRoot].map((doc) => + frontiersLiteral( + [...doc.shallowSinceFrontiers()].sort((a, b) => + a.peer === b.peer ? a.counter - b.counter : a.peer < b.peer ? -1 : 1, + ), + ), + ); + if (roots[0] !== roots[1]) { + throw new FuzzMismatch( + "shallow-root", + "root", + `shallow root${context}: loro-js ${roots[0]}, Rust ${roots[1]}`, + ); + } + assertSameValue( + "rust-imports-ts-shallow", + this.rustValue(rustShallow), + latest, + context, + ); + const fromRust = this.newDoc(); + fromRust.import(rustFull.export({ mode: "shallow-snapshot", frontiers: root })); + assertSameValue("ts-imports-rust-shallow", this.value(fromRust), latest, context); + } + if (!this.checks.checkout) continue; + const since = base.frontiersToVV(shallow.shallowSinceFrontiers()); + for (const frontiers of versions) { + const order = since.compare(base.frontiersToVV(frontiers)); + if (order !== -1 && order !== 0) continue; + this.log(`shallow.checkout(${frontiersLiteral(frontiers)});`); + let actual: NormalizedValue; + try { + shallow.checkout(frontiers); + actual = this.value(shallow); + } catch (error) { + const message = String((error as Error).message); + throw new FuzzMismatch( + "shallow-checkout-throws", + normalizeMessage(message), + `shallow.checkout(${frontiersLiteral(frontiers)}) threw${context}: ${message}`, + ); + } + assertSameValue( + "shallow-checkout", + actual, + this.value(this.replayAt(base, frontiers)), + `${context} checkout ${frontiersLiteral(frontiers)}`, + ); + } + shallow.checkoutToLatest(); + assertSameValue( + "shallow-latest", + this.value(shallow), + latest, + `${context} after checkouts`, + ); + } + } +} + +function safeTextPos(text: LoroText, pos: number): number { + // Never split a surrogate pair. + const value = text.toString(); + let clamped = Math.min(pos, value.length); + const code = value.charCodeAt(clamped - 1); + if (clamped > 0 && code >= 0xd800 && code <= 0xdbff) clamped -= 1; + return clamped; +} + +function shuffle(items: T[], random: Random): void { + for (let index = items.length - 1; index > 0; index -= 1) { + const other = random.below(index + 1); + [items[index], items[other]] = [items[other]!, items[index]!]; + } +} + +function normalizeMessage(message: string): string { + return message + .split("\n")[0]! + .replace(/cid:[^\s)"]+/g, "CID") + .replace(/\d+/g, "N") + .slice(0, 120); +} + +/** Runs a scenario; returns undefined on success or the first failure. */ +export function runScenario( + scenario: Scenario, + options: RunOptions = {}, +): FuzzFailure | undefined { + const runner = new Runner(scenario, options); + let phase = "setup"; + try { + for (const [index, action] of scenario.actions.entries()) { + phase = `${action.kind}#${index}`; + if (action.kind === "op") runner.applyOp(index, action); + else if (action.kind === "sync") runner.sync(index, action); + else runner.checkoutAction(index, action); + } + phase = "final"; + runner.finish(); + return undefined; + } catch (error) { + if (error instanceof FuzzMismatch) { + return { + check: error.check, + signature: `${error.check}:${error.key}`, + message: error.message, + trace: runner.trace, + }; + } + const message = (error as Error)?.stack ?? String(error); + const kind = phase.replace(/#\d+$/, ""); + return { + check: `throws:${kind}`, + signature: `throws:${kind}:${normalizeMessage((error as Error)?.message ?? String(error))}`, + message: `${phase} threw: ${message}`, + trace: runner.trace, + }; + } +} + +/** + * Delta-debugging over the action list. Keeps a reduction only when the + * failure keeps the same signature. + */ +export function minimizeScenario( + scenario: Scenario, + signature: string, + options: RunOptions = {}, + budget = 4_000, +): Scenario { + let current = scenario; + let runs = 0; + const fails = (candidate: Scenario): boolean => { + runs += 1; + return ( + runScenario(candidate, { ...options, onTrace: undefined })?.signature === signature + ); + }; + let chunk = Math.max(1, Math.floor(current.actions.length / 2)); + while (chunk >= 1 && runs < budget) { + let removed = false; + for (let start = 0; start < current.actions.length && runs < budget; ) { + const actions = [ + ...current.actions.slice(0, start), + ...current.actions.slice(start + chunk), + ]; + const candidate = { ...current, actions }; + if (actions.length < current.actions.length && fails(candidate)) { + current = candidate; + removed = true; + } else { + start += chunk; + } + } + if (!removed) chunk = Math.floor(chunk / 2); + } + // Fewer peers make the reproduction easier to read. + for (let peers = 2; peers < current.peers && runs < budget; peers += 1) { + const actions = current.actions.filter((action) => + action.kind === "sync" + ? action.from < peers && action.to < peers + : action.peer < peers, + ); + const candidate = { ...current, peers, actions }; + if (fails(candidate)) { + current = candidate; + break; + } + } + return current; +} + +export function describeScenario(scenario: Scenario): string { + return JSON.stringify({ + seed: scenario.seed, + peers: scenario.peers, + features: scenario.features, + actions: scenario.actions, + }); +} diff --git a/loro-js/tests/revived-child-events.test.ts b/loro-js/tests/revived-child-events.test.ts new file mode 100644 index 000000000..0cec087ff --- /dev/null +++ b/loro-js/tests/revived-child-events.test.ts @@ -0,0 +1,183 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc, LoroList, LoroText } from "../src/index"; +import type { ContainerID, Delta, LoroEventBatch } from "../src/index"; + +// A child container that an event attaches starts from empty for a listener, +// so, as in Rust's DocState::apply_diff, its event must carry its whole state +// ("revival") rather than a delta against the state it had while hidden. + +function peers(count: number): LoroDoc[] { + return Array.from({ length: count }, (_, index) => { + const doc = new LoroDoc(); + doc.setPeerId(index + 1); + return doc; + }); +} + +function merged(...docs: LoroDoc[]): LoroDoc { + const doc = new LoroDoc(); + for (const source of docs) doc.import(source.export({ mode: "update" })); + return doc; +} + +function eventsOf(batches: readonly LoroEventBatch[]): unknown[] { + return batches.flatMap((batch) => + batch.events.map((event) => [ + event.target, + JSON.parse(JSON.stringify(event.diff)) as unknown, + ]), + ); +} + +describe("events for child containers attached by a checkout", () => { + // The expected events are exactly what loro-crdt emits. + test("carry the full state of a map child set back from null", () => { + const [p1] = peers(1) as [LoroDoc]; + const text = p1.getMap("map").setContainer("key", new LoroText()); + text.insert(0, "😀"); + p1.getMap("map").set("key", null); + p1.commit(); + const doc = merged(p1); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + doc.checkout([{ peer: "1", counter: 1 }]); + expect(doc.toJSON()).toEqual({ map: { key: "😀" } }); + expect(eventsOf(batches)).toEqual([ + ["cid:root-map:Map", { type: "map", updated: { key: "😀" } }], + [text.id, { type: "text", diff: [{ insert: "😀" }] }], + ]); + }); + + test("carry the full state of a list child whose element is restored", () => { + const [p1] = peers(1) as [LoroDoc]; + const child = p1.getList("list").insertContainer(0, new LoroList()); + p1.commit(); + child.insert(0, 5); + p1.commit(); + p1.getList("list").delete(0, 1); + p1.commit(); + const doc = merged(p1); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + doc.checkout([{ peer: "1", counter: 1 }]); + expect(doc.toJSON()).toEqual({ list: [[5]] }); + expect(eventsOf(batches)).toEqual([ + ["cid:root-list:List", { type: "list", diff: [{ insert: [[5]] }] }], + [child.id, { type: "list", diff: [{ insert: [5] }] }], + ]); + }); + + test("replace a delta against the hidden state with the full state", () => { + const [p1, p2] = peers(2) as [LoroDoc, LoroDoc]; + const text = p1.getMap("map").setContainer("key", new LoroText()); + p1.commit(); + p2.import(p1.export({ mode: "update" })); + // Edited concurrently with the key being overwritten. + (p2.getContainerById(text.id) as LoroText).insert(0, "ab"); + p2.commit(); + p1.getMap("map").set("key", "scalar"); + p1.commit(); + const doc = merged(p1, p2); + expect(doc.toJSON()).toEqual({ map: { key: "scalar" } }); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + doc.checkout([ + { peer: "1", counter: 0 }, + { peer: "2", counter: 1 }, + ]); + expect(doc.toJSON()).toEqual({ map: { key: "ab" } }); + // Before, the text event deleted "ab" from a child the listener had just + // created empty. + expect(eventsOf(batches)).toEqual([ + ["cid:root-map:Map", { type: "map", updated: { key: "ab" } }], + [text.id, { type: "text", diff: [{ insert: "ab" }] }], + ]); + }); + + test("revive a list child that a whole-value list diff deletes and re-inserts", () => { + const [p1, p2, p3] = peers(3) as [LoroDoc, LoroDoc, LoroDoc]; + p1.getList("list").insert(0, "s"); + p1.commit(); + p2.import(p1.export({ mode: "update" })); + const text = p2.getList("list").insertContainer(1, new LoroText()); + text.insert(0, "a"); + p2.commit(); + p3.import(p1.export({ mode: "update" })); + p3.import(p2.export({ mode: "update" })); + const version = p3.frontiers(); + p1.getList("list").insert(1, true); + p1.commit(); + p3.import(p1.export({ mode: "update" })); + p3.checkout(version); + // Imported while detached, so checkoutToLatest replays and diffs values. + p2.getList("list").push(false); + p2.commit(); + p3.import(p2.export({ mode: "update", from: p3.oplogVersion() })); + + const batches: LoroEventBatch[] = []; + p3.subscribe((batch) => batches.push(batch)); + const model = new Map([ + ["cid:root-list:List", ["s", { child: text.id }]], + [text.id, "a"], + ]); + p3.checkoutToLatest(); + // Apply the events the way a listener does: an inserted child restarts + // from empty. + for (const event of batches.flatMap((batch) => batch.events)) { + if (event.diff.type === "text") { + model.set( + event.target, + applyText(model.get(event.target) as string, event.diff.diff), + ); + } else if (event.diff.type === "list") { + const next: unknown[] = []; + const current = (model.get(event.target) ?? []) as unknown[]; + let index = 0; + for (const item of event.diff.diff) { + if ("retain" in item) { + next.push(...current.slice(index, index + item.retain)); + index += item.retain; + } else if ("delete" in item) { + index += item.delete; + } else { + for (const value of item.insert) { + if (value instanceof LoroText) { + model.set(value.id, ""); + next.push({ child: value.id }); + } else { + next.push(value); + } + } + } + } + next.push(...current.slice(index)); + model.set(event.target, next); + } + } + const list = model.get("cid:root-list:List") as unknown[]; + const resolved = list.map((value) => + typeof value === "object" && value !== null && "child" in value + ? model.get((value as { child: ContainerID }).child) + : value, + ); + expect(p3.toJSON()).toEqual({ list: ["s", true, "a", false] }); + expect(resolved).toEqual(["s", true, "a", false]); + }); +}); + +function applyText(text: string, delta: readonly Delta[]): string { + let output = ""; + let cursor = 0; + for (const item of delta) { + if ("retain" in item) { + output += text.slice(cursor, cursor + item.retain); + cursor += item.retain; + } else if ("delete" in item) { + cursor += item.delete; + } else { + output += item.insert; + } + } + return output + text.slice(cursor); +} diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 2b8f3fc02..5d23e8e6b 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -349,8 +349,7 @@ describe("loro-wasm-compatible runtime", () => { const doc = new LoroDoc(); doc.setPeerId(2); doc.import(base); - const checkedOut = doc.frontiers(); - doc.checkout(checkedOut); + doc.detach(); source.getText("text").push(" latest"); doc.import(source.export({ mode: "update", from: doc.oplogVersion() })); @@ -361,6 +360,52 @@ describe("loro-wasm-compatible runtime", () => { expect(doc.getText("text").toString()).toBe("base latest"); }); + // Rust's checkout returns early when the target is the current version and + // re-attaches when that version is the latest one. + test("keeps a checkout of the current version attached like loro-crdt", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("text").insert(0, "x"); + doc.commit(); + doc.getText("text").insert(1, "y"); + doc.commit(); + const batches: LoroEventBatch[] = []; + doc.subscribe((batch) => batches.push(batch)); + + doc.checkout(doc.frontiers()); + expect(doc.isDetached()).toBe(false); + expect(batches).toHaveLength(0); + const remote = new LoroDoc(); + remote.setPeerId(2); + remote.getText("text").insert(0, "B"); + remote.commit(); + doc.import(remote.export({ mode: "update" })); + expect(doc.getText("text").toString()).toBe("xyB"); + + // A dominated extra frontier still names the current version. + doc.checkout([...doc.frontiers(), { peer: "1", counter: 0 }]); + expect(doc.isDetached()).toBe(false); + + doc.checkout([{ peer: "1", counter: 0 }]); + doc.checkout([{ peer: "1", counter: 0 }]); + expect(doc.isDetached()).toBe(true); + expect(doc.getText("text").toString()).toBe("x"); + // Checking out the latest version from an older one stays detached. + doc.checkout(doc.oplogFrontiers()); + expect(doc.isDetached()).toBe(true); + expect(doc.getText("text").toString()).toBe("xyB"); + + // detach() at the latest version, then checkout of it, re-attaches. + doc.checkoutToLatest(); + doc.detach(); + doc.checkout(doc.frontiers()); + expect(doc.isDetached()).toBe(false); + doc.getText("text").insert(0, "!"); + doc.commit(); + expect(doc.getText("text").toString()).toBe("!xyB"); + expect(doc.version().compare(doc.oplogVersion())).toBe(0); + }); + test("holds causally incomplete updates pending until dependencies arrive", () => { const source = new LoroDoc(); source.setPeerId(1); @@ -1726,22 +1771,42 @@ describe("loro-wasm-compatible runtime", () => { expect(() => root.ensureMergeableMap("regular")).toThrow(/non-mergeable value/u); }); - test("does not record semantic no-op edits", () => { + // loro-crdt skips setting a map key to its current value, but records a + // delete of an absent key and a zero counter increment. Matching it keeps op + // IDs and concurrent outcomes the same for the same API calls. + test("records the same local ops as loro-crdt for no-op-looking edits", () => { const doc = new LoroDoc(); const map = doc.getMap("map"); map.set("value", { nested: [1, true, null] }); - const movable = doc.getMovableList("movable"); - movable.push("same"); - doc.commit(); - const opCount = doc.opCount(); - - map.set("value", { nested: [1, true, null] }); - map.delete("missing"); - movable.set(0, "same"); - doc.getCounter("counter").increment(0); doc.commit(); + const count = (edit: () => void): number => { + const before = doc.opCount(); + edit(); + doc.commit(); + return doc.opCount() - before; + }; - expect(doc.opCount()).toBe(opCount); + expect(count(() => map.set("value", { nested: [1, true, null] }))).toBe(0); + expect(count(() => map.delete("missing"))).toBe(1); + expect(count(() => doc.getCounter("counter").increment(0))).toBe(1); + // Attaching a detached counter increments it by its value, even 0. + expect(count(() => map.setContainer("child", new LoroCounter()))).toBe(2); + }); + + test("lets a delete of an absent map key win against a concurrent set", () => { + const a = new LoroDoc(); + a.setPeerId(1); + const b = new LoroDoc(); + b.setPeerId(2); + a.getMap("map").set("key", 1); + a.commit(); + b.getMap("map").delete("key"); + b.commit(); + a.import(b.export({ mode: "update" })); + b.import(a.export({ mode: "update" })); + // Same lamport; the larger peer's delete wins, as in loro-crdt. + expect(a.toJSON()).toEqual({ map: {} }); + expect(b.toJSON()).toEqual({ map: {} }); }); test("resurfaces preserved state and switches mergeable kinds", () => { diff --git a/loro-js/tests/shallow-root-critical.test.ts b/loro-js/tests/shallow-root-critical.test.ts new file mode 100644 index 000000000..3c17fc04c --- /dev/null +++ b/loro-js/tests/shallow-root-critical.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc } from "../src/index"; +import type { Frontiers } from "../src/index"; + +// Ports of crates/loro/tests/shallow_root_critical.rs. A shallow snapshot keeps +// every op from its root to the latest version, so the root must be a critical +// version: every retained op is causally after it, never concurrent with it +// (loro-dev/loro#1095). The roots below are the ones loro-crdt picks. + +function shallowOf(doc: LoroDoc, frontiers: Frontiers): LoroDoc { + const shallow = new LoroDoc(); + shallow.import(doc.export({ mode: "shallow-snapshot", frontiers })); + return shallow; +} + +/** Every op of `doc` is in the root's causal past or has the root in its own. */ +function expectCritical(doc: LoroDoc, root: Frontiers): void { + if (root.length === 0) return; + const rootVersion = doc.frontiersToVV(root); + for (const [peer, end] of doc.oplogVersion().toJSON()) { + for (let counter = rootVersion.get(peer) ?? 0; counter < end; counter += 1) { + const opVersion = doc.frontiersToVV([{ peer, counter }]); + const order = rootVersion.compare(opVersion); + if (order !== -1 && order !== 0) { + throw new Error(`${counter}@${peer} is concurrent with the root`); + } + } + } +} + +describe("shallow snapshot root", () => { + test("keeps the full history for independent heads", () => { + for (let peers = 2; peers <= 4; peers += 1) { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("t").insert(0, "a"); + doc.commit(); + for (let peer = 2; peer <= peers; peer += 1) { + const other = new LoroDoc(); + other.setPeerId(peer); + other.getText("t").insert(0, "b"); + other.commit(); + doc.import(other.export({ mode: "update" })); + } + const shallow = shallowOf(doc, doc.oplogFrontiers()); + expect(shallow.shallowSinceFrontiers()).toEqual([]); + expect(shallow.toJSON()).toEqual(doc.toJSON()); + } + }); + + test("still trims to a root shared by every head", () => { + const base = new LoroDoc(); + base.setPeerId(100); + base.getText("t").insert(0, "root"); + base.commit(); + const snapshot = base.export({ mode: "snapshot" }); + for (let peer = 1; peer <= 5; peer += 1) { + const fork = new LoroDoc(); + fork.setPeerId(peer); + fork.import(snapshot); + fork.getText("t").insert(0, "x"); + fork.commit(); + base.import(fork.export({ mode: "update" })); + } + const shallow = shallowOf(base, base.oplogFrontiers()); + expect(shallow.shallowSinceFrontiers()).toEqual([{ peer: "100", counter: 3 }]); + expect(shallow.toJSON()).toEqual(base.toJSON()); + }); + + test("moves below a past version when a later branch forked below it", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("t").insert(0, "0"); + doc.commit(); + const atA0 = doc.export({ mode: "snapshot" }); + doc.getText("t").insert(0, "1"); + doc.commit(); + doc.getText("t").insert(0, "2"); + doc.commit(); + const target = doc.oplogFrontiers(); + const fork = new LoroDoc(); + fork.setPeerId(2); + fork.import(atA0); + fork.getText("t").insert(1, "B"); + fork.commit(); + doc.import(fork.export({ mode: "update" })); + + const shallow = shallowOf(doc, target); + // The requested version 2@1 is concurrent with 0@2; loro-crdt uses 0@1. + expect(shallow.shallowSinceFrontiers()).toEqual([{ peer: "1", counter: 0 }]); + expectCritical(doc, shallow.shallowSinceFrontiers()); + shallow.checkout(target); + expect(shallow.getText("t").toString()).toBe("210"); + shallow.checkoutToLatest(); + expect(shallow.getText("t").toString()).toBe("210B"); + }); + + test("keeps concurrent list order through checkouts of the retained range", () => { + const p2 = new LoroDoc(); + p2.setPeerId(2); + const p4 = new LoroDoc(); + p4.setPeerId(4); + p4.getList("l").insert(0, false); + p4.commit(); + p2.getList("l").insert(0, 45); + p2.commit(); + p2.getMap("m").set("k", 1); + p2.commit(); + p2.getMap("m").set("k", 2); + p2.commit(); + p2.import(p4.export({ mode: "update" })); + expect(p2.getList("l").toArray()).toEqual([45, false]); + + // 4@0 is concurrent with 2@1, so the root cannot be 2@1. + const shallow = shallowOf(p2, [{ peer: "2", counter: 1 }]); + expect(shallow.shallowSinceFrontiers()).toEqual([]); + for (const frontiers of [ + [ + { peer: "4", counter: 0 }, + { peer: "2", counter: 2 }, + ], + [{ peer: "2", counter: 2 }], + [{ peer: "2", counter: 1 }], + ] as Frontiers[]) { + shallow.checkout(frontiers); + const full = new LoroDoc(); + full.import(p2.export({ mode: "update" })); + full.checkout(frontiers); + expect(shallow.toJSON()).toEqual(full.toJSON()); + } + shallow.checkoutToLatest(); + // Before the fix the retained concurrent insert came back as [false, 45]. + expect(shallow.getList("l").toArray()).toEqual([45, false]); + }); + + test("is critical on random multi-peer histories", () => { + let random = 0x2c1b_3c6d; + const next = (limit: number): number => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return (random >>> 0) % limit; + }; + for (let round = 0; round < 40; round += 1) { + const docs = [1, 2, 3].map((peer) => { + const doc = new LoroDoc(); + doc.setPeerId(peer); + return doc; + }); + const targets: Frontiers[] = []; + for (let step = 0; step < 24; step += 1) { + const doc = docs[next(docs.length)]!; + if (next(4) === 0) { + const other = docs[next(docs.length)]!; + if (other !== doc) doc.import(other.export({ mode: "update" })); + } else { + const text = doc.getText("t"); + text.insert(next(text.length + 1), String.fromCharCode(97 + next(26))); + doc.commit(); + if (next(3) === 0) targets.push(doc.oplogFrontiers()); + } + } + const doc = docs[0]!; + for (const other of docs.slice(1)) doc.import(other.export({ mode: "update" })); + for (const target of targets) { + const shallow = shallowOf(doc, target); + const root = shallow.shallowSinceFrontiers(); + expectCritical(doc, root); + expect(shallow.toJSON()).toEqual(doc.toJSON()); + const rootVersion = doc.frontiersToVV(root); + for (const version of targets) { + const order = rootVersion.compare(doc.frontiersToVV(version)); + if (order !== -1 && order !== 0) continue; + const full = new LoroDoc(); + full.import(doc.export({ mode: "update" })); + full.checkout(version); + shallow.checkout(version); + expect(shallow.toJSON()).toEqual(full.toJSON()); + } + shallow.checkoutToLatest(); + expect(shallow.toJSON()).toEqual(doc.toJSON()); + } + } + }); +}); diff --git a/loro-js/tests/tree-local-positions.test.ts b/loro-js/tests/tree-local-positions.test.ts new file mode 100644 index 000000000..e3164cf93 --- /dev/null +++ b/loro-js/tests/tree-local-positions.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "vitest"; + +import { LoroDoc } from "../src/index"; +import type { LoroTree } from "../src/index"; + +// Expected values are what loro-crdt returns for the same calls. + +function shape(tree: LoroTree): string[] { + return tree.toJSON().map((node) => `${node.id}:${node.fractional_index}`); +} + +/** Two roots created concurrently at index 0 share the position 80. */ +function equalPositions(): LoroDoc { + const p1 = new LoroDoc(); + p1.setPeerId(1); + const p2 = new LoroDoc(); + p2.setPeerId(2); + p1.getTree("tree").createNode(); + p1.commit(); + p2.getTree("tree").createNode(); + p2.commit(); + p1.import(p2.export({ mode: "update" })); + expect(shape(p1.getTree("tree"))).toEqual(["0@1:80", "0@2:80"]); + return p1; +} + +describe("local tree positions", () => { + // Rust's generate_fi_at: there is no index between two equal positions, so + // the right neighbor is moved to a new position in the same transaction. + test("rearranges equal neighbors when creating a node between them", () => { + const doc = equalPositions(); + const tree = doc.getTree("tree"); + const before = doc.opCount(); + const node = tree.createNode(undefined, 1); + doc.commit(); + expect(node.id).toBe("1@1"); + expect(doc.opCount() - before).toBe(2); + expect(shape(tree)).toEqual(["0@1:80", "1@1:817F80", "0@2:8180"]); + + const replay = new LoroDoc(); + replay.import(doc.export({ mode: "update" })); + expect(shape(replay.getTree("tree"))).toEqual(shape(tree)); + }); + + test("rearranges equal neighbors when moving a node between them", () => { + const doc = equalPositions(); + const tree = doc.getTree("tree"); + const extra = tree.createNode(); + doc.commit(); + const before = doc.opCount(); + tree.move(extra.id, undefined, 1); + doc.commit(); + expect(doc.opCount() - before).toBe(2); + expect(shape(tree)).toEqual(["0@1:80", "1@1:817F80", "0@2:8180"]); + }); + + test("records nothing for a move to the current position", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const a = tree.createNode(); + const b = tree.createNode(); + const c = tree.createNode(); + doc.commit(); + const count = (edit: () => void): number => { + const before = doc.opCount(); + edit(); + doc.commit(); + return doc.opCount() - before; + }; + + expect(count(() => tree.move(a.id, undefined, 0))).toBe(0); + // Without an index the node goes last; it already is. + expect(count(() => tree.move(c.id, undefined))).toBe(0); + expect(count(() => c.move(undefined))).toBe(0); + expect(count(() => b.moveAfter(a))).toBe(0); + expect(count(() => a.moveBefore(b))).toBe(0); + expect(tree.toJSON().map((node) => node.id)).toEqual([a.id, b.id, c.id]); + }); + + // Rust's mov_after/mov_before count the index without the moved node. + test("moves after and before a sibling in the same parent", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const a = tree.createNode(); + const b = tree.createNode(); + const c = tree.createNode(); + doc.commit(); + + a.moveAfter(c); + doc.commit(); + expect(tree.toJSON().map((node) => node.id)).toEqual([b.id, c.id, a.id]); + a.moveBefore(b); + doc.commit(); + expect(tree.toJSON().map((node) => node.id)).toEqual([a.id, b.id, c.id]); + b.moveAfter(c); + doc.commit(); + expect(tree.toJSON().map((node) => node.id)).toEqual([a.id, c.id, b.id]); + }); + + test("moves within a parent in time independent of the sibling count", () => { + /** Best of three runs of 1,000 same-parent moves, half of them no-ops. */ + function movesMs(siblings: number): number { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const nodes = Array.from({ length: siblings }, () => tree.createNode()); + doc.commit(); + let best = Infinity; + for (let run = 0; run < 3; run += 1) { + const started = performance.now(); + for (let step = 0; step < 500; step += 1) { + const node = nodes[(step * 7_919) % siblings]!; + tree.move(node.id, undefined, node.index()!); + tree.move(node.id, undefined, 0); + } + doc.commit(); + best = Math.min(best, performance.now() - started); + } + return best; + } + // Counting the siblings by listing them made each move linear in their + // number (16x the siblings took about 16x the time). + const ratio = movesMs(16_000) / movesMs(1_000); + expect(ratio).toBeLessThan(4); + }, 60_000); + + // Rust's is_ancestor_of treats a deleted node's parent as the deleted root. + test("moves a node under its own deleted child", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const parent = tree.createNode(); + const child = tree.createNode(parent.id); + doc.commit(); + tree.delete(child.id); + doc.commit(); + const before = doc.opCount(); + tree.move(parent.id, child.id, 0); + doc.commit(); + expect(doc.opCount() - before).toBe(1); + expect(tree.toJSON()).toEqual([]); + expect(() => tree.move(child.id, child.id, 0)).toThrow(/below itself/); + }); +});