From 7ebbd0ef2c1b0fda54da30a2dea677b5d93bb1d6 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 02:07:20 +0800 Subject: [PATCH 01/20] test(loro.js): add a Rust differential suite for MovableList Drive the Rust WASM build and loro.js with the same random multi-peer edits, crossing engines on every sync, and require equal values, versions, per-element metadata and events. Profiles that diverge on the current MovableList model are recorded as known divergences. Design and findings: context/loro-js-movable-list.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/loro-js-differential-suite.md | 4 + .github/workflows/loro-js.yml | 53 ++ AGENTS.md | 3 + context/loro-js-movable-list.md | 223 +++++ context/loro-js-performance.md | 5 +- loro-js/package.json | 2 + loro-js/scripts/build-reference.mjs | 27 + loro-js/tests/differential/engine.ts | 113 +++ loro-js/tests/differential/fuzz.ts | 766 ++++++++++++++++++ loro-js/tests/differential/harness.ts | 480 +++++++++++ .../tests/differential/movable-list.test.ts | 166 ++++ loro-js/tests/differential/reference.ts | 32 + loro-js/vite.config.ts | 11 +- 13 files changed, 1883 insertions(+), 2 deletions(-) create mode 100644 .changeset/loro-js-differential-suite.md create mode 100644 .github/workflows/loro-js.yml create mode 100644 context/loro-js-movable-list.md create mode 100644 loro-js/scripts/build-reference.mjs create mode 100644 loro-js/tests/differential/engine.ts create mode 100644 loro-js/tests/differential/fuzz.ts create mode 100644 loro-js/tests/differential/harness.ts create mode 100644 loro-js/tests/differential/movable-list.test.ts create mode 100644 loro-js/tests/differential/reference.ts diff --git a/.changeset/loro-js-differential-suite.md b/.changeset/loro-js-differential-suite.md new file mode 100644 index 000000000..ac05280fe --- /dev/null +++ b/.changeset/loro-js-differential-suite.md @@ -0,0 +1,4 @@ +--- +--- + +Add a differential test suite that compares loro.js with the Rust implementation (`pnpm --dir loro-js test:differential`). Tests and scripts only; the published package does not change. diff --git a/.github/workflows/loro-js.yml b/.github/workflows/loro-js.yml new file mode 100644 index 000000000..55a1c6082 --- /dev/null +++ b/.github/workflows/loro-js.yml @@ -0,0 +1,53 @@ +name: loro.js + +on: + push: + branches: ["main"] + paths: + - "loro-js/**" + - "crates/**" + - "pnpm-lock.yaml" + - ".github/workflows/loro-js.yml" + pull_request: + branches: ["main"] + paths: + - "loro-js/**" + - "crates/**" + - "pnpm-lock.yaml" + - ".github/workflows/loro-js.yml" + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + - run: pnpm install --frozen-lockfile + - run: pnpm --dir loro-js check + + differential: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + - uses: denoland/setup-deno@v1 + with: + deno-version: v2.x + - run: rustup target add wasm32-unknown-unknown + - uses: jetli/wasm-bindgen-action@v0.2.0 + with: + version: "0.2.100" + - uses: Swatinem/rust-cache@v2 + - run: pnpm install --frozen-lockfile + - run: pnpm --dir loro-js build:reference + - run: pnpm --dir loro-js test:differential diff --git a/AGENTS.md b/AGENTS.md index cb00f0dd4..1e9a77db0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,9 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [skills/loro/SKILL.md](skills/loro/SKILL.md). - Pure TypeScript runtime indexes, complexity contracts, benchmarks, and remaining gaps: [context/loro-js-performance.md](context/loro-js-performance.md). +- loro.js MovableList model, Rust differential suite (`pnpm --dir loro-js + test:differential`), and known loro.js/Rust divergences: + [context/loro-js-movable-list.md](context/loro-js-movable-list.md). - WASM panic/OOM reporting channels, `__wbindgen_start` glue invariant, and trap-testing recipes: [context/wasm-error-reporting.md](context/wasm-error-reporting.md). diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md new file mode 100644 index 000000000..33be8f520 --- /dev/null +++ b/context/loro-js-movable-list.md @@ -0,0 +1,223 @@ +# loro.js MovableList: Rust-Compatible Model + +Verified against code 2026-09-28. + +`loro-js/` must converge with the Rust implementation and read and write its +encoding. This article states the Rust MovableList semantics that loro.js must +reproduce, the loro.js model that implements them, and how the differential +suite checks the two against each other. Complexity rules still come from +[loro-js-performance.md](loro-js-performance.md). + +Status: the differential suite is in the tree. The "loro.js model" section is +the design of the follow-up change; until it lands, `LoroMovableList` still +uses the old model described under "Why the old loro.js model diverged", and +the suite records those profiles in `KNOWN_DIVERGENT`. + +## Rust semantics (the contract) + +A MovableList is two CRDTs glued together +([`movable_list_state.rs`](../crates/loro-internal/src/state/movable_list_state.rs)): + +- **List items (positions)** form a Fugue sequence. Every `Insert` op creates + one position per value (ID = op ID + offset). Every `Move` op creates one new + position (ID = the move op ID). Positions are deleted by `Delete` ops and by + the source side of a `Move`. The diff calculator's tracker treats a move as + "delete the item at `from`, then insert at `to` in the view after that + delete" (`Tracker::move_item` in + [`tracker.rs`](../crates/loro-internal/src/container/richtext/tracker.rs)), + and it deletes **by index in the op's causal view**, not by ID + (`CrdtRope::delete`). +- **Elements** are keyed by `IdLp` of their `Insert`. Each element has + - `pos`: the winning position, the LWW maximum by `IdLp` = `(lamport, peer)` + over its insert position and every move position that targets it; + - `value`/`value_id`: the LWW maximum over the insert value and every `Set`. +- A position is **pointed** when its element's winning `pos` is that position. + User-visible items are exactly the positions that are alive in the tracker + and pointed. An alive, unpointed position is a *dead* list item: it still + counts in op indices (`IndexType::ForOp`) but not in user indices + (`IndexType::ForUser`). +- An element is visible iff its winning position is alive. Deleting that + position hides the element. A concurrent move with a greater `IdLp` creates a + new alive position and revives it. This is why concurrent delete + move gives + `["a","c","B"]` in `concurrent_delete_and_move_resurrects_moved_element`. +- Checkout uses the same rule per version + (`MovableListHistoryCache::last_pos` / `last_value` in + [`history_cache.rs`](../crates/loro-internal/src/history_cache.rs)). + On a shallow document, `record_shallow_root_state` seeds each root element's + position and value as a candidate for versions at or after the root. +- Local ops (`MovableListHandler` in + [`handler.rs`](../crates/loro-internal/src/handler.rs)) encode **op + indices**: `insert` at `convert_index(pos, ForUser, ForOp)` (the list end is + the op length, after trailing dead items); `move(from, to)` stores the op + indices of the element and of the current user item at `to`; `delete` + emits one `Delete(position id, op index, 1)` per element, adjusting the op + index for the elements already deleted by the same call. +- `getCreatorAt` = element ID peer, `getLastMoverAt` = winning position peer, + `getLastEditorAt` = `value_id` peer. `getCursor` anchors the **position** ID. +- `apply_delta` for a MovableList + ([`movable_list_apply_delta.rs`](../crates/loro-internal/src/handler/movable_list_apply_delta.rs)) + is used by `revertTo`, `applyDiff` and undo. It inserts before it deletes. + A child container that the delta deletes and reinserts becomes a `move`. + Plain values are reinserted as new elements. +- Snapshot state (normative: [docs/encoding-container-states.md](../docs/encoding-container-states.md) + section 8) lists every alive position in order, dead ones included, plus + element IDs and last-set IDs when they differ from the position/element ID. + +Invalid `Move`/`Set` handling (Rust #1125, open when this was written) is in +[movable-list-op-validation.md](movable-list-op-validation.md) once merged; see +the "Validation" section below for the loro.js side. + +## Why the old loro.js model diverged + +loro.js used to store MovableList elements directly in the Fugue sequence and +physically moved them (`SequenceIndex.moveVisible`). It had no separate +positions, so it could not represent a move of a concurrently deleted element, +a delete that targets a position an element has already left, or a dead list +item. Concretely: + +1. concurrent delete + move lost the element (`["a","c"]` instead of + `["a","c","B"]`); +2. importing Rust's "move, then delete" left the element (`["b","a"]` instead + of `["b"]`), because the delete targeted the old position ID; +3. snapshots wrote every position ID as the element ID and every last-set ID + as the element ID, and ignored dead items on read; +4. `getLastEditorAt`/`getLastMoverAt` returned the creator; +5. checkout/diff/`revertTo` replayed or LIS-reordered elements instead of + selecting winners, and shallow retreat had no root-time fallback; +6. undo ignored `move`/`set`, and `applyDiff` treated a MovableList like a List. + +## loro.js model + +`loro-js/src/runtime/movable-list.ts` owns the state; `LoroMovableList` +(`containers.ts`) delegates to it. `LoroMovableList` still extends `LoroList` +for API compatibility. Its inherited element sequence is unused, and reading +`_sequence` on it throws, so a code path that forgets the MovableList branch +fails loudly. + +- `positions: SequenceIndex`: the Fugue sequence of positions. + `deleted` means "not alive in the tracker"; deletion records (`deletedBy`) + hold the delete or move op that removed it, which gives causal views and + version transitions for free. A position's `utf16`/`utf8` metrics are 1 when + it is pointed and 0 otherwise, so the user-visible count is the sequence's + visible metric and user ↔ op index conversion is O(log n) through the + existing metric queries. `SequenceIndex.refreshMetrics` recomputes one + element's node and its ancestors when pointedness changes. +- `elements: Map>` keyed by element `IdLp`. + Each element keeps its current winner position, value and value writer, plus + history arrays sorted by `IdLp` (positions it may point to, and value + candidates). Winner selection at a version scans candidates from the + greatest `IdLp` down until one is included, which is the same work as Rust's + `BTreeSet` range scan. +- Remote ops are applied in causal order: + - `Insert`: Fugue insertion at the op index in the op's causal view, then new + elements. + - `Delete`: deletes the target position IDs. Rust deletes by index; the two + agree for honest ops. + - `Move`: deletes the position at `from` in the causal view (faithful to Rust, + including a forged `from`), inserts the new position at `to` in the causal + view extended by the move itself, then runs LWW on the element's position. + - `Set`: LWW on the element's value. +- Local ops compute op indices exactly as `MovableListHandler` does, so encoded + ops are interchangeable with Rust's. + +### Versions, events, snapshots + +- Version transitions (checkout, `diff`, `revertTo`) toggle only the positions + created or deleted by the retreated/forwarded ops. They then recompute the + winners of the touched elements at the target version and refresh the + pointedness of their old and new positions. That costs + O((affected ops + affected candidates) · log n), with no replay and no LIS. +- Events come from the same primitive state changes. A position that becomes + visible inserts its value at its new user index. A visible position that + stops being visible deletes at its old user index. A value change on a + visible element is a delete plus an insert at the same index. Transitions + delete in descending before-index order, then insert in ascending after-index + order. +- Snapshot export follows section 8 exactly. Import hydrates alive positions + (visible and dead) and visible elements with their real position, element + and last-set IDs. Hydrated state has no tombstones, origins or candidate + history, so the container is marked incomplete. Version transitions on it + replay history once. An import that is not causally after the hydrated + version (it is concurrent with it) also replays history first: the causal + view of such an op can need tombstones the snapshot dropped. The same rule + fixes List and Text, which had the same gap. +- A shallow root store seeds root-time position and value candidates, the + counterpart of `record_shallow_root_state`, so retreat to the root keeps + root-time winners (the MovableList part of Rust #1124 / loro.js #1127). + +### Undo, `revertTo`, `applyDiff` + +`applyDiff` and `revertTo` port Rust's MovableList `apply_delta`: insert +first, defer deletes, and turn a reinserted child container into a `move`. +Undo of MovableList ops applies the container's inverse diff through the same +function, so undoing a `move`/`set` creates a new element with the old value, +as in Rust. loro.js undo still differs from Rust when remote changes +interleave with the undone change (Rust transforms stored diffs; loro.js +replays ops). That applies to List too and is listed under remaining gaps. + +## Validation and import atomicity + +Import validation follows Rust #1125. A `Move`/`Set` whose element is unknown, +lives in another container, is outside the op's causal history, or (on a +shallow doc) was deleted before the root is rejected with an error. So is a +`from`/`to` outside the op-index range of its causal view. A `Move`/`Set` of a +deleted element is applied with the concurrent-op semantics above. +`import`/`importBatch` stage the document state (history indexes, containers, +pending changes, version, subscribers' events, undo stacks) and restore it +when any blob fails, so a rejected import is invisible +([import-batch-atomicity.md](import-batch-atomicity.md) states the Rust +contract). + +## Differential suite + +`loro-js/tests/differential/` drives the Rust WASM build (`loro-crdt` Node +target) and loro.js with the same random multi-peer operations and requires +identical values, versions, frontiers, reachable container IDs, +`getCreatorAt`/`getLastEditorAt`/`getLastMoverAt`, and canonicalized events +after every step. Sync crosses engines, so each engine also decodes the +other's updates and snapshots. + +```sh +pnpm --dir loro-js build:reference # dev WASM build of crates/loro-wasm (nodejs) +pnpm --dir loro-js test:differential # CI-sized seeds +LORO_JS_DIFF_SEEDS=500 LORO_JS_DIFF_STEPS=200 pnpm --dir loro-js test:differential +``` + +`LORO_JS_REFERENCE=/path/to/nodejs/index.js` compares against another Rust +build, for example an unmerged branch. The suite folds away a few differences +that are unrelated to the data model. Each is named in `harness.ts` +(`canonicalBatch`) and in the test file's profile notes, so a fix can remove +its normalization. + +Files: `engine.ts` (the shared API surface), `harness.ts` (twin peers, +canonical events, comparisons), `fuzz.ts` (action generation, transport, +replay and minimization), `movable-list.test.ts` (profiles). A divergence +saves its trace to the temp directory; replay it with +`LORO_JS_DIFF_REPLAY=` and add `LORO_JS_DIFF_MINIMIZE=1` to shrink it. +The suite is excluded from `pnpm test` because it needs the WASM build; CI +runs it in `.github/workflows/loro-js.yml`. + +## Remaining known divergences (not MovableList-specific) + +Found by the differential suite on 2026-09-28. The profiles avoid or normalize +them (see the comments in `harness.ts` and `fuzz.ts`): + +- Undo: loro.js replays inverse ops, while Rust applies stored inverse diffs + transformed by later changes. Results differ for List and MovableList when an + undo targets items that an earlier undo recreated, or when remote changes + interleave. A redo also restores deleted items in deletion order instead of + document order. `UndoManager.undo()` picks its item before committing the + pending transaction. Rust commits redo with origin `"undo"`. The profiles run + undo on one peer, never chained, after an explicit commit. +- `checkout(frontiers)` at the latest frontiers leaves loro.js detached; Rust + stays attached. The profiles skip such checkouts. +- loro.js emits events and `diff()` entries for child containers whose parent + element is already deleted; Rust does not. Only reachable targets are compared. +- A snapshot-hydrated loro.js document has no tombstones or origins. It + mispositions List/Text/MovableList ops that are concurrent with deletes the + snapshot already contains. +- Some shallow-snapshot imports that Rust accepts are rejected by loro.js with + "cannot import updates that depend on an outdated version". +- Multi-blob `importBatch` events are labeled `by: "checkout"` by Rust and + `"import"` by loro.js; checkout events carry origin `"checkout"` only in + Rust. Rust's `oplogVersion()` also counts the pending transaction. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 896482166..a5ed421b6 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -325,4 +325,7 @@ The remaining differences are representation and JavaScript constant factors: Keep randomized index-invariant coverage in `loro-js/tests/indexes.test.ts` and Rust/TypeScript fixture coverage in `loro-js/tests/rust-interop.test.ts` when -changing these structures. +changing these structures. The randomized Rust differential suite +(`pnpm --dir loro-js test:differential`, see +[loro-js-movable-list.md](loro-js-movable-list.md)) checks convergence, events and +encoding interchange against a WASM build of the Rust implementation. diff --git a/loro-js/package.json b/loro-js/package.json index 2785b414c..4641fe84c 100644 --- a/loro-js/package.json +++ b/loro-js/package.json @@ -50,6 +50,7 @@ "bench:subscribers": "pnpm build && node --expose-gc benchmarks/container-subscriber.mjs", "bench:text-buffer": "pnpm build && node --expose-gc benchmarks/text-buffer.mjs", "build": "vp pack", + "build:reference": "node scripts/build-reference.mjs", "check": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm test && pnpm build", "clean": "rm -rf dist", "format": "vp fmt", @@ -57,6 +58,7 @@ "fixtures:rewrite": "pnpm build && node scripts/rewrite-rust-fixture.mjs", "lint": "vp lint", "test": "vp test run", + "test:differential": "LORO_JS_DIFFERENTIAL=1 vp test run", "typecheck": "tsgo -p tsconfig.json --noEmit" }, "devDependencies": { diff --git a/loro-js/scripts/build-reference.mjs b/loro-js/scripts/build-reference.mjs new file mode 100644 index 000000000..cc0742398 --- /dev/null +++ b/loro-js/scripts/build-reference.mjs @@ -0,0 +1,27 @@ +// Builds the Node target of `crates/loro-wasm` (the `loro-crdt` package) in the +// dev profile, for `pnpm test:differential`. This is the subset of +// `pnpm -C crates/loro-wasm build-dev` that the differential suite needs: one +// target, no bundler/browser outputs, and no package tests. +import { spawnSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const wasmDir = fileURLToPath(new URL("../../crates/loro-wasm/", import.meta.url)); + +function run(command, args) { + const result = spawnSync(command, args, { cwd: wasmDir, stdio: "inherit" }); + if (result.status !== 0) { + throw new Error(`${command} ${args.join(" ")} exited with ${result.status}`); + } +} + +run("deno", ["run", "-A", "./scripts/build.ts", "dev", "nodejs"]); +run("pnpm", ["exec", "rollup", "-c"]); + +// Same rewrite as `crates/loro-wasm/scripts/post-rollup.ts` for the Node target. +const indexPath = new URL("nodejs/index.js", `file://${wasmDir}`); +const index = readFileSync(indexPath, "utf8").replace( + /require\(["']loro-wasm["']\)/g, + 'require("./loro_wasm")', +); +writeFileSync(indexPath, index); diff --git a/loro-js/tests/differential/engine.ts b/loro-js/tests/differential/engine.ts new file mode 100644 index 000000000..382eb87e1 --- /dev/null +++ b/loro-js/tests/differential/engine.ts @@ -0,0 +1,113 @@ +/** + * The subset of the `loro-crdt` API that the differential tests drive. Both the + * Rust WASM build and loro.js implement it, so one driver runs against both. + */ + +export interface OpIdLike { + readonly peer: string; + readonly counter: number; +} + +export type FrontiersLike = OpIdLike[]; + +export interface VersionVectorLike { + toJSON(): Map; +} + +export interface ContainerLike { + readonly id: string; + kind(): string; + isAttached(): boolean; + getShallowValue(): unknown; + toJSON(): unknown; +} + +export interface SequenceLike extends ContainerLike { + readonly length: number; + insert(pos: number, value: unknown): void; + delete(pos: number, len: number): void; + insertContainer(pos: number, child: ContainerLike): ContainerLike; +} + +export interface MovableListLike extends SequenceLike { + move(from: number, to: number): void; + set(pos: number, value: unknown): void; + setContainer(pos: number, child: ContainerLike): ContainerLike; + getCreatorAt(pos: number): string | undefined; + getLastEditorAt(pos: number): string | undefined; + getLastMoverAt(pos: number): string | undefined; + getCursor(pos: number, side?: -1 | 0 | 1): CursorLike | undefined; +} + +export interface MapLike extends ContainerLike { + set(key: string, value: unknown): void; + delete(key: string): void; + setContainer(key: string, child: ContainerLike): ContainerLike; +} + +export interface TextLike extends ContainerLike { + readonly length: number; + insert(pos: number, text: string): void; + delete(pos: number, len: number): void; +} + +export interface CursorLike { + encode(): Uint8Array; +} + +export interface EventLike { + readonly target: string; + readonly path: readonly (string | number)[]; + readonly diff: unknown; +} + +export interface EventBatchLike { + readonly by: string; + readonly origin?: string | undefined; + readonly events: readonly EventLike[]; +} + +export interface DocLike { + setPeerId(peer: number | bigint | string): void; + getMovableList(name: string): MovableListLike; + getMap(name: string): MapLike; + getContainerById(id: string): ContainerLike | undefined; + commit(options?: { readonly origin?: string; readonly message?: string }): void; + getPendingTxnLength(): number; + export(mode: unknown): Uint8Array; + import(bytes: Uint8Array): unknown; + importBatch(bytes: Uint8Array[]): unknown; + toJSON(): unknown; + getShallowValue(): Record; + oplogVersion(): VersionVectorLike; + frontiers(): FrontiersLike; + oplogFrontiers(): FrontiersLike; + checkout(frontiers: FrontiersLike): void; + checkoutToLatest(): void; + isDetached(): boolean; + revertTo(frontiers: FrontiersLike): void; + diff(from: FrontiersLike, to: FrontiersLike, forJson: boolean): [string, unknown][]; + subscribe(listener: (event: EventBatchLike) => void): () => void; + isShallow(): boolean; + shallowSinceVV(): VersionVectorLike; + getCursorPos(cursor: CursorLike): { readonly offset: number }; + forkAt(frontiers: FrontiersLike): DocLike; +} + +export interface UndoManagerLike { + undo(): boolean; + redo(): boolean; +} + +export interface EngineModule { + readonly LoroDoc: new () => DocLike; + readonly LoroMap: new () => MapLike; + readonly LoroText: new () => TextLike; + readonly LoroMovableList: new () => MovableListLike; + readonly VersionVector: new (input: Map) => VersionVectorLike; + readonly UndoManager: new ( + doc: DocLike, + config: { readonly mergeInterval?: number; readonly maxUndoSteps?: number }, + ) => UndoManagerLike; + readonly Cursor: { decode(bytes: Uint8Array): CursorLike }; +} diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts new file mode 100644 index 000000000..6796431b5 --- /dev/null +++ b/loro-js/tests/differential/fuzz.ts @@ -0,0 +1,766 @@ +import { writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; + +import type { ContainerLike, DocLike, EngineModule, FrontiersLike } from "./engine"; +import { + ENGINES, + Divergence, + Rng, + Twin, + assertTwinAgrees, + canonicalDiff, + describeError, + mapContainer, + movableList, + plain, + reachableContainers, + sortedFrontiers, + stringify, + textContainer, + vvToMap, + type EngineName, +} from "./harness"; + +export type ChildKind = "Map" | "Text" | "MovableList"; +export type SyncMode = "update" | "range" | "snapshot" | "batch"; + +export type Action = + | { + readonly kind: "insert"; + readonly peer: number; + readonly target: string; + readonly pos: number; + readonly value: string | number; + } + | { + readonly kind: "insertContainer"; + readonly peer: number; + readonly target: string; + readonly pos: number; + readonly child: ChildKind; + } + | { + readonly kind: "delete"; + readonly peer: number; + readonly target: string; + readonly pos: number; + readonly len: number; + } + | { + readonly kind: "move"; + readonly peer: number; + readonly target: string; + readonly from: number; + readonly to: number; + } + | { + readonly kind: "set"; + readonly peer: number; + readonly target: string; + readonly pos: number; + readonly value: string | number; + } + | { + readonly kind: "setContainer"; + readonly peer: number; + readonly target: string; + readonly pos: number; + readonly child: ChildKind; + } + | { + readonly kind: "mapSet"; + readonly peer: number; + readonly target: string; + readonly key: string; + readonly value: string | number; + } + | { + readonly kind: "textInsert"; + readonly peer: number; + readonly target: string; + readonly pos: number; + readonly text: string; + } + | { readonly kind: "commit"; readonly peer: number } + | { + readonly kind: "sync"; + readonly from: number; + readonly to: number; + readonly mode: SyncMode; + readonly cross: boolean; + readonly via?: number; + } + | { + readonly kind: "join"; + readonly from: number; + readonly peer: number; + readonly shallow: boolean; + readonly at: number; + readonly cross: boolean; + } + | { readonly kind: "checkout"; readonly peer: number; readonly at: number } + | { readonly kind: "revert"; readonly peer: number; readonly at: number } + | { + readonly kind: "diff"; + readonly peer: number; + readonly from: number; + readonly to: number; + } + | { readonly kind: "undo"; readonly peer: number } + | { readonly kind: "redo"; readonly peer: number }; + +export interface FuzzProfile { + readonly name: string; + readonly peers: number; + /** Allow move/set/setContainer on movable lists. */ + readonly moves: boolean; + /** Allow nested child containers. */ + readonly children: boolean; + readonly undo: boolean; + readonly history: boolean; + readonly snapshots: boolean; + readonly shallow: boolean; + readonly crossEngine: boolean; + readonly metadata: boolean; + readonly events: boolean; +} + +export const ROOT_LISTS = ["cid:root-list:MovableList", "cid:root-list2:MovableList"]; + +export class FuzzRun { + readonly rng: Rng; + readonly twins = new Map(); + readonly log: Action[] = []; + #nextValue = 0; + /** Last action kind per peer, to keep undo/redo unchained (see `generate`). */ + readonly #lastKind = new Map(); + + constructor( + readonly engines: Record, + readonly profile: FuzzProfile, + readonly seed: number, + ) { + this.rng = new Rng(seed); + for (let peer = 1; peer <= profile.peers; peer += 1) { + this.twins.set( + peer, + new Twin(engines, `peer${peer}`, peer, undefined, { undo: profile.undo }), + ); + } + } + + run(steps: number): void { + for (let step = 0; step < steps; step += 1) { + const action = this.generate(); + this.log.push(action); + try { + this.apply(action); + this.compareAll(`after step ${step} (${action.kind})`); + } catch (error) { + throw new Divergence( + `seed ${this.seed} profile ${this.profile.name} step ${step}: ${describeError(error)}\n` + + `actions:\n${this.log.map((item) => ` ${stringify(item)}`).join("\n")}`, + ); + } + } + for (const action of this.finalSync()) { + this.log.push(action); + try { + this.apply(action); + this.compareAll(`after final ${action.kind}`); + } catch (error) { + throw new Divergence( + `seed ${this.seed} profile ${this.profile.name} final sync: ${describeError(error)}\n` + + `actions:\n${this.log.map((item) => ` ${stringify(item)}`).join("\n")}`, + ); + } + } + this.assertConverged(); + } + + replay(actions: readonly Action[]): void { + for (const [step, action] of actions.entries()) { + this.log.push(action); + this.apply(action); + this.compareAll(`after replayed step ${step} (${action.kind})`); + } + } + + compareAll(context: string): void { + for (const twin of this.twins.values()) { + assertTwinAgrees(twin, context, { + metadata: this.profile.metadata, + events: this.profile.events, + }); + } + } + + /** Full-history peers that exchanged everything must hold the same state. */ + assertConverged(): void { + const full = [...this.twins.values()].filter((twin) => !twin.docs.rust.isShallow()); + const first = full[0]; + if (first === undefined) return; + const expected = stringify(plain(first.docs.rust.toJSON())); + for (const twin of full.slice(1)) { + const version = vvToMap(twin.docs.rust.oplogVersion()); + if (!isDeepStrictEqual(version, vvToMap(first.docs.rust.oplogVersion()))) continue; + const actual = stringify(plain(twin.docs.rust.toJSON())); + if (actual !== expected) { + throw new Divergence( + `seed ${this.seed}: Rust peers with equal versions diverged (${first.name} vs ${twin.name})`, + ); + } + } + } + + *finalSync(): Generator { + const peers = [...this.twins.keys()].filter( + (peer) => !this.twins.get(peer)!.docs.rust.isShallow(), + ); + for (const from of peers) { + for (const to of peers) { + if (from === to) continue; + yield { kind: "sync", from, to, mode: "update", cross: this.profile.crossEngine }; + } + } + } + + generate(): Action { + const action = this.generateAction(); + this.#lastKind.set("peer" in action ? action.peer : action.to, action.kind); + return action; + } + + generateAction(): Action { + const rng = this.rng; + const peers = [...this.twins.keys()]; + const peer = rng.pick(peers); + const twin = this.twins.get(peer)!; + const profile = this.profile; + // loro.js undo replays ops while Rust applies transformed inverse diffs, so + // an undo whose target was recreated by an earlier undo diverges for List + // too. Undo only right after an edit, and redo only right after an undo. + const last = this.#lastKind.get(peer); + const canUndo = profile.undo && last !== "undo" && last !== "redo"; + const canRedo = profile.undo && last === "undo"; + const kind = rng.weighted({ + edit: 12, + commit: 3, + ...(peers.length > 1 ? { sync: 4 } : {}), + ...(canUndo ? { undo: 2 } : {}), + ...(canRedo ? { redo: 2 } : {}), + ...(profile.history ? { checkout: 0.8, revert: 0.4, diff: 0.6 } : {}), + ...(profile.snapshots && peers.length > 1 && this.twins.size < 5 + ? { join: 0.3 } + : {}), + }); + switch (kind) { + case "edit": + return this.generateEdit(peer, twin); + case "commit": + case "undo": + case "redo": + return { kind, peer }; + case "sync": { + const from = rng.pick(peers.filter((item) => item !== peer)) ?? peer; + const mode = rng.weighted({ + update: 4, + range: 1, + ...(profile.snapshots ? { snapshot: 1 } : {}), + batch: 1, + }); + const others = peers.filter((item) => item !== peer && item !== from); + return { + kind: "sync", + from, + to: peer, + mode, + cross: profile.crossEngine && rng.bool(0.6), + ...(mode === "batch" && others.length > 0 ? { via: rng.pick(others) } : {}), + }; + } + case "join": { + const joined = Math.max(...peers) + 1; + return { + kind: "join", + from: peer, + peer: joined, + shallow: profile.shallow && rng.bool(0.5), + at: twin.history.length === 0 ? -1 : rng.int(twin.history.length + 1) - 1, + cross: profile.crossEngine && rng.bool(0.6), + }; + } + case "checkout": + case "revert": + return { + kind, + peer, + at: twin.history.length === 0 ? -1 : rng.int(twin.history.length), + }; + case "diff": + return { + kind: "diff", + peer, + from: twin.history.length === 0 ? -1 : rng.int(twin.history.length + 1) - 1, + to: twin.history.length === 0 ? -1 : rng.int(twin.history.length + 1) - 1, + }; + default: + throw new Error(`unexpected action ${kind}`); + } + } + + generateEdit(peer: number, twin: Twin): Action { + const rng = this.rng; + const doc = twin.docs.rust; + const containers = reachableContainers(doc).filter( + ({ id }) => !ROOT_LISTS.includes(id), + ); + const lists = [ + ...ROOT_LISTS, + ...containers.filter((c) => c.kind === "MovableList").map((c) => c.id), + ]; + const maps = containers.filter((c) => c.kind === "Map").map((c) => c.id); + const texts = containers.filter((c) => c.kind === "Text").map((c) => c.id); + const targetKind = rng.weighted({ + list: 10, + ...(maps.length > 0 ? { map: 1 } : {}), + ...(texts.length > 0 ? { text: 1 } : {}), + }); + if (targetKind === "map") { + return { + kind: "mapSet", + peer, + target: rng.pick(maps), + key: `k${rng.int(3)}`, + value: this.value(), + }; + } + if (targetKind === "text") { + const target = rng.pick(texts); + const length = textContainer(doc, target).length; + return { + kind: "textInsert", + peer, + target, + pos: rng.int(length + 1), + text: `t${this.#nextValue++}`, + }; + } + const target = rng.pick(lists); + const length = movableList(doc, target).length; + const profile = this.profile; + const kind = rng.weighted({ + insert: 4, + ...(profile.children ? { insertContainer: 0.6 } : {}), + ...(length > 0 ? { delete: 2.5 } : {}), + ...(profile.moves && length > 1 ? { move: 4 } : {}), + ...(profile.moves && length > 0 ? { set: 2.5 } : {}), + ...(profile.moves && profile.children && length > 0 ? { setContainer: 0.3 } : {}), + }); + switch (kind) { + case "insert": + return { kind, peer, target, pos: rng.int(length + 1), value: this.value() }; + case "insertContainer": + return { kind, peer, target, pos: rng.int(length + 1), child: this.child() }; + case "delete": { + const pos = rng.int(length); + return { kind, peer, target, pos, len: 1 + rng.int(Math.min(3, length - pos)) }; + } + case "move": + return { kind, peer, target, from: rng.int(length), to: rng.int(length) }; + case "set": + return { kind, peer, target, pos: rng.int(length), value: this.value() }; + case "setContainer": + return { kind, peer, target, pos: rng.int(length), child: this.child() }; + default: + throw new Error(`unexpected edit ${kind}`); + } + } + + value(): string | number { + const value = this.#nextValue++; + return this.rng.bool(0.3) ? value : `v${value}`; + } + + child(): ChildKind { + return this.rng.pick(["Map", "Text", "MovableList"]); + } + + apply(action: Action): void { + switch (action.kind) { + case "insert": + case "insertContainer": + case "delete": + case "move": + case "set": + case "setContainer": + case "mapSet": + case "textInsert": + this.applyEdit(action); + return; + case "commit": + this.twin(action.peer).both("commit", (doc) => doc.commit()); + this.twin(action.peer).record(); + return; + case "undo": + case "redo": { + const twin = this.twin(action.peer); + const managers = twin.undo; + if (managers === undefined) return; + // loro.js picks the undo item before committing the pending transaction, + // Rust commits first. That is unrelated to MovableList; commit up front. + twin.both("commit before undo", (doc) => doc.commit()); + assertTwinAgrees(twin, "after commit before undo", { + metadata: this.profile.metadata, + events: this.profile.events, + }); + const results = twin.both(action.kind, (_doc, engine) => + managers[engine][action.kind](), + ); + if (results !== undefined && results.rust !== results.js) { + throw new Divergence( + `${twin.name}: ${action.kind} returned ${results.rust} in Rust but ${results.js} in loro.js`, + ); + } + twin.record(); + return; + } + case "sync": + this.applySync(action); + return; + case "join": + this.applyJoin(action); + return; + case "checkout": { + const twin = this.twin(action.peer); + const frontiers = this.historyAt(twin, action.at); + // loro.js stays detached after checking out the latest frontiers while + // Rust reattaches; that is unrelated to MovableList, so skip it. + twin.both("commit before checkout", (doc) => doc.commit()); + assertTwinAgrees(twin, "after commit before checkout", { + metadata: this.profile.metadata, + events: this.profile.events, + }); + if ( + isDeepStrictEqual( + sortedFrontiers(frontiers), + sortedFrontiers(twin.docs.rust.frontiers()), + ) + ) { + return; + } + twin.both("checkout", (doc) => doc.checkout(frontiers)); + assertTwinAgrees(twin, `at checkout ${stringify(frontiers)}`, { + metadata: this.profile.metadata, + events: this.profile.events, + }); + twin.both("checkoutToLatest", (doc) => doc.checkoutToLatest()); + return; + } + case "revert": { + const twin = this.twin(action.peer); + const frontiers = this.historyAt(twin, action.at); + twin.both("revertTo", (doc) => { + doc.revertTo(frontiers); + doc.commit(); + }); + twin.record(); + return; + } + case "diff": { + const twin = this.twin(action.peer); + const from = this.historyAt(twin, action.from); + const to = this.historyAt(twin, action.to); + // loro.js also reports containers that are dead at `to`; Rust does not. + const reachable = reachableAt(twin.docs.rust, to); + const results = twin.both("diff", (doc) => + doc + .diff(from, to, false) + .filter(([id]) => reachable === undefined || reachable.has(id)) + .map(([id, diff]) => [id, canonicalDiff(diff)] as const) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)), + ); + if (results !== undefined && !isDeepStrictEqual(results.rust, results.js)) { + throw new Divergence( + `${twin.name}: diff(${stringify(from)}, ${stringify(to)}) differs\n` + + ` rust: ${stringify(results.rust)}\n js: ${stringify(results.js)}`, + ); + } + return; + } + } + } + + twin(peer: number): Twin { + return this.twins.get(peer)!; + } + + historyAt(twin: Twin, index: number): FrontiersLike { + return index < 0 ? [] : (twin.history[index] ?? []); + } + + applyEdit(action: Extract): void { + const twin = this.twin(action.peer); + twin.both(action.kind, (doc, engine) => { + const engineModule = this.engines[engine]; + switch (action.kind) { + case "insert": + movableList(doc, action.target).insert(action.pos, action.value); + return; + case "insertContainer": + movableList(doc, action.target).insertContainer( + action.pos, + newChild(engineModule, action.child), + ); + return; + case "delete": + movableList(doc, action.target).delete(action.pos, action.len); + return; + case "move": + movableList(doc, action.target).move(action.from, action.to); + return; + case "set": + movableList(doc, action.target).set(action.pos, action.value); + return; + case "setContainer": + movableList(doc, action.target).setContainer( + action.pos, + newChild(engineModule, action.child), + ); + return; + case "mapSet": + mapContainer(doc, action.target).set(action.key, action.value); + return; + case "textInsert": + textContainer(doc, action.target).insert(action.pos, action.text); + return; + } + }); + } + + applySync(action: Extract): void { + const source = this.twin(action.from); + const target = this.twin(action.to); + const via = action.via === undefined ? undefined : this.twin(action.via); + // Rust's oplog version counts a pending transaction and loro.js's does not; + // commit first so both halves export the same span. + for (const twin of [source, target, via]) { + if (twin === undefined) continue; + twin.both("commit before sync", (doc) => doc.commit()); + assertTwinAgrees(twin, "after commit before sync", { + metadata: this.profile.metadata, + events: this.profile.events, + }); + } + const exported = (engine: EngineName): Uint8Array[] => { + const sourceEngine: EngineName = action.cross ? other(engine) : engine; + const doc = source.docs[sourceEngine]; + const blobs = [this.exportFor(doc, target.docs[engine], sourceEngine, action.mode)]; + if (via !== undefined) { + blobs.push( + this.exportFor( + via.docs[sourceEngine], + target.docs[engine], + sourceEngine, + "update", + ), + ); + } + return blobs; + }; + const blobs: Record = { + rust: exported("rust"), + js: exported("js"), + }; + target.batchImport = action.mode === "batch"; + try { + target.both(`import from ${source.name} (${action.mode})`, (doc, engine) => + action.mode === "batch" + ? doc.importBatch(blobs[engine]) + : doc.import(blobs[engine][0]!), + ); + } finally { + target.batchImport = false; + } + target.record(); + } + + exportFor( + doc: DocLike, + receiver: DocLike, + sourceEngine: EngineName, + mode: SyncMode, + ): Uint8Array { + const receiverVersion = vvToMap(receiver.oplogVersion()); + switch (mode) { + case "snapshot": + return doc.export({ mode: "snapshot" }); + case "range": { + const spans = [...vvToMap(doc.oplogVersion())].flatMap(([peer, end]) => { + const start = receiverVersion.get(peer) ?? 0; + return end > start ? [{ id: { peer, counter: start }, len: end - start }] : []; + }); + return doc.export({ mode: "updates-in-range", spans }); + } + case "update": + case "batch": + return doc.export({ + mode: "update", + from: new this.engines[sourceEngine].VersionVector(receiverVersion), + }); + } + } + + applyJoin(action: Extract): void { + const source = this.twin(action.from); + source.both("commit before join", (doc) => doc.commit()); + assertTwinAgrees(source, "after commit before join", { + metadata: this.profile.metadata, + events: this.profile.events, + }); + const frontiers = + action.at < 0 + ? source.docs.rust.oplogFrontiers() + : this.historyAt(source, action.at); + const bytes: Record = { + rust: new Uint8Array(), + js: new Uint8Array(), + }; + for (const engine of ENGINES) { + const sourceEngine: EngineName = action.cross ? other(engine) : engine; + const doc = source.docs[sourceEngine]; + bytes[engine] = action.shallow + ? doc.export({ mode: "shallow-snapshot", frontiers }) + : doc.export({ mode: "snapshot" }); + } + const twin = new Twin(this.engines, `peer${action.peer}`, action.peer, undefined, { + undo: this.profile.undo, + }); + this.twins.set(action.peer, twin); + twin.both("import joined snapshot", (doc, engine) => doc.import(bytes[engine])); + twin.record(); + } +} + +/** Container IDs reachable at `frontiers`, or undefined when that cannot be forked. */ +function reachableAt(doc: DocLike, frontiers: FrontiersLike): Set | undefined { + try { + return new Set(reachableContainers(doc.forkAt(frontiers)).map(({ id }) => id)); + } catch { + return undefined; + } +} + +function other(engine: EngineName): EngineName { + return engine === "rust" ? "js" : "rust"; +} + +function newChild(engine: EngineModule, kind: ChildKind): ContainerLike { + switch (kind) { + case "Map": + return new engine.LoroMap(); + case "Text": + return new engine.LoroText(); + case "MovableList": + return new engine.LoroMovableList(); + } +} + +export interface SavedTrace { + readonly profile: FuzzProfile; + readonly seed: number; + readonly actions: readonly Action[]; +} + +/** Replays `trace`, returning the divergence message or undefined when both engines agree. */ +export function replayTrace( + engines: Record, + trace: SavedTrace, +): string | undefined { + try { + new FuzzRun(engines, trace.profile, trace.seed).replay(trace.actions); + return undefined; + } catch (error) { + return error instanceof Divergence + ? describeError(error) + : `crash: ${describeError(error)}`; + } +} + +/** Greedily drops actions while the trace still diverges (not merely crashes). */ +export function minimizeTrace( + engines: Record, + trace: SavedTrace, +): SavedTrace { + let actions = [...trace.actions]; + let changed = true; + while (changed) { + changed = false; + for (let index = actions.length - 1; index >= 0; index -= 1) { + const candidate = actions.filter((_, i) => i !== index); + const message = replayTrace(engines, { ...trace, actions: candidate }); + if (message !== undefined && !message.startsWith("crash")) { + actions = candidate; + changed = true; + } + } + } + return { ...trace, actions }; +} + +function saveTrace(trace: SavedTrace): string { + const safeName = trace.profile.name.replaceAll(/[^a-z0-9]+/giu, "-"); + const path = join(tmpdir(), `loro-js-differential-${safeName}-${trace.seed}.json`); + writeFileSync(path, JSON.stringify(trace)); + return path; +} + +export interface FuzzSummary { + readonly seeds: number; + readonly failures: { readonly seed: number; readonly message: string }[]; +} + +export function fuzz( + engines: Record, + profile: FuzzProfile, + seeds: Iterable, + steps: number, +): FuzzSummary { + const failures: { seed: number; message: string }[] = []; + let count = 0; + for (const seed of seeds) { + count += 1; + const run = new FuzzRun(engines, profile, seed); + try { + run.run(steps); + } catch (error) { + if (!(error instanceof Divergence)) throw error; + const path = saveTrace({ profile, seed, actions: run.log }); + failures.push({ + seed, + message: `${error.message}\nTrace saved to ${path}; replay it with LORO_JS_DIFF_REPLAY=${path}`, + }); + } + } + return { seeds: count, failures }; +} + +export function seedRange(start: number, count: number): number[] { + return Array.from({ length: count }, (_, index) => start + index); +} + +/** Environment overrides for longer local runs. */ +export function fuzzScale( + defaultSeeds: number, + defaultSteps: number, +): { + readonly seeds: number[]; + readonly steps: number; +} { + const start = Number(process.env["LORO_JS_DIFF_SEED"] ?? 1); + const count = Number(process.env["LORO_JS_DIFF_SEEDS"] ?? defaultSeeds); + const steps = Number(process.env["LORO_JS_DIFF_STEPS"] ?? defaultSteps); + return { seeds: seedRange(start, count), steps }; +} diff --git a/loro-js/tests/differential/harness.ts b/loro-js/tests/differential/harness.ts new file mode 100644 index 000000000..8ed209682 --- /dev/null +++ b/loro-js/tests/differential/harness.ts @@ -0,0 +1,480 @@ +import { isDeepStrictEqual } from "node:util"; + +import type { + ContainerLike, + DocLike, + EngineModule, + EventBatchLike, + FrontiersLike, + MapLike, + MovableListLike, + TextLike, + UndoManagerLike, + VersionVectorLike, +} from "./engine"; + +/** + * Differential driver for loro.js against the Rust implementation. + * + * Every simulated peer is a pair of documents with the same peer ID, one per + * engine. Local edits are applied to both halves; syncing exports bytes from + * one peer and imports them into another, optionally crossing engines so each + * engine also decodes the other's encoding. After each step both halves must + * agree on the value, version, per-element metadata, and emitted events. + */ + +export type EngineName = "rust" | "js"; +export const ENGINES: readonly EngineName[] = ["rust", "js"]; + +export class Rng { + #state: number; + + constructor(seed: number) { + this.#state = seed >>> 0 || 1; + } + + next(): number { + // mulberry32 + this.#state = (this.#state + 0x6d2b79f5) >>> 0; + let t = this.#state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; + } + + int(maxExclusive: number): number { + return Math.floor(this.next() * maxExclusive); + } + + bool(probability = 0.5): boolean { + return this.next() < probability; + } + + pick(items: readonly T[]): T { + return items[this.int(items.length)]!; + } + + weighted(weights: Readonly>>): T { + const entries = Object.entries(weights) as [T, number][]; + const total = entries.reduce((sum, [, weight]) => sum + weight, 0); + let roll = this.next() * total; + for (const [key, weight] of entries) { + roll -= weight; + if (roll < 0) return key; + } + return entries.at(-1)![0]; + } +} + +export class Divergence extends Error {} + +type Side = Record; + +export interface TwinOptions { + readonly undo?: boolean; +} + +export class Twin { + readonly docs: Side; + readonly events: Side = { rust: [], js: [] }; + readonly undo: Side | undefined; + /** Frontiers reached by this peer, shared by both halves once they agree. */ + readonly history: FrontiersLike[] = []; + /** Set while `importBatch` runs; see `canonicalBatch`. */ + batchImport = false; + + constructor( + readonly engines: Side, + readonly name: string, + readonly peer: number, + docs?: Side, + options: TwinOptions = {}, + ) { + this.docs = docs ?? { + rust: new engines.rust.LoroDoc(), + js: new engines.js.LoroDoc(), + }; + for (const engine of ENGINES) { + this.docs[engine].setPeerId(peer); + this.docs[engine].subscribe((event) => { + const batch = canonicalBatch(event, this.batchImport); + if (batch.events.length > 0) this.events[engine].push(batch); + }); + } + this.undo = + options.undo === false + ? undefined + : { + rust: new engines.rust.UndoManager(this.docs.rust, { mergeInterval: 0 }), + js: new engines.js.UndoManager(this.docs.js, { mergeInterval: 0 }), + }; + } + + /** Runs `action` on both halves and requires them to both succeed or both throw. */ + both( + label: string, + action: (doc: DocLike, engine: EngineName) => T, + ): Side | undefined { + const results: Partial> = {}; + const errors: Partial> = {}; + for (const engine of ENGINES) { + try { + results[engine] = action(this.docs[engine], engine); + } catch (error) { + errors[engine] = error; + } + } + if ((errors.rust === undefined) !== (errors.js === undefined)) { + throw new Divergence( + `${this.name}: ${label} ` + + (errors.rust === undefined + ? `succeeded in Rust but threw in loro.js: ${describeError(errors.js)}` + : `threw in Rust but succeeded in loro.js: ${describeError(errors.rust)}`), + ); + } + return errors.rust === undefined ? (results as Side) : undefined; + } + + record(): void { + this.history.push(this.docs.rust.frontiers()); + } +} + +export function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function vvToMap(vv: VersionVectorLike): Map { + return new Map([...vv.toJSON()].sort(([left], [right]) => compareStrings(left, right))); +} + +export function sortedFrontiers(frontiers: FrontiersLike): string[] { + return frontiers.map(({ peer, counter }) => `${counter}@${peer}`).sort(compareStrings); +} + +function compareStrings(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function isContainer(value: unknown): value is ContainerLike { + return ( + typeof value === "object" && + value !== null && + typeof (value as { kind?: unknown }).kind === "function" && + typeof (value as { id?: unknown }).id === "string" + ); +} + +/** Replaces container handles with their IDs so values compare structurally. */ +export function plain(value: unknown): unknown { + if (isContainer(value)) return `container:${value.id}`; + if (value instanceof Uint8Array) return { bytes: [...value] }; + if (Array.isArray(value)) return value.map(plain); + if (value instanceof Map) { + return Object.fromEntries( + [...value].map(([key, item]) => [String(key), plain(item)]), + ); + } + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => compareStrings(left, right)) + .map(([key, item]) => [key, plain(item)]), + ); + } + return value; +} + +type DeltaItem = + | { readonly insert: unknown; readonly attributes?: unknown } + | { readonly delete: number } + | { readonly retain: number; readonly attributes?: unknown }; + +/** + * Canonicalizes a list or text delta. Deletes and inserts at the same position + * commute, so each gap between retains is rewritten as one delete followed by + * the concatenated inserts. + */ +export function canonicalDelta(delta: readonly DeltaItem[], text: boolean): DeltaItem[] { + const output: DeltaItem[] = []; + let deleted = 0; + let inserts: { value: unknown; attributes: unknown }[] = []; + const flush = (): void => { + if (deleted > 0) output.push({ delete: deleted }); + for (const insert of inserts) { + const last = output.at(-1); + if ( + last !== undefined && + "insert" in last && + isDeepStrictEqual(plain(last.attributes), plain(insert.attributes)) + ) { + const merged = text + ? (last.insert as string) + (insert.value as string) + : [...(last.insert as unknown[]), ...(insert.value as unknown[])]; + output[output.length - 1] = + insert.attributes === undefined + ? { insert: merged } + : { insert: merged, attributes: insert.attributes }; + } else { + output.push( + insert.attributes === undefined + ? { insert: insert.value } + : { insert: insert.value, attributes: insert.attributes }, + ); + } + } + deleted = 0; + inserts = []; + }; + for (const item of delta) { + if ("insert" in item) { + const attributes = + item.attributes === undefined || + Object.keys(item.attributes as object).length === 0 + ? undefined + : plain(item.attributes); + inserts.push({ value: text ? item.insert : plain(item.insert), attributes }); + } else if ("delete" in item) { + deleted += item.delete; + } else { + flush(); + const attributes = + item.attributes === undefined || + Object.keys(item.attributes as object).length === 0 + ? undefined + : plain(item.attributes); + const last = output.at(-1); + if ( + last !== undefined && + "retain" in last && + isDeepStrictEqual(last.attributes, attributes) + ) { + output[output.length - 1] = + attributes === undefined + ? { retain: last.retain + item.retain } + : { retain: last.retain + item.retain, attributes }; + } else { + output.push( + attributes === undefined + ? { retain: item.retain } + : { retain: item.retain, attributes }, + ); + } + } + } + flush(); + const last = output.at(-1); + if (last !== undefined && "retain" in last && last.attributes === undefined) + output.pop(); + return output; +} + +export function canonicalDiff(diff: unknown): unknown { + const typed = diff as { type: string; diff?: DeltaItem[]; updated?: unknown }; + if (typed.type === "list") { + return { type: "list", diff: canonicalDelta(typed.diff ?? [], false) }; + } + if (typed.type === "text") { + return { type: "text", diff: canonicalDelta(typed.diff ?? [], true) }; + } + return plain(diff); +} + +/** + * Normalizes an event batch for comparison. Two known differences unrelated to + * the data model are folded away: + * + * - Rust labels checkout events with origin `"checkout"`; loro.js leaves the + * origin empty. + * - Rust's multi-blob `importBatch` imports while detached and reattaches with a + * checkout, so its events say `by: "checkout"`; loro.js says `"import"`. + * - Rust commits both undo and redo with origin `"undo"`; loro.js uses + * `"redo"` for redo. + */ +export function canonicalBatch( + batch: EventBatchLike, + batchImport = false, +): EventBatchLike { + const by = batchImport && batch.by === "checkout" ? "import" : batch.by; + const origin = batch.by === "checkout" ? "" : (batch.origin ?? ""); + return { + by, + origin: origin === "redo" ? "undo" : origin, + events: batch.events + .map((event) => ({ + target: event.target, + path: [...event.path], + diff: canonicalDiff(event.diff), + })) + .filter((event) => !isEmptyDiff(event.diff)) + .sort((left, right) => compareStrings(left.target, right.target)), + }; +} + +function isEmptyDiff(diff: unknown): boolean { + const typed = diff as { type: string; diff?: unknown[]; updated?: object }; + if (typed.type === "list" || typed.type === "text") return typed.diff?.length === 0; + if (typed.type === "map") return Object.keys(typed.updated ?? {}).length === 0; + return false; +} + +export interface ContainerInfo { + readonly id: string; + readonly kind: string; + readonly path: readonly (string | number)[]; +} + +/** Lists the reachable containers of `doc`, parents before children. */ +export function reachableContainers(doc: DocLike): ContainerInfo[] { + const output: ContainerInfo[] = []; + const queue: ContainerInfo[] = Object.entries(doc.getShallowValue()) + .sort(([left], [right]) => compareStrings(left, right)) + .map(([name, id]) => ({ id, kind: kindOf(id), path: [name] })); + while (queue.length > 0) { + const info = queue.shift()!; + output.push(info); + const container = doc.getContainerById(info.id); + if (container === undefined) continue; + const shallow = container.getShallowValue(); + const children: [string | number, unknown][] = Array.isArray(shallow) + ? shallow.map((value, index) => [index, value]) + : typeof shallow === "object" && shallow !== null + ? Object.entries(shallow).sort(([left], [right]) => compareStrings(left, right)) + : []; + for (const [key, value] of children) { + if (typeof value === "string" && value.startsWith("cid:")) { + queue.push({ id: value, kind: kindOf(value), path: [...info.path, key] }); + } + } + } + return output; +} + +function kindOf(id: string): string { + return id.slice(id.lastIndexOf(":") + 1); +} + +export interface MovableListMetadata { + readonly creator: (string | undefined)[]; + readonly editor: (string | undefined)[]; + readonly mover: (string | undefined)[]; +} + +export function movableListMetadata(list: MovableListLike): MovableListMetadata { + const creator: (string | undefined)[] = []; + const editor: (string | undefined)[] = []; + const mover: (string | undefined)[] = []; + for (let index = 0; index < list.length; index += 1) { + creator.push(list.getCreatorAt(index)); + editor.push(list.getLastEditorAt(index)); + mover.push(list.getLastMoverAt(index)); + } + return { creator, editor, mover }; +} + +export interface CompareOptions { + /** Compare getCreatorAt/getLastEditorAt/getLastMoverAt. */ + readonly metadata?: boolean; + /** Compare the event batches collected since the last comparison. */ + readonly events?: boolean; +} + +export function assertTwinAgrees( + twin: Twin, + context: string, + options: CompareOptions = {}, +): void { + const fail = (message: string, rust: unknown, js: unknown): never => { + throw new Divergence( + `${twin.name} ${context}: ${message}\n rust: ${stringify(rust)}\n js: ${stringify(js)}`, + ); + }; + const { rust, js } = twin.docs; + const rustValue = plain(rust.toJSON()); + const jsValue = plain(js.toJSON()); + if (!isDeepStrictEqual(rustValue, jsValue)) fail("values differ", rustValue, jsValue); + if (rust.getPendingTxnLength() !== js.getPendingTxnLength()) { + fail( + "pending transaction lengths differ", + rust.getPendingTxnLength(), + js.getPendingTxnLength(), + ); + } + // Rust's oplog version already counts the pending transaction; loro.js's does + // not. Only committed versions are compared. + if (rust.getPendingTxnLength() === 0) { + const rustVersion = vvToMap(rust.oplogVersion()); + const jsVersion = vvToMap(js.oplogVersion()); + if (!isDeepStrictEqual(rustVersion, jsVersion)) { + fail("oplog versions differ", rustVersion, jsVersion); + } + const rustFrontiers = sortedFrontiers(rust.frontiers()); + const jsFrontiers = sortedFrontiers(js.frontiers()); + if (!isDeepStrictEqual(rustFrontiers, jsFrontiers)) { + fail("frontiers differ", rustFrontiers, jsFrontiers); + } + } + if (rust.isDetached() !== js.isDetached()) { + fail("detached state differs", rust.isDetached(), js.isDetached()); + } + const rustContainers = reachableContainers(rust); + const jsContainers = reachableContainers(js); + if (!isDeepStrictEqual(rustContainers, jsContainers)) { + fail("reachable containers differ", rustContainers, jsContainers); + } + if (options.metadata !== false) { + for (const info of rustContainers) { + if (info.kind !== "MovableList") continue; + const rustMeta = movableListMetadata( + rust.getContainerById(info.id) as MovableListLike, + ); + const jsMeta = movableListMetadata(js.getContainerById(info.id) as MovableListLike); + if (!isDeepStrictEqual(rustMeta, jsMeta)) { + fail(`movable-list metadata of ${info.id} differs`, rustMeta, jsMeta); + } + } + } + if (options.events !== false) { + // loro.js also reports changes to child containers whose parent element is + // already deleted; Rust does not. Compare only events of reachable targets. + const reachable = new Set(rustContainers.map(({ id }) => id)); + const rustEvents = onlyReachable(twin.events.rust.splice(0), reachable); + const jsEvents = onlyReachable(twin.events.js.splice(0), reachable); + if (!isDeepStrictEqual(rustEvents, jsEvents)) + fail("events differ", rustEvents, jsEvents); + } else { + twin.events.rust.length = 0; + twin.events.js.length = 0; + } +} + +function onlyReachable( + batches: readonly EventBatchLike[], + reachable: ReadonlySet, +): EventBatchLike[] { + return batches + .map((batch) => ({ + ...batch, + events: batch.events.filter((event) => reachable.has(event.target)), + })) + .filter((batch) => batch.events.length > 0); +} + +export function stringify(value: unknown): string { + return JSON.stringify(value, (_key, item: unknown) => + item instanceof Map ? Object.fromEntries(item) : item, + ); +} + +export function movableList(doc: DocLike, id: string): MovableListLike { + return doc.getContainerById(id) as MovableListLike; +} + +export function mapContainer(doc: DocLike, id: string): MapLike { + return doc.getContainerById(id) as MapLike; +} + +export function textContainer(doc: DocLike, id: string): TextLike { + return doc.getContainerById(id) as TextLike; +} diff --git a/loro-js/tests/differential/movable-list.test.ts b/loro-js/tests/differential/movable-list.test.ts new file mode 100644 index 000000000..79440f4e5 --- /dev/null +++ b/loro-js/tests/differential/movable-list.test.ts @@ -0,0 +1,166 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, test } from "vitest"; + +import { + fuzz, + fuzzScale, + minimizeTrace, + replayTrace, + type FuzzProfile, + type SavedTrace, +} from "./fuzz"; +import { loadLoroJs, loadReference } from "./reference"; + +/** + * Randomized differential tests of MovableList (and its child containers) + * against the Rust implementation. See context/loro-js-movable-list.md. + * + * Profiles leave out behavior that diverges for reasons unrelated to the + * MovableList model; they are listed in that article under "Remaining known + * divergences". In short: undo is only exercised on a single peer, and + * checkout never targets the current frontiers. + */ + +const engines = { rust: loadReference(), js: loadLoroJs() }; + +const none: FuzzProfile = { + name: "", + peers: 3, + moves: false, + children: false, + undo: false, + history: false, + snapshots: false, + shallow: false, + crossEngine: true, + metadata: true, + events: true, +}; + +interface Case { + readonly name: string; + readonly profile: FuzzProfile; + readonly seeds: number; + readonly steps: number; +} + +const cases: Case[] = [ + { + name: "insert/delete", + profile: { ...none, name: "insert/delete", children: true, history: true }, + seeds: 40, + steps: 80, + }, + { + name: "insert/delete undo (one peer)", + profile: { ...none, name: "insert/delete undo (one peer)", peers: 1, undo: true }, + seeds: 40, + steps: 60, + }, + { + name: "move/set", + profile: { ...none, name: "move/set", moves: true }, + seeds: 40, + steps: 80, + }, + { + name: "move/set with children", + profile: { ...none, name: "move/set with children", moves: true, children: true }, + seeds: 30, + steps: 80, + }, + { + name: "move/set history", + profile: { ...none, name: "move/set history", moves: true, history: true }, + seeds: 30, + steps: 80, + }, + { + name: "move/set undo (one peer)", + profile: { + ...none, + name: "move/set undo (one peer)", + peers: 1, + moves: true, + undo: true, + }, + seeds: 30, + steps: 60, + }, + { + name: "snapshots", + profile: { ...none, name: "snapshots", moves: true, children: true, snapshots: true }, + seeds: 30, + steps: 80, + }, + { + name: "shallow snapshots", + profile: { + ...none, + name: "shallow snapshots", + moves: true, + history: true, + snapshots: true, + shallow: true, + }, + seeds: 30, + steps: 80, + }, +]; + +/** + * Profiles that still diverge because loro.js models a MovableList element as + * an item that moves inside the Fugue sequence. The Rust-compatible model in + * context/loro-js-movable-list.md removes them from this set. + */ +const KNOWN_DIVERGENT = new Set([ + "insert/delete", + "insert/delete undo (one peer)", + "move/set", + "move/set with children", + "move/set history", + "move/set undo (one peer)", + "snapshots", + "shallow snapshots", +]); + +describe("MovableList matches the Rust implementation", () => { + test.each(cases)( + "$name", + ({ name, profile, seeds, steps }) => { + const scale = fuzzScale(seeds, steps); + const summary = fuzz(engines, profile, scale.seeds, scale.steps); + const expected = KNOWN_DIVERGENT.has(name) ? "diverges" : "converges"; + const first = summary.failures[0]; + expect({ + outcome: first === undefined ? "converges" : "diverges", + // Show the first divergence of a profile that should converge. + detail: + expected === "converges" && first !== undefined + ? `${summary.failures.length}/${summary.seeds} seeds diverged\n${first.message}` + : undefined, + }).toEqual({ outcome: expected, detail: undefined }); + }, + 600_000, + ); +}); + +// Debugging aid: LORO_JS_DIFF_REPLAY= replays one failing trace, +// and LORO_JS_DIFF_MINIMIZE=1 first shrinks it to a minimal failing sequence. +const replayPath = process.env["LORO_JS_DIFF_REPLAY"]; +test.runIf(replayPath !== undefined)( + "replays a saved trace", + () => { + let trace = JSON.parse(readFileSync(replayPath!, "utf8")) as SavedTrace; + if (process.env["LORO_JS_DIFF_MINIMIZE"] === "1") + trace = minimizeTrace(engines, trace); + const message = replayTrace(engines, trace); + expect( + message === undefined + ? [] + : [message, trace.actions.map((action) => JSON.stringify(action)).join("\n")], + ).toEqual([]); + }, + 600_000, +); diff --git a/loro-js/tests/differential/reference.ts b/loro-js/tests/differential/reference.ts new file mode 100644 index 000000000..65e0b9f95 --- /dev/null +++ b/loro-js/tests/differential/reference.ts @@ -0,0 +1,32 @@ +import { existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; + +import * as loroJs from "../../src/index"; + +import type { EngineModule } from "./engine"; + +/** + * Loads the Rust implementation through the workspace `loro-crdt` Node build. + * + * Build it with `pnpm --dir loro-js build:reference` (a dev WASM build of + * `crates/loro-wasm`), or point `LORO_JS_REFERENCE` at another `nodejs/index.js` + * build, for example one made from an unmerged Rust branch. + */ +export function loadReference(): EngineModule { + const path = + process.env["LORO_JS_REFERENCE"] ?? + fileURLToPath(new URL("../../../crates/loro-wasm/nodejs/index.js", import.meta.url)); + if (!existsSync(path)) { + throw new Error( + `Rust reference build not found at ${path}. ` + + "Run `pnpm --dir loro-js build:reference` first.", + ); + } + const require = createRequire(import.meta.url); + return require(path) as EngineModule; +} + +export function loadLoroJs(): EngineModule { + return loroJs as unknown as EngineModule; +} diff --git a/loro-js/vite.config.ts b/loro-js/vite.config.ts index d827ce4be..8151b4e13 100644 --- a/loro-js/vite.config.ts +++ b/loro-js/vite.config.ts @@ -35,6 +35,15 @@ export default defineConfig({ }, test: { environment: "node", - include: ["tests/**/*.test.ts"], + // The differential suite needs a WASM build of the Rust implementation; run it + // with `pnpm test:differential` after `pnpm build:reference`. + include: + process.env["LORO_JS_DIFFERENTIAL"] === "1" + ? ["tests/differential/**/*.test.ts"] + : ["tests/**/*.test.ts"], + exclude: + process.env["LORO_JS_DIFFERENTIAL"] === "1" + ? ["**/node_modules/**"] + : ["**/node_modules/**", "tests/differential/**"], }, }); From dfe945d0f916ffc64f2098ecc0a261907fa82e36 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 02:54:43 +0800 Subject: [PATCH 02/20] test(loro.js): compare event effects and replicate unordered reverts Event deltas that differ only in shape are checked by effect; reverts whose op order Rust leaves to hash order are checked by value and replicated from Rust; peers that both engines reject are retired; peer IDs are never reused; undo stays unchained; the minimizer keeps those rules. See context/loro-js-movable-list.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/tests/differential/engine.ts | 2 + loro-js/tests/differential/fuzz.ts | 122 +++++++++++++++-- loro-js/tests/differential/harness.ts | 128 +++++++++++++++++- .../tests/differential/movable-list.test.ts | 6 +- 4 files changed, 242 insertions(+), 16 deletions(-) diff --git a/loro-js/tests/differential/engine.ts b/loro-js/tests/differential/engine.ts index 382eb87e1..c842bec0e 100644 --- a/loro-js/tests/differential/engine.ts +++ b/loro-js/tests/differential/engine.ts @@ -91,7 +91,9 @@ export interface DocLike { isShallow(): boolean; shallowSinceVV(): VersionVectorLike; getCursorPos(cursor: CursorLike): { readonly offset: number }; + getPathToContainer(id: string): (string | number)[] | undefined; forkAt(frontiers: FrontiersLike): DocLike; + fork(): DocLike; } export interface UndoManagerLike { diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index 6796431b5..0acbbff51 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -134,6 +134,8 @@ export class FuzzRun { readonly twins = new Map(); readonly log: Action[] = []; #nextValue = 0; + /** Peer IDs are never reused, including those of peers that left the run. */ + #nextPeer: number; /** Last action kind per peer, to keep undo/redo unchained (see `generate`). */ readonly #lastKind = new Map(); @@ -143,6 +145,7 @@ export class FuzzRun { readonly seed: number, ) { this.rng = new Rng(seed); + this.#nextPeer = profile.peers + 1; for (let peer = 1; peer <= profile.peers; peer += 1) { this.twins.set( peer, @@ -229,7 +232,9 @@ export class FuzzRun { generate(): Action { const action = this.generateAction(); - this.#lastKind.set("peer" in action ? action.peer : action.to, action.kind); + if (countsForUndoChain(action)) { + this.#lastKind.set("peer" in action ? action.peer : action.to, action.kind); + } return action; } @@ -282,7 +287,7 @@ export class FuzzRun { }; } case "join": { - const joined = Math.max(...peers) + 1; + const joined = this.#nextPeer++; return { kind: "join", from: peer, @@ -368,8 +373,12 @@ export class FuzzRun { const pos = rng.int(length); return { kind, peer, target, pos, len: 1 + rng.int(Math.min(3, length - pos)) }; } - case "move": - return { kind, peer, target, from: rng.int(length), to: rng.int(length) }; + case "move": { + // A move onto the same index is a no-op; it would hide a chained undo. + const from = rng.int(length); + const to = (from + 1 + rng.int(length - 1)) % length; + return { kind, peer, target, from, to }; + } case "set": return { kind, peer, target, pos: rng.int(length), value: this.value() }; case "setContainer": @@ -462,10 +471,14 @@ export class FuzzRun { case "revert": { const twin = this.twin(action.peer); const frontiers = this.historyAt(twin, action.at); - twin.both("revertTo", (doc) => { - doc.revertTo(frontiers); - doc.commit(); - }); + if (this.#revertOrderIsUnspecified(twin, frontiers)) { + this.#revertThroughRust(twin, frontiers); + } else { + twin.both("revertTo", (doc) => { + doc.revertTo(frontiers); + doc.commit(); + }); + } twin.record(); return; } @@ -493,6 +506,58 @@ export class FuzzRun { } } + /** + * Rust applies a revert's container diffs in `FxHashMap` order within one + * depth, so when several containers at the same depth change, the two + * engines write the same values in a different op order. Twins must share + * one history, so such a revert is checked by value and then replicated. + */ + #revertOrderIsUnspecified(twin: Twin, frontiers: FrontiersLike): boolean { + let diff: [string, unknown][]; + try { + twin.docs.rust.commit(); + diff = twin.docs.rust.diff(twin.docs.rust.frontiers(), frontiers, false); + } catch { + return false; + } + const depths = new Map(); + for (const [id] of diff) { + const depth = twin.docs.rust.getPathToContainer(id)?.length ?? 0; + depths.set(depth, (depths.get(depth) ?? 0) + 1); + } + return [...depths.values()].some((count) => count > 1); + } + + #revertThroughRust(twin: Twin, frontiers: FrontiersLike): void { + twin.both("commit before revert", (doc) => doc.commit()); + assertTwinAgrees(twin, "after commit before revert", { + metadata: this.profile.metadata, + events: this.profile.events, + }); + const fork = twin.docs.js.fork(); + fork.revertTo(frontiers); + fork.commit(); + const before = vvToMap(twin.docs.js.oplogVersion()); + twin.docs.rust.revertTo(frontiers); + twin.docs.rust.commit(); + const expected = plain(twin.docs.rust.toJSON()); + if (!isDeepStrictEqual(plain(fork.toJSON()), expected)) { + throw new Divergence( + `${twin.name}: revertTo(${stringify(frontiers)}) values differ\n` + + ` rust: ${stringify(expected)}\n js: ${stringify(plain(fork.toJSON()))}`, + ); + } + twin.docs.js.import( + twin.docs.rust.export({ + mode: "update", + from: new this.engines.rust.VersionVector(before), + }), + ); + // The Rust half reports a local change, the loro.js half an import. + twin.events.rust.length = 0; + twin.events.js.length = 0; + } + twin(peer: number): Twin { return this.twins.get(peer)!; } @@ -575,15 +640,25 @@ export class FuzzRun { js: exported("js"), }; target.batchImport = action.mode === "batch"; + let imported: unknown; try { - target.both(`import from ${source.name} (${action.mode})`, (doc, engine) => - action.mode === "batch" - ? doc.importBatch(blobs[engine]) - : doc.import(blobs[engine][0]!), + imported = target.both( + `import from ${source.name} (${action.mode})`, + (doc, engine) => + action.mode === "batch" + ? doc.importBatch(blobs[engine]) + : doc.import(blobs[engine][0]!), ); } finally { target.batchImport = false; } + if (imported === undefined) { + // Both rejected it, typically a change concurrent with a shallow root. + // Rust keeps the blob's other changes; loro.js imports atomically. The + // halves can now differ by design, so this peer leaves the run. + this.twins.delete(action.to); + return; + } target.record(); } @@ -614,6 +689,7 @@ export class FuzzRun { } applyJoin(action: Extract): void { + this.#nextPeer = Math.max(this.#nextPeer, action.peer + 1); const source = this.twin(action.from); source.both("commit before join", (doc) => doc.commit()); assertTwinAgrees(source, "after commit before join", { @@ -700,6 +776,7 @@ export function minimizeTrace( changed = false; for (let index = actions.length - 1; index >= 0; index -= 1) { const candidate = actions.filter((_, i) => i !== index); + if (!keepsUndoUnchained(candidate)) continue; const message = replayTrace(engines, { ...trace, actions: candidate }); if (message !== undefined && !message.startsWith("crash")) { actions = candidate; @@ -710,6 +787,27 @@ export function minimizeTrace( return { ...trace, actions }; } +/** Edits, undo and redo decide whether the next undo would be chained. */ +function countsForUndoChain(action: Action): boolean { + if (action.kind === "move") return action.from !== action.to; + return "target" in action || action.kind === "undo" || action.kind === "redo"; +} + +/** The same rule `FuzzRun.generate` follows: undo after an edit, redo after an undo. */ +function keepsUndoUnchained(actions: readonly Action[]): boolean { + const last = new Map(); + for (const action of actions) { + if (!countsForUndoChain(action)) continue; + const peer = "peer" in action ? action.peer : action.to; + const previous = last.get(peer); + if (action.kind === "undo" && (previous === "undo" || previous === "redo")) + return false; + if (action.kind === "redo" && previous !== "undo") return false; + last.set(peer, action.kind); + } + return true; +} + function saveTrace(trace: SavedTrace): string { const safeName = trace.profile.name.replaceAll(/[^a-z0-9]+/giu, "-"); const path = join(tmpdir(), `loro-js-differential-${safeName}-${trace.seed}.json`); diff --git a/loro-js/tests/differential/harness.ts b/loro-js/tests/differential/harness.ts index 8ed209682..8e4bbb808 100644 --- a/loro-js/tests/differential/harness.ts +++ b/loro-js/tests/differential/harness.ts @@ -82,6 +82,8 @@ export class Twin { readonly history: FrontiersLike[] = []; /** Set while `importBatch` runs; see `canonicalBatch`. */ batchImport = false; + /** List and Text values at the last comparison, to replay event deltas. */ + readonly lists = new Map(); constructor( readonly engines: Side, @@ -441,12 +443,134 @@ export function assertTwinAgrees( const reachable = new Set(rustContainers.map(({ id }) => id)); const rustEvents = onlyReachable(twin.events.rust.splice(0), reachable); const jsEvents = onlyReachable(twin.events.js.splice(0), reachable); - if (!isDeepStrictEqual(rustEvents, jsEvents)) - fail("events differ", rustEvents, jsEvents); + if (!isDeepStrictEqual(rustEvents, jsEvents)) { + const reason = equivalentEvents(twin.lists, rustEvents, jsEvents); + if (reason !== undefined) fail(`events differ (${reason})`, rustEvents, jsEvents); + eventShapeDifferences.count += 1; + } } else { twin.events.rust.length = 0; twin.events.js.length = 0; } + twin.lists.clear(); + for (const info of rustContainers) { + if (info.kind !== "MovableList" && info.kind !== "List" && info.kind !== "Text") + continue; + const value = rust.getContainerById(info.id)?.getShallowValue(); + if (Array.isArray(value) || typeof value === "string") twin.lists.set(info.id, value); + } +} + +/** + * Event batches whose list or text deltas differ in shape but not in effect. + * Rust derives an import's list event from the net change; loro.js composes the + * change of each op, which can show an element that was hidden and shown again + * within one import as a delete plus an insert of the same value. After a + * history replay loro.js diffs text before and after instead of using the ops. + */ +export const eventShapeDifferences = { count: 0 }; + +/** Returns why the batches are not equivalent, or undefined when they are. */ +function equivalentEvents( + lists: ReadonlyMap, + rust: readonly EventBatchLike[], + js: readonly EventBatchLike[], +): string | undefined { + if (rust.length !== js.length) return "batch counts differ"; + const rustLists = new Map(); + const jsLists = new Map(); + for (let index = 0; index < rust.length; index += 1) { + const left = rust[index]!; + const right = js[index]!; + if (left.by !== right.by || left.origin !== right.origin) + return "batch labels differ"; + if (left.events.length !== right.events.length) return "event counts differ"; + for (let eventIndex = 0; eventIndex < left.events.length; eventIndex += 1) { + const rustEvent = left.events[eventIndex]!; + const jsEvent = right.events[eventIndex]!; + if (rustEvent.target !== jsEvent.target) return "event targets differ"; + if (!isDeepStrictEqual(rustEvent.path, jsEvent.path)) return "event paths differ"; + const rustDiff = rustEvent.diff as { type: string; diff?: DeltaItem[] }; + const jsDiff = jsEvent.diff as { type: string; diff?: DeltaItem[] }; + const sequence = + rustDiff.type === jsDiff.type && + (rustDiff.type === "list" || rustDiff.type === "text"); + if (!sequence) { + if (!isDeepStrictEqual(rustDiff, jsDiff)) return `${rustDiff.type} diffs differ`; + continue; + } + // Text attributes are compared structurally; only plain inserts are replayed. + if (rustDiff.type === "text" && hasAttributes(rustDiff.diff!, jsDiff.diff!)) { + return "text diffs with attributes differ"; + } + const start = lists.get(rustEvent.target) ?? (rustDiff.type === "text" ? "" : []); + const rustValue = applySequenceDelta( + rustLists.get(rustEvent.target) ?? start, + rustDiff.diff!, + ); + const jsValue = applySequenceDelta( + jsLists.get(jsEvent.target) ?? start, + jsDiff.diff!, + ); + if (rustValue === undefined || jsValue === undefined) + return "a list delta overruns"; + if (!isDeepStrictEqual(rustValue, jsValue)) + return "list deltas have different effects"; + rustLists.set(rustEvent.target, rustValue); + jsLists.set(jsEvent.target, jsValue); + } + } + return undefined; +} + +function hasAttributes(...deltas: (readonly DeltaItem[])[]): boolean { + return deltas.some((delta) => + delta.some((item) => "attributes" in item && item.attributes !== undefined), + ); +} + +function applySequenceDelta( + value: readonly unknown[] | string, + delta: readonly DeltaItem[], +): unknown[] | string | undefined { + if (typeof value !== "string") return applyListDelta(value, delta); + // Text event indices count UTF-16 code units. + const units = applyListDelta( + value.split(""), + delta.map((item) => + "insert" in item ? { insert: (item.insert as string).split("") } : item, + ), + ); + return units === undefined ? undefined : units.join(""); +} + +function applyListDelta( + value: readonly unknown[], + delta: readonly DeltaItem[], +): unknown[] | undefined { + const output: unknown[] = []; + let index = 0; + for (const item of delta) { + if ("retain" in item) { + if (index + item.retain > value.length) return undefined; + output.push(...value.slice(index, index + item.retain)); + index += item.retain; + } else if ("delete" in item) { + if (index + item.delete > value.length) return undefined; + index += item.delete; + } else { + // Shallow values name child containers by ID; events carry handles. + output.push( + ...(item.insert as unknown[]).map((inserted) => + typeof inserted === "string" && inserted.startsWith("container:") + ? inserted.slice("container:".length) + : inserted, + ), + ); + } + } + output.push(...value.slice(index)); + return output; } function onlyReachable( diff --git a/loro-js/tests/differential/movable-list.test.ts b/loro-js/tests/differential/movable-list.test.ts index 79440f4e5..327241f1d 100644 --- a/loro-js/tests/differential/movable-list.test.ts +++ b/loro-js/tests/differential/movable-list.test.ts @@ -1,4 +1,4 @@ -import { readFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; import { describe, expect, test } from "vitest"; @@ -153,8 +153,10 @@ test.runIf(replayPath !== undefined)( "replays a saved trace", () => { let trace = JSON.parse(readFileSync(replayPath!, "utf8")) as SavedTrace; - if (process.env["LORO_JS_DIFF_MINIMIZE"] === "1") + if (process.env["LORO_JS_DIFF_MINIMIZE"] === "1") { trace = minimizeTrace(engines, trace); + writeFileSync(`${replayPath!}.min.json`, JSON.stringify(trace)); + } const message = replayTrace(engines, trace); expect( message === undefined From 47eb6d7103c068b79b80bb60296290839070f9ed Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:17:57 +0800 Subject: [PATCH 03/20] test(loro.js): join shallow snapshots through Rust and keep replayed undo unchained Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/tests/differential/fuzz.ts | 94 ++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 32 deletions(-) diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index 0acbbff51..622521642 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -138,6 +138,12 @@ export class FuzzRun { #nextPeer: number; /** Last action kind per peer, to keep undo/redo unchained (see `generate`). */ readonly #lastKind = new Map(); + /** + * Value after each peer's last undo/redo. Rust skips an undo item that turns + * out to be a no-op and undoes the previous one too, which then chains; so + * undo again only after the value changed. + */ + readonly #valueAtUndo = new Map(); constructor( readonly engines: Record, @@ -231,11 +237,7 @@ export class FuzzRun { } generate(): Action { - const action = this.generateAction(); - if (countsForUndoChain(action)) { - this.#lastKind.set("peer" in action ? action.peer : action.to, action.kind); - } - return action; + return this.generateAction(); } generateAction(): Action { @@ -248,7 +250,8 @@ export class FuzzRun { // an undo whose target was recreated by an earlier undo diverges for List // too. Undo only right after an edit, and redo only right after an undo. const last = this.#lastKind.get(peer); - const canUndo = profile.undo && last !== "undo" && last !== "redo"; + const canUndo = + profile.undo && last !== "undo" && last !== "redo" && this.#changedSinceUndo(peer); const canRedo = profile.undo && last === "undo"; const kind = rng.weighted({ edit: 12, @@ -418,6 +421,15 @@ export class FuzzRun { const twin = this.twin(action.peer); const managers = twin.undo; if (managers === undefined) return; + const last = this.#lastKind.get(action.peer); + if ( + action.kind === "undo" + ? last === "undo" || last === "redo" || !this.#changedSinceUndo(action.peer) + : last !== "undo" + ) { + throw new ChainedUndo(`${action.kind} after ${last ?? "nothing"}`); + } + this.#lastKind.set(action.peer, action.kind); // loro.js picks the undo item before committing the pending transaction, // Rust commits first. That is unrelated to MovableList; commit up front. twin.both("commit before undo", (doc) => doc.commit()); @@ -433,6 +445,7 @@ export class FuzzRun { `${twin.name}: ${action.kind} returned ${results.rust} in Rust but ${results.js} in loro.js`, ); } + this.#valueAtUndo.set(action.peer, stringify(plain(twin.docs.rust.toJSON()))); twin.record(); return; } @@ -558,6 +571,14 @@ export class FuzzRun { twin.events.js.length = 0; } + #changedSinceUndo(peer: number): boolean { + const previous = this.#valueAtUndo.get(peer); + return ( + previous === undefined || + previous !== stringify(plain(this.twin(peer).docs.rust.toJSON())) + ); + } + twin(peer: number): Twin { return this.twins.get(peer)!; } @@ -568,7 +589,7 @@ export class FuzzRun { applyEdit(action: Extract): void { const twin = this.twin(action.peer); - twin.both(action.kind, (doc, engine) => { + const applied = twin.both(action.kind, (doc, engine) => { const engineModule = this.engines[engine]; switch (action.kind) { case "insert": @@ -603,6 +624,10 @@ export class FuzzRun { return; } }); + // Only a real edit breaks an undo chain (see `generateAction`). + if (applied !== undefined && (action.kind !== "move" || action.from !== action.to)) { + this.#lastKind.set(action.peer, action.kind); + } } applySync(action: Extract): void { @@ -706,10 +731,31 @@ export class FuzzRun { }; for (const engine of ENGINES) { const sourceEngine: EngineName = action.cross ? other(engine) : engine; - const doc = source.docs[sourceEngine]; - bytes[engine] = action.shallow - ? doc.export({ mode: "shallow-snapshot", frontiers }) - : doc.export({ mode: "snapshot" }); + bytes[engine] = source.docs[sourceEngine].export({ mode: "snapshot" }); + } + if (action.shallow) { + // loro.js roots a shallow snapshot at the meet of the requested heads, + // Rust at the latest single-head critical version (loro-dev/loro#1095), + // so replicas made from each keep different history. Both halves load + // Rust's; loro.js's must still import into both engines with its value. + const rustShallow = source.docs.rust.export({ + mode: "shallow-snapshot", + frontiers, + }); + const jsShallow = source.docs.js.export({ mode: "shallow-snapshot", frontiers }); + const expected = plain(source.docs.rust.toJSON()); + for (const engine of ENGINES) { + const check = new this.engines[engine].LoroDoc(); + check.import(jsShallow); + if (!isDeepStrictEqual(plain(check.toJSON()), expected)) { + throw new Divergence( + `${engine} imported loro.js's shallow snapshot of ${source.name} as ` + + `${stringify(plain(check.toJSON()))}, expected ${stringify(expected)}`, + ); + } + } + bytes.rust = rustShallow; + bytes.js = rustShallow; } const twin = new Twin(this.engines, `peer${action.peer}`, action.peer, undefined, { undo: this.profile.undo, @@ -776,7 +822,6 @@ export function minimizeTrace( changed = false; for (let index = actions.length - 1; index >= 0; index -= 1) { const candidate = actions.filter((_, i) => i !== index); - if (!keepsUndoUnchained(candidate)) continue; const message = replayTrace(engines, { ...trace, actions: candidate }); if (message !== undefined && !message.startsWith("crash")) { actions = candidate; @@ -787,26 +832,11 @@ export function minimizeTrace( return { ...trace, actions }; } -/** Edits, undo and redo decide whether the next undo would be chained. */ -function countsForUndoChain(action: Action): boolean { - if (action.kind === "move") return action.from !== action.to; - return "target" in action || action.kind === "undo" || action.kind === "redo"; -} - -/** The same rule `FuzzRun.generate` follows: undo after an edit, redo after an undo. */ -function keepsUndoUnchained(actions: readonly Action[]): boolean { - const last = new Map(); - for (const action of actions) { - if (!countsForUndoChain(action)) continue; - const peer = "peer" in action ? action.peer : action.to; - const previous = last.get(peer); - if (action.kind === "undo" && (previous === "undo" || previous === "redo")) - return false; - if (action.kind === "redo" && previous !== "undo") return false; - last.set(peer, action.kind); - } - return true; -} +/** + * A replayed or minimized trace that chains undo, which the generator never + * does (see `generateAction`); `replayTrace` reports it as a crash. + */ +class ChainedUndo extends Error {} function saveTrace(trace: SavedTrace): string { const safeName = trace.profile.name.replaceAll(/[^a-z0-9]+/giu, "-"); From 4ceb7c7bfd5bfd2f241211c533e3baaecdb6e0f8 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:34:10 +0800 Subject: [PATCH 04/20] test(loro.js): accept no-op sequence events and replicate reverts whose diff shape differs Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/tests/differential/fuzz.ts | 61 +++++++++++++++++++++++---- loro-js/tests/differential/harness.ts | 39 +++++++++++++++-- 2 files changed, 87 insertions(+), 13 deletions(-) diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index 622521642..e3572cce9 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -12,6 +12,7 @@ import { assertTwinAgrees, canonicalDiff, describeError, + eventShapeDifferences, mapContainer, movableList, plain, @@ -484,7 +485,7 @@ export class FuzzRun { case "revert": { const twin = this.twin(action.peer); const frontiers = this.historyAt(twin, action.at); - if (this.#revertOrderIsUnspecified(twin, frontiers)) { + if (this.#revertIsUnspecified(twin, frontiers)) { this.#revertThroughRust(twin, frontiers); } else { twin.both("revertTo", (doc) => { @@ -520,25 +521,42 @@ export class FuzzRun { } /** + * Whether the ops of a revert are not determined by the value it restores. * Rust applies a revert's container diffs in `FxHashMap` order within one * depth, so when several containers at the same depth change, the two - * engines write the same values in a different op order. Twins must share - * one history, so such a revert is checked by value and then replicated. + * engines write the same values in a different op order. A revert's list ops + * also follow the shape of `diff(current, target)`: an insert before or after + * a deletion at the same index lands on either side of the deleted positions. + * Both shapes describe the same change (see `eventShapeDifferences`). Twins + * must share one history, so such a revert is checked by value and then + * replicated. */ - #revertOrderIsUnspecified(twin: Twin, frontiers: FrontiersLike): boolean { - let diff: [string, unknown][]; + #revertIsUnspecified(twin: Twin, frontiers: FrontiersLike): boolean { + let rustDiff: [string, unknown][]; + let jsDiff: [string, unknown][]; try { - twin.docs.rust.commit(); - diff = twin.docs.rust.diff(twin.docs.rust.frontiers(), frontiers, false); + twin.both("commit before diff", (doc) => doc.commit()); + rustDiff = twin.docs.rust.diff(twin.docs.rust.frontiers(), frontiers, false); + jsDiff = twin.docs.js.diff(twin.docs.js.frontiers(), frontiers, false); } catch { return false; } const depths = new Map(); - for (const [id] of diff) { + for (const [id] of rustDiff) { const depth = twin.docs.rust.getPathToContainer(id)?.length ?? 0; depths.set(depth, (depths.get(depth) ?? 0) + 1); } - return [...depths.values()].some((count) => count > 1); + if ([...depths.values()].some((count) => count > 1)) return true; + const shape = (diff: [string, unknown][]) => + new Map(diff.map(([id, change]) => [id, orderedDiff(change)])); + const rustShape = shape(rustDiff); + if ( + !isDeepStrictEqual(rustShape, shape(jsDiff.filter(([id]) => rustShape.has(id)))) + ) { + eventShapeDifferences.count += 1; + return true; + } + return false; } #revertThroughRust(twin: Twin, frontiers: FrontiersLike): void { @@ -790,6 +808,31 @@ function newChild(engine: EngineModule, kind: ChildKind): ContainerLike { } } +/** + * `canonicalDiff`, except that list deltas keep the order of inserts and + * deletes at one index; adjacent items of one kind are still merged. + */ +function orderedDiff(diff: unknown): unknown { + const typed = diff as { type: string; diff?: Record[] }; + if (typed.type !== "list") return canonicalDiff(diff); + const output: Record[] = []; + for (const item of typed.diff!) { + const last = output.at(-1); + if (last !== undefined && "insert" in item && "insert" in last) { + last.insert = [...(last.insert as unknown[]), ...(plain(item.insert) as unknown[])]; + } else if (last !== undefined && "delete" in item && "delete" in last) { + last.delete = (last.delete as number) + (item.delete as number); + } else if (last !== undefined && "retain" in item && "retain" in last) { + last.retain = (last.retain as number) + (item.retain as number); + } else { + output.push( + "insert" in item ? { insert: plain(item.insert) } : { ...item }, + ); + } + } + return { type: "list", diff: output }; +} + export interface SavedTrace { readonly profile: FuzzProfile; readonly seed: number; diff --git a/loro-js/tests/differential/harness.ts b/loro-js/tests/differential/harness.ts index 8e4bbb808..0caa01927 100644 --- a/loro-js/tests/differential/harness.ts +++ b/loro-js/tests/differential/harness.ts @@ -465,17 +465,22 @@ export function assertTwinAgrees( * Event batches whose list or text deltas differ in shape but not in effect. * Rust derives an import's list event from the net change; loro.js composes the * change of each op, which can show an element that was hidden and shown again - * within one import as a delete plus an insert of the same value. After a - * history replay loro.js diffs text before and after instead of using the ops. + * within one import as a delete plus an insert of the same value. When several + * imported moves bring an element back to its index, Rust's net change cancels + * and it reports nothing, so sequence events without effect are dropped. After + * a history replay loro.js diffs text before and after instead of using the ops. */ export const eventShapeDifferences = { count: 0 }; /** Returns why the batches are not equivalent, or undefined when they are. */ function equivalentEvents( lists: ReadonlyMap, - rust: readonly EventBatchLike[], - js: readonly EventBatchLike[], + rustBatches: readonly EventBatchLike[], + jsBatches: readonly EventBatchLike[], ): string | undefined { + const rust = withoutNoOpSequenceEvents(lists, rustBatches); + const js = withoutNoOpSequenceEvents(lists, jsBatches); + if (rust === undefined || js === undefined) return "a list delta overruns"; if (rust.length !== js.length) return "batch counts differ"; const rustLists = new Map(); const jsLists = new Map(); @@ -523,6 +528,32 @@ function equivalentEvents( return undefined; } +/** Drops list and text events that leave their target unchanged, then empty batches. */ +function withoutNoOpSequenceEvents( + lists: ReadonlyMap, + batches: readonly EventBatchLike[], +): EventBatchLike[] | undefined { + const current = new Map(lists); + const output: EventBatchLike[] = []; + for (const batch of batches) { + const events: EventBatchLike["events"][number][] = []; + for (const event of batch.events) { + const diff = event.diff as { type: string; diff?: DeltaItem[] }; + if ((diff.type !== "list" && diff.type !== "text") || hasAttributes(diff.diff!)) { + events.push(event); + continue; + } + const before = current.get(event.target) ?? (diff.type === "text" ? "" : []); + const after = applySequenceDelta(before, diff.diff!); + if (after === undefined) return undefined; + current.set(event.target, after); + if (!isDeepStrictEqual(before, after)) events.push(event); + } + if (events.length > 0) output.push({ ...batch, events }); + } + return output; +} + function hasAttributes(...deltas: (readonly DeltaItem[])[]): boolean { return deltas.some((delta) => delta.some((item) => "attributes" in item && item.attributes !== undefined), From 17e8f2c069770c79d9f8854620dfb36374f93580 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:46:58 +0800 Subject: [PATCH 05/20] test(loro.js): skip shallow editors and start range exports at the shallow root Both follow Rust issues listed in context/loro-js-movable-list.md. Co-Authored-By: Claude Opus 5.5 (1M context) --- context/loro-js-movable-list.md | 20 ++++++++++++++++++++ loro-js/tests/differential/fuzz.ts | 10 ++++++---- loro-js/tests/differential/harness.ts | 8 ++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index 33be8f520..b3e632ad8 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -221,3 +221,23 @@ them (see the comments in `harness.ts` and `fuzz.ts`): - Multi-blob `importBatch` events are labeled `by: "checkout"` by Rust and `"import"` by loro.js; checkout events carry origin `"checkout"` only in Rust. Rust's `oplogVersion()` also counts the pending transaction. + +Differences in form, not in effect, which the suite checks by value: + +- `revertTo` applies same-depth containers in Rust's `FxHashMap` order, and + its list ops follow the insert/delete order of `diff(current, target)` at one + index, which Rust's `DeltaRope` composition decides. When either differs, + the suite checks the revert by value and replicates Rust's ops. +- Event deltas can differ in shape, or be a no-op where Rust reports nothing; + the suite then checks that each engine's deltas turn the previous value into + the new one. + +Rust issues the suite works around: + +- On a shallow document, `getLastEditorAt` can report the root list item's or + the last mover's peer instead of the setter's (`record_shallow_root_state` + in `history_cache.rs` seeds the value writer with the list item ID). The + suite does not compare editors on shallow documents. +- `updates-in-range` on a shallow document exports nothing for a span that + starts in trimmed history (`ChangeStore::iter_blocks`). The suite starts + such spans at the shallow root. diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index e3572cce9..d9fe1bbb6 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -716,8 +716,12 @@ export class FuzzRun { case "snapshot": return doc.export({ mode: "snapshot" }); case "range": { + // Rust exports nothing for a span that starts in history trimmed by a + // shallow snapshot (context/loro-js-movable-list.md), so spans start at + // the shallow root. + const trimmed = vvToMap(doc.shallowSinceVV()); const spans = [...vvToMap(doc.oplogVersion())].flatMap(([peer, end]) => { - const start = receiverVersion.get(peer) ?? 0; + const start = Math.max(receiverVersion.get(peer) ?? 0, trimmed.get(peer) ?? 0); return end > start ? [{ id: { peer, counter: start }, len: end - start }] : []; }); return doc.export({ mode: "updates-in-range", spans }); @@ -825,9 +829,7 @@ function orderedDiff(diff: unknown): unknown { } else if (last !== undefined && "retain" in item && "retain" in last) { last.retain = (last.retain as number) + (item.retain as number); } else { - output.push( - "insert" in item ? { insert: plain(item.insert) } : { ...item }, - ); + output.push("insert" in item ? { insert: plain(item.insert) } : { ...item }); } } return { type: "list", diff: output }; diff --git a/loro-js/tests/differential/harness.ts b/loro-js/tests/differential/harness.ts index 0caa01927..cef02259d 100644 --- a/loro-js/tests/differential/harness.ts +++ b/loro-js/tests/differential/harness.ts @@ -426,12 +426,20 @@ export function assertTwinAgrees( fail("reachable containers differ", rustContainers, jsContainers); } if (options.metadata !== false) { + // Rust seeds shallow-root elements' last editor from the wrong op once it + // diffs across the root (see context/loro-js-movable-list.md), so a + // shallow document's editors are not compared. + const shallow = rust.isShallow(); for (const info of rustContainers) { if (info.kind !== "MovableList") continue; const rustMeta = movableListMetadata( rust.getContainerById(info.id) as MovableListLike, ); const jsMeta = movableListMetadata(js.getContainerById(info.id) as MovableListLike); + if (shallow) { + rustMeta.editor.length = 0; + jsMeta.editor.length = 0; + } if (!isDeepStrictEqual(rustMeta, jsMeta)) { fail(`movable-list metadata of ${info.id} differs`, rustMeta, jsMeta); } From 95006a51cb57ae5ba497a432c0f288cb9eff8872 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 04:05:31 +0800 Subject: [PATCH 06/20] test(loro.js): measure revert container depths at the revert target Co-Authored-By: Claude Opus 5.5 (1M context) --- loro-js/tests/differential/fuzz.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index d9fe1bbb6..7a02f3e3a 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -541,9 +541,20 @@ export class FuzzRun { } catch { return false; } + // Depths at the target: a container the revert brings back has no path now. + // Rust cannot fork a shallow document; without the target, such a + // container's depth is unknown and the order counts as unspecified. + let atTarget: DocLike | undefined; + try { + atTarget = twin.docs.rust.forkAt(frontiers); + } catch { + atTarget = undefined; + } const depths = new Map(); for (const [id] of rustDiff) { - const depth = twin.docs.rust.getPathToContainer(id)?.length ?? 0; + const path = (atTarget ?? twin.docs.rust).getPathToContainer(id); + if (path === undefined && atTarget === undefined) return true; + const depth = path?.length ?? 0; depths.set(depth, (depths.get(depth) ?? 0) + 1); } if ([...depths.values()].some((count) => count > 1)) return true; From 17366fd6791531bfcfef54660d202c0fcd28b543 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 01:01:25 +0800 Subject: [PATCH 07/20] test(loro.js): run the MovableList differential suite on the shared Rust reference Load Rust through tests/support/rust-reference.ts like the other differential suites, so it runs in `pnpm test` (and in CI's test-loro-js) and skips without a WASM build. Drop the separate workflow and scripts. insert/delete now converges on main. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/loro-js.yml | 53 ------------------- context/loro-js-movable-list.md | 37 +++++++------ context/loro-js-performance.md | 2 +- loro-js/AGENTS.md | 3 ++ loro-js/package.json | 2 - loro-js/scripts/build-reference.mjs | 27 ---------- .../tests/differential/movable-list.test.ts | 8 +-- loro-js/tests/differential/reference.ts | 28 +++------- loro-js/vite.config.ts | 11 +--- 9 files changed, 38 insertions(+), 133 deletions(-) delete mode 100644 .github/workflows/loro-js.yml delete mode 100644 loro-js/scripts/build-reference.mjs diff --git a/.github/workflows/loro-js.yml b/.github/workflows/loro-js.yml deleted file mode 100644 index 55a1c6082..000000000 --- a/.github/workflows/loro-js.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: loro.js - -on: - push: - branches: ["main"] - paths: - - "loro-js/**" - - "crates/**" - - "pnpm-lock.yaml" - - ".github/workflows/loro-js.yml" - pull_request: - branches: ["main"] - paths: - - "loro-js/**" - - "crates/**" - - "pnpm-lock.yaml" - - ".github/workflows/loro-js.yml" - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - with: - version: 10 - - uses: actions/setup-node@v4 - with: - node-version: 22.x - - run: pnpm install --frozen-lockfile - - run: pnpm --dir loro-js check - - differential: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v4 - with: - version: 10 - - uses: actions/setup-node@v4 - with: - node-version: 22.x - - uses: denoland/setup-deno@v1 - with: - deno-version: v2.x - - run: rustup target add wasm32-unknown-unknown - - uses: jetli/wasm-bindgen-action@v0.2.0 - with: - version: "0.2.100" - - uses: Swatinem/rust-cache@v2 - - run: pnpm install --frozen-lockfile - - run: pnpm --dir loro-js build:reference - - run: pnpm --dir loro-js test:differential diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index b3e632ad8..a2119d294 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -1,6 +1,6 @@ # loro.js MovableList: Rust-Compatible Model -Verified against code 2026-09-28. +Verified against code 2026-09-30. `loro-js/` must converge with the Rust implementation and read and write its encoding. This article states the Rust MovableList semantics that loro.js must @@ -11,7 +11,9 @@ suite checks the two against each other. Complexity rules still come from Status: the differential suite is in the tree. The "loro.js model" section is the design of the follow-up change; until it lands, `LoroMovableList` still uses the old model described under "Why the old loro.js model diverged", and -the suite records those profiles in `KNOWN_DIVERGENT`. +the suite records those profiles in `KNOWN_DIVERGENT`. The insert/delete-only +profile converges on `main` since the Fugue fixes of loro-dev/loro#1136 and +#1131, so it is not in that set. ## Rust semantics (the contract) @@ -177,25 +179,29 @@ identical values, versions, frontiers, reachable container IDs, after every step. Sync crosses engines, so each engine also decodes the other's updates and snapshots. +It uses the reference that the other loro.js differential suites share +(`loro-js/tests/support/rust-reference.ts`): `crates/loro-wasm/nodejs`, built by +`pnpm -C crates/loro-wasm build-dev`, or the build `LORO_WASM_NODEJS` points +at (for example an unmerged Rust branch). Without a build the suite skips; +`pnpm test-loro-js` sets `LORO_REQUIRE_WASM_REFERENCE=1` and CI runs it after +`release-wasm`, so there it always runs. + ```sh -pnpm --dir loro-js build:reference # dev WASM build of crates/loro-wasm (nodejs) -pnpm --dir loro-js test:differential # CI-sized seeds -LORO_JS_DIFF_SEEDS=500 LORO_JS_DIFF_STEPS=200 pnpm --dir loro-js test:differential +pnpm -C crates/loro-wasm build-dev +cd loro-js +pnpm vitest run tests/differential/movable-list.test.ts # CI-sized seeds +LORO_JS_DIFF_SEEDS=500 LORO_JS_DIFF_STEPS=200 pnpm vitest run tests/differential/movable-list.test.ts ``` -`LORO_JS_REFERENCE=/path/to/nodejs/index.js` compares against another Rust -build, for example an unmerged branch. The suite folds away a few differences -that are unrelated to the data model. Each is named in `harness.ts` -(`canonicalBatch`) and in the test file's profile notes, so a fix can remove -its normalization. +The suite folds away a few differences that are unrelated to the data model. +Each is named in `harness.ts` (`canonicalBatch`) and in the test file's profile +notes, so a fix can remove its normalization. Files: `engine.ts` (the shared API surface), `harness.ts` (twin peers, canonical events, comparisons), `fuzz.ts` (action generation, transport, replay and minimization), `movable-list.test.ts` (profiles). A divergence saves its trace to the temp directory; replay it with `LORO_JS_DIFF_REPLAY=` and add `LORO_JS_DIFF_MINIMIZE=1` to shrink it. -The suite is excluded from `pnpm test` because it needs the WASM build; CI -runs it in `.github/workflows/loro-js.yml`. ## Remaining known divergences (not MovableList-specific) @@ -209,13 +215,14 @@ them (see the comments in `harness.ts` and `fuzz.ts`): document order. `UndoManager.undo()` picks its item before committing the pending transaction. Rust commits redo with origin `"undo"`. The profiles run undo on one peer, never chained, after an explicit commit. -- `checkout(frontiers)` at the latest frontiers leaves loro.js detached; Rust - stays attached. The profiles skip such checkouts. +- `checkout(frontiers)` at the latest frontiers left loro.js detached; Rust + stays attached. Fixed by loro-dev/loro#1143; the profiles still skip such + checkouts. - loro.js emits events and `diff()` entries for child containers whose parent element is already deleted; Rust does not. Only reachable targets are compared. - A snapshot-hydrated loro.js document has no tombstones or origins. It mispositions List/Text/MovableList ops that are concurrent with deletes the - snapshot already contains. + snapshot already contains (loro-dev/loro#1163). - Some shallow-snapshot imports that Rust accepts are rejected by loro.js with "cannot import updates that depend on an outdated version". - Multi-blob `importBatch` events are labeled `by: "checkout"` by Rust and diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 6c7725214..474a4a631 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -539,6 +539,6 @@ The remaining differences are representation and JavaScript constant factors: Keep randomized index-invariant coverage in `loro-js/tests/indexes.test.ts` and Rust/TypeScript fixture coverage in `loro-js/tests/rust-interop.test.ts` when changing these structures. The randomized Rust differential suite -(`pnpm --dir loro-js test:differential`, see +(`loro-js/tests/differential/`, see [loro-js-movable-list.md](loro-js-movable-list.md)) checks convergence, events and encoding interchange against a WASM build of the Rust implementation. diff --git a/loro-js/AGENTS.md b/loro-js/AGENTS.md index 3e46a5bb0..e66274c5b 100644 --- a/loro-js/AGENTS.md +++ b/loro-js/AGENTS.md @@ -12,6 +12,9 @@ the same binary and JSON data as Rust (`loro-crdt`) and give the same results. - Multi-peer fuzzing of all containers against replay, checkout, event, and shallow-snapshot oracles, optionally in lockstep with Rust: [context/loro-js-differential-fuzz.md](../context/loro-js-differential-fuzz.md). +- MovableList model (Rust's positions + LWW elements) and its multi-peer Rust + differential suite (`tests/differential/`): + [context/loro-js-movable-list.md](../context/loro-js-movable-list.md). - Rich text: style anchors live in the Text sequence and op positions count them. Read [context/loro-js-richtext-anchors.md](../context/loro-js-richtext-anchors.md) before changing Text positions, marks, or Text snapshot state. diff --git a/loro-js/package.json b/loro-js/package.json index 4641fe84c..2785b414c 100644 --- a/loro-js/package.json +++ b/loro-js/package.json @@ -50,7 +50,6 @@ "bench:subscribers": "pnpm build && node --expose-gc benchmarks/container-subscriber.mjs", "bench:text-buffer": "pnpm build && node --expose-gc benchmarks/text-buffer.mjs", "build": "vp pack", - "build:reference": "node scripts/build-reference.mjs", "check": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm test && pnpm build", "clean": "rm -rf dist", "format": "vp fmt", @@ -58,7 +57,6 @@ "fixtures:rewrite": "pnpm build && node scripts/rewrite-rust-fixture.mjs", "lint": "vp lint", "test": "vp test run", - "test:differential": "LORO_JS_DIFFERENTIAL=1 vp test run", "typecheck": "tsgo -p tsconfig.json --noEmit" }, "devDependencies": { diff --git a/loro-js/scripts/build-reference.mjs b/loro-js/scripts/build-reference.mjs deleted file mode 100644 index cc0742398..000000000 --- a/loro-js/scripts/build-reference.mjs +++ /dev/null @@ -1,27 +0,0 @@ -// Builds the Node target of `crates/loro-wasm` (the `loro-crdt` package) in the -// dev profile, for `pnpm test:differential`. This is the subset of -// `pnpm -C crates/loro-wasm build-dev` that the differential suite needs: one -// target, no bundler/browser outputs, and no package tests. -import { spawnSync } from "node:child_process"; -import { readFileSync, writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; - -const wasmDir = fileURLToPath(new URL("../../crates/loro-wasm/", import.meta.url)); - -function run(command, args) { - const result = spawnSync(command, args, { cwd: wasmDir, stdio: "inherit" }); - if (result.status !== 0) { - throw new Error(`${command} ${args.join(" ")} exited with ${result.status}`); - } -} - -run("deno", ["run", "-A", "./scripts/build.ts", "dev", "nodejs"]); -run("pnpm", ["exec", "rollup", "-c"]); - -// Same rewrite as `crates/loro-wasm/scripts/post-rollup.ts` for the Node target. -const indexPath = new URL("nodejs/index.js", `file://${wasmDir}`); -const index = readFileSync(indexPath, "utf8").replace( - /require\(["']loro-wasm["']\)/g, - 'require("./loro_wasm")', -); -writeFileSync(indexPath, index); diff --git a/loro-js/tests/differential/movable-list.test.ts b/loro-js/tests/differential/movable-list.test.ts index 327241f1d..2228d17a6 100644 --- a/loro-js/tests/differential/movable-list.test.ts +++ b/loro-js/tests/differential/movable-list.test.ts @@ -22,7 +22,8 @@ import { loadLoroJs, loadReference } from "./reference"; * checkout never targets the current frontiers. */ -const engines = { rust: loadReference(), js: loadLoroJs() }; +const rust = loadReference(); +const engines = { rust: rust!, js: loadLoroJs() }; const none: FuzzProfile = { name: "", @@ -115,7 +116,6 @@ const cases: Case[] = [ * context/loro-js-movable-list.md removes them from this set. */ const KNOWN_DIVERGENT = new Set([ - "insert/delete", "insert/delete undo (one peer)", "move/set", "move/set with children", @@ -125,7 +125,7 @@ const KNOWN_DIVERGENT = new Set([ "shallow snapshots", ]); -describe("MovableList matches the Rust implementation", () => { +describe.skipIf(rust === undefined)("MovableList matches the Rust implementation", () => { test.each(cases)( "$name", ({ name, profile, seeds, steps }) => { @@ -149,7 +149,7 @@ describe("MovableList matches the Rust implementation", () => { // Debugging aid: LORO_JS_DIFF_REPLAY= replays one failing trace, // and LORO_JS_DIFF_MINIMIZE=1 first shrinks it to a minimal failing sequence. const replayPath = process.env["LORO_JS_DIFF_REPLAY"]; -test.runIf(replayPath !== undefined)( +test.runIf(replayPath !== undefined && rust !== undefined)( "replays a saved trace", () => { let trace = JSON.parse(readFileSync(replayPath!, "utf8")) as SavedTrace; diff --git a/loro-js/tests/differential/reference.ts b/loro-js/tests/differential/reference.ts index 65e0b9f95..7c33f9adb 100644 --- a/loro-js/tests/differential/reference.ts +++ b/loro-js/tests/differential/reference.ts @@ -1,30 +1,16 @@ -import { existsSync } from "node:fs"; -import { createRequire } from "node:module"; -import { fileURLToPath } from "node:url"; - import * as loroJs from "../../src/index"; +import { loadRustReference } from "../support/rust-reference"; import type { EngineModule } from "./engine"; /** - * Loads the Rust implementation through the workspace `loro-crdt` Node build. - * - * Build it with `pnpm --dir loro-js build:reference` (a dev WASM build of - * `crates/loro-wasm`), or point `LORO_JS_REFERENCE` at another `nodejs/index.js` - * build, for example one made from an unmerged Rust branch. + * The Rust implementation from the shared differential reference + * (`tests/support/rust-reference.ts`: `crates/loro-wasm/nodejs`, or + * `LORO_WASM_NODEJS`). Undefined when that build is missing, so the suite + * skips; `LORO_REQUIRE_WASM_REFERENCE=1` makes a missing build an error. */ -export function loadReference(): EngineModule { - const path = - process.env["LORO_JS_REFERENCE"] ?? - fileURLToPath(new URL("../../../crates/loro-wasm/nodejs/index.js", import.meta.url)); - if (!existsSync(path)) { - throw new Error( - `Rust reference build not found at ${path}. ` + - "Run `pnpm --dir loro-js build:reference` first.", - ); - } - const require = createRequire(import.meta.url); - return require(path) as EngineModule; +export function loadReference(): EngineModule | undefined { + return loadRustReference() as unknown as EngineModule | undefined; } export function loadLoroJs(): EngineModule { diff --git a/loro-js/vite.config.ts b/loro-js/vite.config.ts index 8151b4e13..d827ce4be 100644 --- a/loro-js/vite.config.ts +++ b/loro-js/vite.config.ts @@ -35,15 +35,6 @@ export default defineConfig({ }, test: { environment: "node", - // The differential suite needs a WASM build of the Rust implementation; run it - // with `pnpm test:differential` after `pnpm build:reference`. - include: - process.env["LORO_JS_DIFFERENTIAL"] === "1" - ? ["tests/differential/**/*.test.ts"] - : ["tests/**/*.test.ts"], - exclude: - process.env["LORO_JS_DIFFERENTIAL"] === "1" - ? ["**/node_modules/**"] - : ["**/node_modules/**", "tests/differential/**"], + include: ["tests/**/*.test.ts"], }, }); From ca3f781a3d36b47ae5a17bf513c75f697249bd08 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 02:53:49 +0800 Subject: [PATCH 08/20] fix(loro.js): model MovableList like Rust (positions and LWW elements) MovableList keeps a Fugue sequence of list items and elements whose position and value are last-writer-wins registers, as Rust does. Moves of concurrently deleted elements, deletes after moves, snapshots (list item, element and last-set IDs, dead items), metadata getters, version transitions, revertTo/applyDiff (Rust's apply_delta), undo and events now match Rust. Snapshot-hydrated documents replay history before concurrent imports, and a few shallow-snapshot incompatibilities are fixed. The differential suite passes every profile. Design and remaining gaps: context/loro-js-movable-list.md. Rebuilt on current main: MovableList joins #1126's per-container snapshot completion (its snapshot state names Rust IDs), applyDiff/revertTo/undo use main's port of Rust #1138's apply_delta on the new model, and undo transforms only over changes the UndoManager did not make. Co-Authored-By: Claude Opus 5.5 --- .changeset/loro-js-movable-list-model.md | 18 + context/loro-js-movable-list.md | 141 +- context/loro-js-performance.md | 26 +- crates/loro/tests/loro_js_interop.rs | 41 + loro-js/scripts/rewrite-rust-fixture.mjs | 9 + loro-js/src/runtime/containers.ts | 438 ++--- loro-js/src/runtime/document.ts | 1480 +++++++---------- loro-js/src/runtime/movable-list.ts | 896 ++++++++++ loro-js/src/runtime/sequence-index.ts | 51 + .../tests/differential/movable-list.test.ts | 10 +- .../fixtures/rust/movable-move-delete.blob | Bin 0 -> 110 bytes .../fixtures/rust/movable-snapshot-peer3.blob | Bin 0 -> 109 bytes .../fixtures/rust/movable-snapshot-peer4.blob | Bin 0 -> 103 bytes .../rust/movable-snapshot-updates.blob | Bin 0 -> 291 bytes .../tests/fixtures/rust/movable-snapshot.blob | Bin 0 -> 462 bytes .../fixtures/rust/movable-snapshot.ts.blob | Bin 0 -> 490 bytes .../fixtures/rust/runtime-snapshot.ts.blob | Bin 1124 -> 1136 bytes loro-js/tests/movable-list.test.ts | 414 +++++ loro-js/tests/runtime.test.ts | 6 + 19 files changed, 2331 insertions(+), 1199 deletions(-) create mode 100644 .changeset/loro-js-movable-list-model.md create mode 100644 loro-js/src/runtime/movable-list.ts create mode 100644 loro-js/tests/fixtures/rust/movable-move-delete.blob create mode 100644 loro-js/tests/fixtures/rust/movable-snapshot-peer3.blob create mode 100644 loro-js/tests/fixtures/rust/movable-snapshot-peer4.blob create mode 100644 loro-js/tests/fixtures/rust/movable-snapshot-updates.blob create mode 100644 loro-js/tests/fixtures/rust/movable-snapshot.blob create mode 100644 loro-js/tests/fixtures/rust/movable-snapshot.ts.blob create mode 100644 loro-js/tests/movable-list.test.ts diff --git a/.changeset/loro-js-movable-list-model.md b/.changeset/loro-js-movable-list-model.md new file mode 100644 index 000000000..963fd96bc --- /dev/null +++ b/.changeset/loro-js-movable-list-model.md @@ -0,0 +1,18 @@ +--- +"loro.js": minor +--- + +`LoroMovableList` now follows Rust's model: list items (positions) and elements are separate, and an element's position and value are last-writer-wins registers. This fixes convergence with Rust and `loro-crdt`: + +- A move of an element that another peer deleted concurrently keeps the element (`["a","c","B"]`, as in Rust), and a delete after a move removes the moved element. +- Snapshots write and read Rust's MovableList layout, including list item IDs, element IDs, last-set IDs and dead list items. Before, loro.js wrote every position and last-set ID as the element ID. +- `getLastEditorAt` and `getLastMoverAt` return the last setter and mover instead of the creator. +- Checkout, `diff`, `revertTo`, `applyDiff` and undo select winners the way Rust does. `revertTo`/`applyDiff` follow Rust's `apply_delta`, and undoing a `move` or `set` reinserts the old value. +- Import and checkout events report the whole state of a child container that a concurrent move brings back. +- `set` records an op even when the value is unchanged, and `move(i, i)` is a no-op, as in Rust. `getIdAt` and `getCursor` of a MovableList anchor the list item, like Rust's cursor. + +Also fixed while making loro.js match Rust: + +- A document loaded from a snapshot replays history before importing changes that are concurrent with the snapshot. Before, List, Text and MovableList could place such changes at the wrong index. +- Shallow documents accept changes rooted at the empty version when the shallow root is empty, skip changes already inside the shallow root, and a shallow snapshot rooted at a peer's first op now writes the start version that Rust expects. +- Importing a shallow snapshot emits events for containers that exist only in the shallow root state. diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index a2119d294..9acc89db8 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -8,12 +8,8 @@ reproduce, the loro.js model that implements them, and how the differential suite checks the two against each other. Complexity rules still come from [loro-js-performance.md](loro-js-performance.md). -Status: the differential suite is in the tree. The "loro.js model" section is -the design of the follow-up change; until it lands, `LoroMovableList` still -uses the old model described under "Why the old loro.js model diverged", and -the suite records those profiles in `KNOWN_DIVERGENT`. The insert/delete-only -profile converges on `main` since the Fugue fixes of loro-dev/loro#1136 and -#1131, so it is not in that set. +Status: the model below is implemented. Import validation and atomic rollback +("Validation and import atomicity") land in follow-up changes. ## Rust semantics (the contract) @@ -129,37 +125,56 @@ fails loudly. winners of the touched elements at the target version and refresh the pointedness of their old and new positions. That costs O((affected ops + affected candidates) · log n), with no replay and no LIS. -- Events come from the same primitive state changes. A position that becomes - visible inserts its value at its new user index. A visible position that - stops being visible deletes at its old user index. A value change on a - visible element is a delete plus an insert at the same index. Transitions - delete in descending before-index order, then insert in ascending after-index - order. +- Events come from the same primitive state changes. Hiding a visible item + deletes it at its user index. A new winning position follows Rust's + `convert_update_to_event_pos`: insert at the new index (plus one when it + follows the old one), then delete the old item, so a move onto the same + slot cancels. A value change is a delete plus an insert. Transitions delete + in descending before-index order, then insert in ascending after-index + order. Rust derives an import's event from the net change while loro.js + composes the ops, so a multi-op import can show an element that was hidden + and shown again as a delete plus an insert of the same value; the effect is + the same. Getting Rust's exact shape would need an O(n) snapshot per import. +- Revival, as in Rust's `DocState::apply_diff`: in an import or checkout + batch, a child container of an element that was hidden when the batch + started and is visible at its end reports its whole state (and so do its + descendants), for example a Text brought back by a concurrent move. `diff()` + does the same. - Snapshot export follows section 8 exactly. Import hydrates alive positions (visible and dead) and visible elements with their real position, element and last-set IDs. Hydrated state has no tombstones, origins or candidate history, so the container is marked incomplete. Version transitions on it - replay history once. An import that is not causally after the hydrated - version (it is concurrent with it) also replays history first: the causal - view of such an op can need tombstones the snapshot dropped. The same rule - fixes List and Text, which had the same gap. + replay history once. `LoroDoc.#snapshotStateVersion` records the hydrated + version; `#needsHistoryReplay` replays history before an import (or an + attach) containing a record that does not depend on that whole version, or a + MovableList move/set naming an element the state lacks. Such an op's causal + view can need tombstones the snapshot dropped. The same rule fixes List and + Text, which had the same gap. `#discardDeferredSnapshotState` (also in + loro.js #1126) drops the lazy state first so it is not hydrated again on top + of the replay. - A shallow root store seeds root-time position and value candidates, the counterpart of `record_shallow_root_state`, so retreat to the root keeps root-time winners (the MovableList part of Rust #1124 / loro.js #1127). ### Undo, `revertTo`, `applyDiff` -`applyDiff` and `revertTo` port Rust's MovableList `apply_delta`: insert -first, defer deletes, and turn a reinserted child container into a `move`. -Undo of MovableList ops applies the container's inverse diff through the same -function, so undoing a `move`/`set` creates a new element with the old value, -as in Rust. loro.js undo still differs from Rust when remote changes -interleave with the undone change (Rust transforms stored diffs; loro.js -replays ops). That applies to List too and is listed under remaining gaps. +`applyDiff` and `revertTo` port Rust's MovableList `apply_delta` +(`applyMovableListDelta` in `document.ts`): insert first, defer deletes, and +turn a reinserted child container into a `move`, following the container +remap chain. + +Undo follows Rust's `undo_internal` for the MovableList containers of an undo +item (`#undoMovableLists`). A is the inverse diff from the span's last op back +to its dependencies, B is the change since the span, and A transformed over B +(A's inserts first, like `transform(.., left_priority = true)`) is applied with +`apply_delta`. Undoing a `move`/`set` therefore inserts the old value as a new +element and deletes the current one, and a child container moves back. Other +containers keep loro.js's op-based undo. B also contains local changes that +Rust's per-item remote diff excludes; see the remaining divergences. ## Validation and import atomicity -Import validation follows Rust #1125. A `Move`/`Set` whose element is unknown, +Planned; not implemented yet. Import validation follows Rust #1125. A `Move`/`Set` whose element is unknown, lives in another container, is outside the op's causal history, or (on a shallow doc) was deleted before the root is rejected with an error. So is a `from`/`to` outside the op-index range of its causal view. A `Move`/`Set` of a @@ -203,48 +218,50 @@ replay and minimization), `movable-list.test.ts` (profiles). A divergence saves its trace to the temp directory; replay it with `LORO_JS_DIFF_REPLAY=` and add `LORO_JS_DIFF_MINIMIZE=1` to shrink it. -## Remaining known divergences (not MovableList-specific) +## Remaining known divergences Found by the differential suite on 2026-09-28. The profiles avoid or normalize -them (see the comments in `harness.ts` and `fuzz.ts`): - -- Undo: loro.js replays inverse ops, while Rust applies stored inverse diffs - transformed by later changes. Results differ for List and MovableList when an - undo targets items that an earlier undo recreated, or when remote changes - interleave. A redo also restores deleted items in deletion order instead of - document order. `UndoManager.undo()` picks its item before committing the - pending transaction. Rust commits redo with origin `"undo"`. The profiles run - undo on one peer, never chained, after an explicit commit. +them; each normalization is commented in `harness.ts` or `fuzz.ts`. + +loro.js, not specific to MovableList: + +- Undo: loro.js replays inverse ops for List, Text, Map and Tree, while Rust + applies stored inverse diffs transformed by later remote changes. Results + differ when an undo targets items that an earlier undo recreated, or when + remote changes interleave. A redo also restores deleted List items in + deletion order instead of document order, `UndoManager.undo()` picks its + item before committing the pending transaction, and Rust commits redo with + origin `"undo"`. The profiles run undo on one peer, never chained, after an + explicit commit. - `checkout(frontiers)` at the latest frontiers left loro.js detached; Rust stays attached. Fixed by loro-dev/loro#1143; the profiles still skip such checkouts. - loro.js emits events and `diff()` entries for child containers whose parent element is already deleted; Rust does not. Only reachable targets are compared. -- A snapshot-hydrated loro.js document has no tombstones or origins. It - mispositions List/Text/MovableList ops that are concurrent with deletes the - snapshot already contains (loro-dev/loro#1163). -- Some shallow-snapshot imports that Rust accepts are rejected by loro.js with - "cannot import updates that depend on an outdated version". -- Multi-blob `importBatch` events are labeled `by: "checkout"` by Rust and - `"import"` by loro.js; checkout events carry origin `"checkout"` only in - Rust. Rust's `oplogVersion()` also counts the pending transaction. - -Differences in form, not in effect, which the suite checks by value: - -- `revertTo` applies same-depth containers in Rust's `FxHashMap` order, and - its list ops follow the insert/delete order of `diff(current, target)` at one - index, which Rust's `DeltaRope` composition decides. When either differs, - the suite checks the revert by value and replicates Rust's ops. -- Event deltas can differ in shape, or be a no-op where Rust reports nothing; - the suite then checks that each engine's deltas turn the previous value into - the new one. - -Rust issues the suite works around: - -- On a shallow document, `getLastEditorAt` can report the root list item's or - the last mover's peer instead of the setter's (`record_shallow_root_state` - in `history_cache.rs` seeds the value writer with the list item ID). The - suite does not compare editors on shallow documents. -- `updates-in-range` on a shallow document exports nothing for a span that - starts in trimmed history (`ChangeStore::iter_blocks`). The suite starts - such spans at the shallow root. +- Cursors: loro.js List/Text add one to a side-1 cursor's offset and anchor an + end cursor to the last item; Rust does neither, and resolves a MovableList + cursor whose item was moved away to `undefined`. Not compared. +- Event labels: multi-blob `importBatch` is `by: "checkout"` in Rust, and + checkout events carry origin `"checkout"` only in Rust. Rust's + `oplogVersion()` also counts the pending transaction. + +Differences in form, not in effect: + +- `revertTo`/`applyDiff` apply same-depth containers in Rust's `FxHashMap` + order over arena indices, so the two engines write the same values with a + different op order when several same-depth containers change. The suite + checks such a revert by value on a fork and replicates Rust's ops. +- Event deltas of multi-op imports and of history replays can differ in shape + (see "Versions, events, snapshots"); the suite then checks that each + engine's delta turns the previous value into the new one. + +Rust issues found: + +- On a shallow document, `checkout(root)` reports `getLastEditorAt` from the + root list item's peer: `record_shallow_root_state` (`history_cache.rs`) + seeds the value writer with the list item ID. The full history reports the + real setter, and so does loro.js. +- An update whose changes partly depend on history before the shallow root + applies the other changes and then returns + `ImportUpdatesThatDependsOnOutdatedVersion`, so the import is not atomic. + loro.js rejects the whole blob; the suite retires such a peer. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 474a4a631..fd8a8b100 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -63,12 +63,17 @@ JavaScript constant factor. imports apply only newly integrated records. - Retreat and comparable-version transitions toggle only the affected sequence elements, map keys, tree nodes, counters, text-style entries, and - movable-list values. Map and Tree winner lookup uses per-subject/per-peer - arrays with binary search. MovableList moves retain before/after neighbor - anchors and an operation history per container. Direct switches between - concurrent move branches replay only the affected container's order history, - then apply the minimum move set selected by a longest-increasing-subsequence - pass. Unrelated document history and container state are not rebuilt. + MovableList positions and elements. Map and Tree winner lookup uses + per-subject/per-peer arrays with binary search. A MovableList + (`movable-list.ts`, [loro-js-movable-list.md](loro-js-movable-list.md)) + keeps a Fugue `SequenceIndex` of positions whose 0/1 metric marks the + user-visible ones, so user index ↔ op index ↔ position lookups are + O(log n). A version switch toggles the positions that the switched ops + created or deleted, then reselects each touched element's winning position + and value from its candidates, newest first. It costs + O((affected ops + skipped candidates) · log n), like Rust's `last_pos` scan, + with no replay. Unrelated document history and container state are not + rebuilt. - Contiguous Text/List insertion and deletion transitions reuse the physical ID runs and reversible lazy subtree visibility in both directions. Without an event subscriber, hiding or showing one complete run is expected O(log n + @@ -521,10 +526,11 @@ The remaining differences are representation and JavaScript constant factors: restored text/list values in its event, so its work is proportional to that emitted output. Without a subscriber, both hide and show transitions use the reversible lazy visibility layer and stay proportional to affected ID runs. -- Importing interleaved concurrent MovableList moves can canonicalize the - affected container once. Initial snapshot hydration and fallback transitions - with incomplete history can likewise materialize complete touched containers - when their returned state or subscriber event requires it. +- State hydrated from a latest-state snapshot has no tombstones or MovableList + candidate history. The first import that is concurrent with that state, or + names a MovableList element it lacks, replays history once + (`#needsHistoryReplay`); so do version transitions on such a MovableList. + Later imports and transitions are incremental again. - The million-operation C1.1 concurrent-text trace still exposes a large constant-factor and retained-memory gap. Its local edit phase is about 4x the WASM adapter, and parsing the 6.5 MB snapshot takes 3.62 seconds versus 43 ms. diff --git a/crates/loro/tests/loro_js_interop.rs b/crates/loro/tests/loro_js_interop.rs index 60fb124b3..371b90fee 100644 --- a/crates/loro/tests/loro_js_interop.rs +++ b/crates/loro/tests/loro_js_interop.rs @@ -49,6 +49,14 @@ const TS_RICHTEXT_SNAPSHOT: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/richtext-snapshot.ts.blob"); const TS_RICHTEXT_EXPECTED_JSON: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/richtext.expected.json"); +const TS_MOVABLE_SNAPSHOT: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/movable-snapshot.ts.blob"); +const RUST_MOVABLE_UPDATES: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/movable-snapshot-updates.blob"); +const RUST_MOVABLE_PEER3: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/movable-snapshot-peer3.blob"); +const RUST_MOVABLE_PEER4: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/movable-snapshot-peer4.blob"); const TS_CURSOR: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/cursor.ts.blob"); const TS_AWARENESS: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/awareness.ts.blob"); @@ -368,3 +376,36 @@ fn imports_legacy_typescript_tree_snapshots_with_unordered_siblings() { ); } } + +fn movable_list_metadata(doc: &LoroDoc) -> Vec<(Option, Option, Option)> { + let list = doc.get_movable_list("list"); + (0..list.len()) + .map(|index| { + ( + list.get_creator_at(index), + list.get_last_mover_at(index), + list.get_last_editor_at(index), + ) + }) + .collect() +} + +/// loro.js writes a MovableList snapshot with its list item, element and +/// last-set IDs, so later edits converge with a replica that has the history. +#[test] +fn imports_typescript_movable_list_snapshot_with_its_metadata() { + let expected = LoroDoc::new(); + expected.import(RUST_MOVABLE_UPDATES).unwrap(); + let doc = LoroDoc::new(); + doc.import(TS_MOVABLE_SNAPSHOT) + .expect("Rust should import a MovableList snapshot produced by loro.js"); + assert_eq!(doc.get_deep_value(), expected.get_deep_value()); + assert_eq!(movable_list_metadata(&doc), movable_list_metadata(&expected)); + + for update in [RUST_MOVABLE_PEER4, RUST_MOVABLE_PEER3] { + doc.import(update).unwrap(); + expected.import(update).unwrap(); + } + assert_eq!(doc.get_deep_value(), expected.get_deep_value()); + assert_eq!(movable_list_metadata(&doc), movable_list_metadata(&expected)); +} diff --git a/loro-js/scripts/rewrite-rust-fixture.mjs b/loro-js/scripts/rewrite-rust-fixture.mjs index 616c0820d..5cb013df3 100644 --- a/loro-js/scripts/rewrite-rust-fixture.mjs +++ b/loro-js/scripts/rewrite-rust-fixture.mjs @@ -213,6 +213,15 @@ writeFileSync( treeLeft.export({ mode: "shallow-snapshot", frontiers: treeShallowRoot }), ); +// MovableList state written by loro.js: moved elements, sets by other peers, +// and a dead position left by concurrent moves (see movable-snapshot.blob). +const movableFromRust = new LoroDoc(); +movableFromRust.import(fixture("movable-snapshot-updates.blob")); +writeFileSync( + fixtureUrl("movable-snapshot.ts.blob"), + movableFromRust.export({ mode: "snapshot" }), +); + const cursorDoc = new LoroDoc(); cursorDoc.setPeerId(99); cursorDoc.getText("text").insert(0, "abc"); diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 7bf15d5d6..8c5e85547 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -8,6 +8,7 @@ import type { ContainerId as CodecContainerId, Id as CodecId } from "../codec/ty import { formatContainerId, formatTreeId, parseContainerId } from "./ids"; import type { LoroDoc } from "./document"; import { fractionalIndexBetween, fractionalIndexesBetween } from "./fractional-index"; +import { MovableListState, type MovableElement } from "./movable-list"; import { OrderedIndex } from "./ordered-index"; import { SequenceIndex, SequenceSpan } from "./sequence-index"; import type { SequenceIdRun } from "./sequence-index"; @@ -53,32 +54,6 @@ export interface SequenceElement { deletedBy?: CodecId[] | undefined; deletedByPeer?: bigint | undefined; deletedByCounter?: number | undefined; - valueHistory?: SequenceValueMeta[] | undefined; - moveHistory?: SequenceMoveMeta[] | undefined; -} - -export interface SequenceValueMeta { - readonly id: CodecId; - readonly lamport: number; - readonly value: RuntimeValue; -} - -export interface SequenceMoveMeta { - readonly id: CodecId; - readonly lamport: number; - readonly beforePrevious: CodecId | undefined; - readonly beforeNext: CodecId | undefined; - readonly afterPrevious: CodecId | undefined; - readonly afterNext: CodecId | undefined; - /** - * The element physically after the moved one before the move, deleted or - * not; `null` when it was last. Undo puts the element back there, so undone - * moves and restored deletions keep their relative order. Undefined when - * unknown (e.g. metadata rebuilt from a snapshot). - */ - readonly beforePhysicalNext?: CodecId | null | undefined; - /** The element physically before it, deleted or not; `null` when it was first. */ - readonly beforePhysicalPrevious?: CodecId | null | undefined; } export type CausalVersion = ReadonlyMap; @@ -97,9 +72,8 @@ interface ListState extends SequenceContainerState { readonly detachedCounter: number; } -interface MovableListState extends ListState { - readonly valueHistoryComplete: boolean; - readonly moveHistoryComplete: boolean; +interface MovableListSwapState extends SequenceContainerState { + readonly movable: MovableListState; } interface TextState extends SequenceContainerState { @@ -116,6 +90,7 @@ interface ParentLink { readonly binding?: | { readonly kind: "map"; readonly key: string } | { readonly kind: "sequence"; readonly element: SequenceElement } + | { readonly kind: "movable"; readonly element: MovableElement } | { readonly kind: "tree"; readonly record: TreeNodeRecord }; } @@ -549,9 +524,6 @@ export class LoroList extends LoroContainer { deleted: false, originLeft: undefined, originRight: undefined, - ...(this instanceof LoroMovableList - ? { valueHistory: [{ id, lamport, value }] } - : {}), }; }); this._sequence.insertAtVisible(position, elements); @@ -575,19 +547,9 @@ export class LoroList extends LoroContainer { deleted: false, originLeft: undefined, originRight: undefined, - ...(this instanceof LoroMovableList - ? { valueHistory: [{ id, lamport: elementLamport, value }] } - : {}), }; }); - insertFugueElements( - this._sequence, - position, - elements, - causalVersion, - // Moves break the origin-tree preorder used by the direct-child index. - !(this instanceof LoroMovableList), - ); + insertFugueElements(this._sequence, position, elements, causalVersion); this._bindChildren(elements); } @@ -759,8 +721,6 @@ class TextSequenceSpan extends SequenceSpan { #attributes: Map> | undefined; #attributeMeta: Map> | undefined; #attributeHistory: Map> | undefined; - #valueHistory: Map | undefined; - #moveHistory: Map | undefined; static fromText( text: string, @@ -816,8 +776,6 @@ class TextSequenceSpan extends SequenceSpan { offset, element.attributeHistory, ); - span.#valueHistory = setOffsetMap(span.#valueHistory, offset, element.valueHistory); - span.#moveHistory = setOffsetMap(span.#moveHistory, offset, element.moveHistory); span.retain(offset, element); } span.#deletedBits >>>= 0; @@ -934,8 +892,6 @@ class TextSequenceSpan extends SequenceSpan { output.#attributes = copyOffsetMap(this.#attributes, start, end); output.#attributeMeta = copyOffsetMap(this.#attributeMeta, start, end); output.#attributeHistory = copyOffsetMap(this.#attributeHistory, start, end); - output.#valueHistory = copyOffsetMap(this.#valueHistory, start, end); - output.#moveHistory = copyOffsetMap(this.#moveHistory, start, end); return output; } @@ -986,12 +942,6 @@ class TextSequenceSpan extends SequenceSpan { other.#attributeHistory, oldLength, ); - this.#valueHistory = appendOffsetMap( - this.#valueHistory, - other.#valueHistory, - oldLength, - ); - this.#moveHistory = appendOffsetMap(this.#moveHistory, other.#moveHistory, oldLength); return true; } @@ -1107,22 +1057,6 @@ class TextSequenceSpan extends SequenceSpan { this.#attributeHistory = setOffsetMap(this.#attributeHistory, offset, value); } - valueHistoryAt(offset: number): SequenceValueMeta[] | undefined { - return this.#valueHistory?.get(offset); - } - - setValueHistoryAt(offset: number, value: SequenceValueMeta[] | undefined): void { - this.#valueHistory = setOffsetMap(this.#valueHistory, offset, value); - } - - moveHistoryAt(offset: number): SequenceMoveMeta[] | undefined { - return this.#moveHistory?.get(offset); - } - - setMoveHistoryAt(offset: number, value: SequenceMoveMeta[] | undefined): void { - this.#moveHistory = setOffsetMap(this.#moveHistory, offset, value); - } - #number(offset: number, column: number): number { if (this.#numbers !== undefined) { return this.#numbers[offset * TEXT_NUMBER_STRIDE + column]!; @@ -1348,22 +1282,6 @@ class PackedTextElement implements TextElement, CodecId { this.#span.setAttributeHistoryAt(this.#offset, value); } - get valueHistory(): SequenceValueMeta[] | undefined { - return this.#span.valueHistoryAt(this.#offset); - } - - set valueHistory(value: SequenceValueMeta[] | undefined) { - this.#span.setValueHistoryAt(this.#offset, value); - } - - get moveHistory(): SequenceMoveMeta[] | undefined { - return this.#span.moveHistoryAt(this.#offset); - } - - set moveHistory(value: SequenceMoveMeta[] | undefined) { - this.#span.setMoveHistoryAt(this.#offset, value); - } - _retarget(span: TextSequenceSpan, offset: number): void { this.#span = span; this.#offset = offset; @@ -2611,21 +2529,121 @@ function styleValuesEqual(left: unknown, right: unknown): boolean { } export class LoroMovableList extends LoroList { - _valueHistoryComplete = true; - _moveHistoryComplete = true; + _state: MovableListState = new MovableListState((element) => + this._bindElement(element), + ); + /** Values of a list that is not attached to a document. */ + _detachedValues: RuntimeValue[] = []; + + constructor() { + super(); + // A MovableList keeps positions and elements in `_state`; the inherited + // element sequence stays empty. Fail loudly if a list-only path reads it. + Object.defineProperty(this, "_sequence", { + get(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); + }, + }); + } kind(): "MovableList" { return "MovableList"; } + override get _elements(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); + } + + override get length(): number { + if (this._doc === undefined) return this._detachedValues.length; + this._ensureHydrated(); + return this._state.length; + } + + override get(index: number): T | undefined { + if (this._doc === undefined) { + return cloneRuntimeValue(this._detachedValues[index]) as T | undefined; + } + this._ensureHydrated(); + return cloneRuntimeValue(this._state.elementAt(index)?.value) as T | undefined; + } + + override toArray(): T[] { + return this._rawValues().map((value) => cloneRuntimeValue(value)) as T[]; + } + + override toJSON(): unknown[] { + return this._rawValues().map((value) => runtimeValueToJson(value)); + } + + override getShallowValue(): unknown[] { + return this._rawValues().map((value) => runtimeValueToShallow(value)); + } + + override insert(pos: number, value: T): void { + this._validateInsertPosition(pos); + if (isContainer(value)) + throw new TypeError("use insertContainer() to attach a child container"); + if (this._doc === undefined) { + this._detachedValues.splice(pos, 0, normalizeDetachedValue(value)); + return; + } + this._doc._listInsert(this, pos, value); + } + + override delete(pos: number, len: number): void { + validateRange(pos, len, this.length); + if (len === 0) return; + if (this._doc === undefined) { + this._detachedValues.splice(pos, len); + return; + } + this._doc._sequenceDelete(this, pos, len); + } + + override insertContainer(pos: number, child: C): C { + this._validateInsertPosition(pos); + if (this._doc === undefined) { + this._detachedValues.splice(pos, 0, child); + child._parentLink = { container: this }; + return child; + } + return this._doc._listInsertContainer(this, pos, child); + } + + /** The ID of the list item (position) at `pos`, which a cursor also anchors. */ + override getIdAt(pos: number): { peer: string; counter: number } | undefined { + if (this._doc === undefined) return undefined; + this._ensureHydrated(); + const id = this._state.positionAt(pos)?.id; + return id === undefined + ? undefined + : { peer: id.peer.toString(), counter: id.counter }; + } + + /** Like Rust, a cursor anchors the list item ID, which a move leaves behind. */ + override getCursor(pos: number, side: Side = 0): Cursor | undefined { + if (!Number.isSafeInteger(pos) || pos < 0 || this._doc === undefined) + return undefined; + this._ensureHydrated(); + const length = this._state.length; + if (length === 0) return new Cursor(this.id, undefined, side === 0 ? -1 : side, 0); + if (pos >= length) return new Cursor(this.id, undefined, 1, length); + return new Cursor(this.id, this._state.positionAt(pos)!.id, side, pos); + } + move(from: number, to: number): void { + // Like Rust's attached `mov`, moving an index onto itself is a no-op even + // when it is out of range. + if (this._doc !== undefined && from === to && Number.isSafeInteger(from)) return; validateIndex(from, this.length); if (!Number.isSafeInteger(to) || to < 0 || to >= this.length) { throw new RangeError(`movable-list destination ${to} is out of range`); } if (from === to) return; if (this._doc === undefined) { - this._applyMove(from, to); + const [value] = this._detachedValues.splice(from, 1); + this._detachedValues.splice(to, 0, value!); return; } this._doc._movableMove(this, from, to); @@ -2640,7 +2658,7 @@ export class LoroMovableList extends LoroList { if (isContainer(value)) throw new TypeError("use setContainer() to attach a child container"); if (this._doc === undefined) { - this._sequence.atVisible(pos)!.value = normalizeDetachedValue(value); + this._detachedValues[pos] = normalizeDetachedValue(value); return; } this._doc._movableSet(this, pos, value); @@ -2649,163 +2667,95 @@ export class LoroMovableList extends LoroList { setContainer(pos: number, child: C): C { validateIndex(pos, this.length); if (this._doc === undefined) { - const element = this._sequence.atVisible(pos)!; - element.value = child; - this._bindChildren([element]); + this._detachedValues[pos] = child; + child._parentLink = { container: this }; return child; } return this._doc._movableSetContainer(this, pos, child); } getCreatorAt(pos: number): string | undefined { - return this._sequence.atVisible(pos)?.id.peer.toString(); + if (this._doc === undefined) return undefined; + this._ensureHydrated(); + return this._state.elementAt(pos)?.peer.toString(); } getLastMoverAt(pos: number): string | undefined { - return this._sequence.atVisible(pos)?.id.peer.toString(); + if (this._doc === undefined) return undefined; + this._ensureHydrated(); + return this._state.positionAt(pos)?.id.peer.toString(); } getLastEditorAt(pos: number): string | undefined { - return this._sequence.atVisible(pos)?.id.peer.toString(); + if (this._doc === undefined) return undefined; + this._ensureHydrated(); + return this._state.elementAt(pos)?.valueWriter.peer.toString(); } - /** Physical placement for the next `_applyMove` of `element` (used by undo). */ - _physicalMoveHint: - | { readonly element: SequenceElement; readonly before: SequenceElement | undefined } - | undefined; + /** Current values without cloning, attached or not. */ + _rawValues(): RuntimeValue[] { + if (this._doc === undefined) return this._detachedValues; + this._ensureHydrated(); + return this._state.visibleElements().map((element) => element.value); + } - _applyMove( - from: number, - to: number, - operation?: Pick, - replaceExisting = false, - ): void { - const element = this._sequence.atVisible(from); - if (element === undefined) return; - const beforePrevious = this._sequence.previousVisible(element)?.id; - const beforeNext = this._sequence.nextVisible(element)?.id; - const physical = this._sequence.physicalIndexOf(element); - const beforePhysicalNext = - physical === undefined - ? undefined - : (this._sequence.atPhysical(physical + 1)?.id ?? null); - const beforePhysicalPrevious = - physical === undefined - ? undefined - : physical === 0 - ? null - : (this._sequence.atPhysical(physical - 1)?.id ?? null); - const hint = this._physicalMoveHint; - this._physicalMoveHint = undefined; - if (hint !== undefined && hint.element === element) { - this._sequence.moveBefore(element, hint.before); - } else { - this._sequence.moveVisible(from, to); - } - if (operation === undefined) return; - const meta: SequenceMoveMeta = { - ...operation, - beforePrevious, - beforeNext, - beforePhysicalNext, - beforePhysicalPrevious, - afterPrevious: this._sequence.previousVisible(element)?.id, - afterNext: this._sequence.nextVisible(element)?.id, - }; - let history = element.moveHistory; - if (history === undefined) { - history = []; - element.moveHistory = history; - } - const index = lowerBoundSequenceMoveMeta(history, meta); - const existing = history[index]; - if ( - existing !== undefined && - existing.id.peer === meta.id.peer && - existing.id.counter === meta.id.counter - ) { - if (replaceExisting) history[index] = meta; - } else { - history.splice(index, 0, meta); - } + override _visibleElements(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); } - _moveToAnchors( - element: SequenceElement, - previousId: CodecId | undefined, - nextId: CodecId | undefined, - ): void { - if (element.deleted) return; - if (nextId === undefined) { - if (this._sequence.nextVisible(element) !== undefined) { - this._sequence.moveBefore(element, undefined); - } - return; - } - const next = this._sequence.findById(nextId); - if (next !== undefined && !next.deleted && next !== element) { - this._sequence.moveBefore(element, next); - return; - } - const previous = - previousId === undefined ? undefined : this._sequence.findById(previousId); - if (previous !== undefined && !previous.deleted && previous !== element) { - const successor = this._sequence.nextVisible(previous); - if (successor !== element) this._sequence.moveBefore(element, successor); - return; - } - if (previousId === undefined) { - const first = this._sequence.atVisible(0); - if (first !== undefined && first !== element) { - this._sequence.moveBefore(element, first); - } - } + override _visibleElementsRange(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); } - _applySet( - element: SequenceElement, - value: RuntimeValue, - meta?: SequenceValueMeta, - ): void { - if (meta === undefined) { - element.value = value; - this._bindChildren([element]); - return; - } - let history = element.valueHistory; - if (history === undefined) { - history = []; - element.valueHistory = history; + override _visibleElementAt(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); + } + + override _valuesRange(start: number, end: number): unknown[] { + if (this._doc === undefined) { + return this._detachedValues + .slice(start, end) + .map((value) => cloneRuntimeValue(value)); } - const index = lowerBoundSequenceValueMeta(history, meta); - const existing = history[index]; - if ( - existing === undefined || - existing.id.peer !== meta.id.peer || - existing.id.counter !== meta.id.counter - ) { - history.splice(index, 0, meta); + this._ensureHydrated(); + return this._state + .visibleElementsRange(start, end) + .map((element) => cloneRuntimeValue(element.value)); + } + + override _insertVisible(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); + } + + override _insertFugue(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); + } + + override _deleteIdSpan(): never { + throw new Error("LoroMovableList has no element sequence; use _state"); + } + + _bindElement(element: MovableElement): void { + if (element.value instanceof LoroContainer) { + element.value._setParentBinding(this, { kind: "movable", element }); } - const winner = history.at(-1)!; - element.value = winner.value; - this._bindChildren([element]); } override _reset(): void { - super._reset(); - this._valueHistoryComplete = true; - this._moveHistoryComplete = true; + this._state.reset(); + this._detachedValues = []; } + /** Installs `state`, or an empty state, and returns the replaced state. */ override _swapState(state?: SequenceContainerState): SequenceContainerState { - const previous: MovableListState = { - ...(super._swapState(state) as ListState), - valueHistoryComplete: this._valueHistoryComplete, - moveHistoryComplete: this._moveHistoryComplete, + const previous: MovableListSwapState = { + sequence: this._state.positions as unknown as SequenceIndex, + movable: this._state, }; - const next = state as MovableListState | undefined; - this._valueHistoryComplete = next?.valueHistoryComplete ?? true; - this._moveHistoryComplete = next?.moveHistoryComplete ?? true; + this._state = + (state as MovableListSwapState | undefined)?.movable ?? + new MovableListState((element) => this._bindElement(element)); + for (const element of this._state.allElements()) this._bindElement(element); return previous; } } @@ -3519,6 +3469,9 @@ function containerValueWithId(container: Container): unknown { }; return container._childrenOf(undefined).map(visit); } + if (container instanceof LoroMovableList) { + return container._rawValues().map((value) => runtimeValueDeepWithId(value)); + } return container ._visibleElements() .map((element) => runtimeValueDeepWithId(element.value)); @@ -3572,7 +3525,7 @@ function normalizeDetachedValue(value: unknown): RuntimeValue { throw new TypeError(`unsupported Loro value type: ${typeof value}`); } -function insertFugueElements( +export function insertFugueElements( sequence: SequenceIndex, position: number, inserted: T[], @@ -4041,49 +3994,6 @@ function compareWriters(left: LastWriter, right: LastWriter): number { ); } -function compareSequenceValueMeta( - left: SequenceValueMeta, - right: SequenceValueMeta, -): number { - return compareWriters( - { peer: left.id.peer, lamport: left.lamport }, - { peer: right.id.peer, lamport: right.lamport }, - ); -} - -function lowerBoundSequenceValueMeta( - history: readonly SequenceValueMeta[], - meta: SequenceValueMeta, -): number { - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (compareSequenceValueMeta(history[middle]!, meta) < 0) low = middle + 1; - else high = middle; - } - return low; -} - -function lowerBoundSequenceMoveMeta( - history: readonly SequenceMoveMeta[], - meta: SequenceMoveMeta, -): number { - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - const current = history[middle]!; - const order = compareWriters( - { peer: current.id.peer, lamport: current.lamport }, - { peer: meta.id.peer, lamport: meta.lamport }, - ); - if (order < 0) low = middle + 1; - else high = middle; - } - return low; -} - function validateRange(position: number, length: number, total: number): void { if ( !Number.isSafeInteger(position) || diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index c6966592f..58262b3bc 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -67,13 +67,17 @@ import { type MapRecord, type RuntimeValue, type SequenceElement, - type SequenceMoveMeta, - type SequenceValueMeta, type TextElement, type TextStyle, type TreeNodeRecord, } from "./containers"; import { SequenceEventDiff } from "./event-diff"; +import { + sameMovableListStates, + type MovableElement, + type MovableListEvents, + type MovableTransitionOp, +} from "./movable-list"; import { codecTypeToPublic, containerIdsEqual, @@ -194,8 +198,6 @@ interface EventRecording { readonly eventStates: Map; } -type MovableMoveTransitionMode = "anchors" | "replay"; - type SnapshotSequenceState = // The snapshot state of `version` is installed. It has no history for the // operations of `version`; operations applied since then are indexed. @@ -284,8 +286,6 @@ export class LoroDoc = Record>(); #mapOperationHistory = new Map>(); #treeOperationHistory = new Map>(); - #movableOrderHistory = new Map>(); - #movableMovePeers = new Map>(); #containersWithOperations = new Set(); #containerKeys = new WeakMap(); #pendingHistory = new Map(); @@ -708,11 +708,11 @@ export class LoroDoc = Record 0 && !this.#detached) { + this.#prepareSnapshotImport(added, beforeVersion); const recording = this.#hasEventSubscribers() ? { beforeValues: new Map(), eventStates: new Map() } : undefined; this.#applyRecords(added, recording); - this.#canonicalizeImportedMovableMoves(added, recording); if (recording !== undefined) { beforeValues = recording.beforeValues; preparedDiffs = this.#recordedEventDiffs(recording); @@ -901,17 +901,25 @@ export class LoroDoc = Record 0) { + this.#prepareSnapshotImport(integration.added, beforeVersion); const recording = this.#hasEventSubscribers() ? { beforeValues: new Map(), eventStates: new Map() } : undefined; this.#applyRecords(integration.added, recording); - this.#canonicalizeImportedMovableMoves(integration.added, recording); if (recording !== undefined) { beforeValues = recording.beforeValues; preparedDiffs = this.#recordedEventDiffs(recording); @@ -946,6 +954,7 @@ export class LoroDoc = Record this.#decodeImportData(decodeDocument(blob))) .sort( @@ -1015,15 +1024,15 @@ export class LoroDoc = Record = Record = Record = Record(); let mapKeysAtFrom = new Map>(); let mapKeysAtTo = new Map>(); + let revived: LoroContainer[] = []; if (useIncrementalTransition || retreatRecords.length === 0) { const recording: EventRecording = { beforeValues: new Map(), @@ -1224,7 +1216,6 @@ export class LoroDoc = Record = Record>(); @@ -1292,13 +1292,7 @@ export class LoroDoc = Record = Record = Record(); + const moved = new Map(); for (const sourceId of insertedChildren) { let id: ContainerID | undefined = sourceId; let child = this.#containers.get(id); @@ -1402,9 +1394,8 @@ export class LoroDoc = Record = Record(); - const movedElements = new Set(); + const movedElements = new Set(); let predecessor: Predecessor | undefined; index = 0; for (const operation of delta) { @@ -1433,7 +1424,7 @@ export class LoroDoc = Record = Record = Record = Record id.counter < (version.get(id.peer) ?? 0); + if (parent instanceof LoroMovableList) { + if (binding?.kind !== "movable") return false; + return parent._state.visibleValueAt(binding.element, included)?.value === child; + } if (binding?.kind !== "sequence") return false; const element = binding.element; - const included = (id: CodecId): boolean => id.counter < (version.get(id.peer) ?? 0); if (!included(element.id)) return false; - if (parent._sequence.someDeletion(element, included)) return false; - const childId = child._codecId!; - if (childId.kind !== "normal") return false; - const value = latestIncludedSequenceValue(element.valueHistory, version); - const valueId = value === undefined ? element.id : value.id; - return valueId.peer === childId.peer && valueId.counter === childId.counter; + return !parent._sequence.someDeletion(element, included); } applyDiff( @@ -3147,22 +3137,11 @@ export class LoroDoc = Record = Record @@ -3446,6 +3444,11 @@ export class LoroDoc = Record = Record= writtenFrom) || isTracked(id); const parsedPeer = parsePeerId(peer); const records = this.#historyByPeer.get(parsedPeer) ?? []; + const movableLists = new Set(); + const operations: { record: HistoryRecord; operation: DecodedOperation }[] = []; let recordIndex = Math.min( records.length - 1, lowerBoundHistory(records, range.end) - 1, @@ -3495,9 +3500,136 @@ export class LoroDoc = Record= 0; operationIndex -= 1) { const operation = record.change.operations[operationIndex]!; if (operation.counter + operation.length <= range.start) break; - this.#undoOperation(record, operation, parsedPeer, range, tracked); + const container = this.#containers.get(formatContainerId(operation.container)); + if (container instanceof LoroMovableList) movableLists.add(container); + else operations.push({ record, operation }); + } + } + if (movableLists.size > 0) { + this.#undoMovableLists(parsedPeer, range, movableLists, tracked); + } + for (const { record, operation } of operations) { + this.#undoOperation(record, operation, parsedPeer, range, tracked); + } + } + + /** + * Undoes MovableList changes the way Rust's `undo_internal` does: apply the + * inverse diff of the span, transformed over what changed since, through + * `apply_delta`. Undoing a move or set therefore reinserts the old value as a + * new element, and moves a child container back. + */ + #undoMovableLists( + peer: bigint, + range: CounterSpan, + lists: ReadonlySet, + isTracked: (id: CodecId) => boolean, + ): void { + const first = this.#recordContaining({ peer, counter: range.start }); + if (first === undefined) return; + const deps = + range.start > first.change.id.counter + ? [{ peer, counter: range.start - 1 }] + : first.change.dependencies; + const spanEnd: Frontiers = [ + { peer: peer.toString() as PeerID, counter: range.end - 1 }, + ]; + const inverse = new Map(this.diff(spanEnd, deps.map(formatOpId), false)); + // Rust transforms the inverse only over changes this UndoManager did not + // make (its remote diff), so a list's inverse is transformed only when such + // a change touched that list. + const spanEndVersion = this.#versionForExistingFrontiers(spanEnd); + const untrackedLists = new Set(); + for (const { change } of this.#recordsInVersionRange( + spanEndVersion, + this.#historyVersion(), + )) { + for (const operation of change.operations) { + const first = { peer: change.id.peer, counter: operation.counter }; + const last = { ...first, counter: operation.counter + operation.length - 1 }; + if (!isTracked(first) || !isTracked(last)) { + untrackedLists.add(this.#containerKey(operation.container)); + } } } + const since = new Map( + [...lists].some((list) => untrackedLists.has(list.id)) + ? this.diff(spanEnd, this.frontiers(), false) + : [], + ); + const remap = new Map(); + for (const list of lists) { + if (this._isContainerDeleted(list)) continue; + const undo = inverse.get(list.id); + if (undo?.type !== "list") continue; + const later = untrackedLists.has(list.id) ? since.get(list.id) : undefined; + const delta = + later?.type === "list" ? transformListDelta(undo.diff, later.diff) : undo.diff; + this.#applyContainerDiff( + list, + { type: "list", diff: this.#withoutLiveChildInserts(list, delta, remap) }, + remap, + new Map(), + ); + } + // Reinserted child containers get their whole state from the inverse diff. + for (const [id, diff] of inverse) { + const child = remap.get(id); + if (child !== undefined) this.#applyContainerDiff(child, diff, remap, new Map()); + } + } + + /** + * Rust's `apply_delta` skips a `from_move` insert whose child still exists + * elsewhere in the list; undo transforms leave such inserts behind when a + * later change moved the child (context/movable-list-apply-diff.md). loro.js + * deltas carry no `from_move`, so for undo an inserted child that is still + * visible outside the delta's deleted ranges is dropped instead. + */ + #withoutLiveChildInserts( + list: LoroMovableList, + delta: readonly Delta[], + remap: ReadonlyMap, + ): Delta[] { + const deleted: { readonly start: number; readonly end: number }[] = []; + let index = 0; + for (const item of delta) { + if ("retain" in item) index += item.retain; + else if ("delete" in item) { + deleted.push({ start: index, end: index + item.delete }); + index += item.delete; + } + } + const live = (value: unknown): boolean => { + let id = diffContainerId(value); + let child = id === undefined ? undefined : this.#containers.get(id); + for ( + let hops = 0; + child === undefined && id !== undefined && hops < 64; + hops += 1 + ) { + id = remap.get(id)?.id; + child = id === undefined ? undefined : this.#containers.get(id); + } + if (child === undefined || child.parent() !== list) return false; + const binding = child._parentLink?.binding ?? recoverParentBinding(child, list); + if (binding?.kind !== "movable" || binding.element.value !== child) return false; + const position = list._state.indexOf(binding.element); + return ( + position !== undefined && + !deleted.some(({ start, end }) => position >= start && position < end) + ); + }; + const output: Delta[] = []; + for (const item of delta) { + if (!("insert" in item)) { + output.push(item); + continue; + } + const kept = item.insert.filter((value) => !live(value)); + if (kept.length > 0) output.push({ ...item, insert: kept }); + } + return output; } _transformUndoCursors(cursors: readonly Cursor[]): Cursor[] { @@ -3523,7 +3655,9 @@ export class LoroDoc = Record boolean, + // Only the old MovableList move undo read this; MovableList undo now + // applies Rust's inverse diff (#undoMovableLists). + _isTracked: (id: CodecId) => boolean, ): void { const container = this.#containers.get(formatContainerId(operation.container)); if (container === undefined) return; @@ -3585,17 +3719,6 @@ export class LoroDoc = Record = Record, - peer: bigint, - counter: number, - isTracked: (id: CodecId) => boolean, - ): void { - const element = list._sequence.findByLamport( - content.elementId.peer, - content.elementId.lamport, - ); - if (element === undefined || element.deleted) return; - const history = element.moveHistory ?? []; - let metaIndex = history.length - 1; - while ( - metaIndex >= 0 && - (history[metaIndex]!.id.peer !== peer || history[metaIndex]!.id.counter !== counter) - ) { - metaIndex -= 1; - } - if (metaIndex < 0) return; - // Later moves the UndoManager tracks (edits it recorded and undid, and the - // inverse ops it wrote) do not block undoing this one. Any other later - // move, remote or from an excluded origin, keeps its position, as in Rust. - for (let later = metaIndex + 1; later < history.length; later += 1) { - if (!isTracked(history[later]!.id)) return; - } - const meta = history[metaIndex]!; - const from = list._sequence.visibleIndexOf(element)!; - // Put the element back into its old physical slot, next to deleted - // neighbors too, so a later restore of those keeps the relative order. The - // slot is next to the physical predecessor unless that has since been - // moved by an untracked op, else before the successor under the same rule, - // as Rust keeps an element where later remote moves leave it. - const slotNeighbor = ( - id: CodecId | null | undefined, - ): { found: boolean; element: SequenceElement | undefined } => { - if (id === undefined) return { found: false, element: undefined }; - if (id === null) return { found: true, element: undefined }; - const neighbor = list._sequence.findById(id); - if (neighbor === undefined) return { found: false, element: undefined }; - const movedSince = (neighbor.moveHistory ?? []).some( - (move) => move.lamport > meta.lamport && !isTracked(move.id), - ); - return movedSince - ? { found: false, element: undefined } - : { found: true, element: neighbor }; - }; - const physicalAfter = ( - anchor: SequenceElement | undefined, - ): SequenceElement | undefined => { - let index = anchor === undefined ? 0 : list._sequence.physicalIndexOf(anchor)! + 1; - let next = list._sequence.atPhysical(index); - if (next === element) next = list._sequence.atPhysical((index += 1)); - return next; - }; - const previousSlot = slotNeighbor(meta.beforePhysicalPrevious); - const nextSlot = slotNeighbor(meta.beforePhysicalNext); - if (previousSlot.found || nextSlot.found) { - const before = previousSlot.found - ? physicalAfter(previousSlot.element) - : nextSlot.element; - const physicalFrom = list._sequence.physicalIndexOf(element)!; - let to = - before === undefined ? list.length - 1 : list._sequence.visibleIndexOf(before)!; - if ( - before !== undefined && - physicalFrom < list._sequence.physicalIndexOf(before)! - ) { - to -= 1; - } - if (to === from) { - // Already at that visible index: only the local physical slot changes, - // which no other peer observes, so no op is written. - list._sequence.moveBefore(element, before); - return; - } - list._physicalMoveHint = { element, before }; - try { - this._movableMove(list, from, to); - } finally { - list._physicalMoveHint = undefined; - } - return; - } - const visibleIndex = (id: CodecId | undefined): number | undefined => { - if (id === undefined) return undefined; - const neighbor = list._sequence.findById(id); - return neighbor === undefined || neighbor.deleted - ? undefined - : list._sequence.visibleIndexOf(neighbor); - }; - let to: number; - const previous = visibleIndex(meta.beforePrevious); - const next = visibleIndex(meta.beforeNext); - if (previous !== undefined) to = previous < from ? previous + 1 : previous; - else if (next !== undefined) to = next < from ? next : next - 1; - else if (meta.beforePrevious === undefined) to = 0; - else if (meta.beforeNext === undefined) to = list.length - 1; - else return; - if (to !== from) list.move(from, to); - } - #deleteInsertedElements( container: LoroText | LoroList, peer: bigint, @@ -4132,9 +4148,23 @@ export class LoroDoc = Record( @@ -4142,20 +4172,53 @@ export class LoroDoc = Record ({ - type: container instanceof LoroMovableList ? "movable-list-insert" : "list-insert", - position, + type: movable ? "movable-list-insert" : "list-insert", + position: movable + ? this.#movableOpIndexForInsert(container as LoroMovableList, position) + : position, values: [{ type: "container-type", value: rawType }], })); } + /** Rust's `convert_index(pos, ForUser, ForOp)`: dead items count in op indices. */ + #movableOpIndexForInsert(container: LoroMovableList, position: number): number { + this._ensureContainerHydrated(container); + return container._state.opIndexForInsert(position); + } + _sequenceDelete(container: LoroList, position: number, length: number): void { - this.#deleteSequenceRuns( - container, - position, - length, - container instanceof LoroMovableList ? "movable-list-delete" : "list-delete", - ); + if (container instanceof LoroMovableList) { + this.#deleteMovableElements(container, position, length); + return; + } + this.#deleteSequenceRuns(container, position, length, "list-delete"); + } + + /** + * Like `MovableListHandler::delete_with_txn`: one op per element, each naming + * the list item ID and its op index after the earlier deletes. + */ + #deleteMovableElements( + container: LoroMovableList, + position: number, + length: number, + ): void { + this._ensureContainerHydrated(container); + for (let deleted = 0; deleted < length; deleted += 1) { + const target = container._state.positionAt(position)!; + this.#appendAndApply( + container, + { + type: "movable-list-delete", + startId: { ...target.id }, + position: container._state.opIndexOf(target), + length: 1n, + }, + 1, + ); + } } _textInsert(container: LoroText, position: number, text: string): void { @@ -4187,29 +4250,33 @@ export class LoroDoc = Record = Record ({ type: "movable-list-set", - elementId: { peer: element.id.peer, lamport: element.lamport }, + elementId: { peer: element.peer, lamport: element.lamport }, value: { type: "container-type", value: rawType }, })); } @@ -4506,6 +4574,9 @@ export class LoroDoc = Record = Record void) | undefined { if (!force && !this.#hasEventSubscribers()) return undefined; + // MovableListState reports its own changes; see #applyOperation. + if (container instanceof LoroMovableList) return undefined; const content = operation.content; if (container instanceof LoroText) { const state = this.#sequenceEventState(recording, container, "text"); @@ -4575,7 +4648,7 @@ export class LoroDoc = Record { const first = container._sequence.findById({ peer: changeId.peer, @@ -4592,7 +4665,7 @@ export class LoroDoc = Record = Record { - const to = element.deleted - ? undefined - : container._sequence.visibleIndexOf(element); - if (to === undefined || to === from) return; - state.diff.delete(from, 1); - state.diff.insertList(to, [value]); - }; - } - if (content.type === "movable-list-set") { - const element = container._sequence.findByLamport( - content.elementId.peer, - content.elementId.lamport, - ); - const position = - element === undefined || element.deleted - ? undefined - : container._sequence.visibleIndexOf(element); - if (element === undefined || position === undefined) return undefined; - const selected = element; - const previous = cloneRuntimeValue(element.value); - return () => { - const value = cloneRuntimeValue(selected.value); - if (eventValuesEqual(previous, value)) return; - state.diff.delete(position, 1); - state.diff.insertList(position, [value]); - }; - } } else if (container instanceof LoroMap) { if (content.type === "map-insert" || content.type === "map-delete") { const state = this.#mapEventState(recording, container); @@ -4708,6 +4742,14 @@ export class LoroDoc = Record diff.delete(position, length), + insertList: (position, values) => diff.insertList(position, values), + }; + } + #mapEventState( recording: EventRecording, container: LoroMap, @@ -5030,6 +5072,7 @@ export class LoroDoc = Record = Record + this.#decodeRuntimeValue( + value, + keys, + { peer: operationId.peer, counter: operationId.counter + index }, + container, + ), + ); + (container as LoroMovableList)._state.applyInsert( + content.position, + values, + operationId, + lamport, + causalVersion, + movableEvents, + ); + return; + } + case "movable-list-delete": + (container as LoroMovableList)._state.applyDelete( + content.startId, + Number(content.length), + operationId, + movableEvents, + ); + return; + case "list-insert": { const values = content.values.map((value, index) => this.#decodeRuntimeValue( value, @@ -5089,7 +5158,6 @@ export class LoroDoc = Record = Record = Record 0; for (const { change } of records) { - const knownEnd = this.#shallowStartVersion.get(change.id.peer) ?? 0; - if (change.id.counter + changeLength(change) <= knownEnd) continue; - if (change.dependencies.length === 0) { + // Changes up to the root are already part of the root state; a change that + // straddles the root continues from the root op. + const rootEnd = Math.max( + this.#shallowStartVersion.get(change.id.peer) ?? 0, + this.#shallowRootVersion.get(change.id.peer) ?? 0, + ); + if (change.id.counter < rootEnd) continue; + if (change.dependencies.length === 0 && trimmed) { throw new Error("cannot import updates that depend on an outdated version"); } @@ -5453,6 +5512,9 @@ export class LoroDoc = Record = Record(); - if (movableMoveMode === "replay") { - for (const { change } of records) { - for (const operation of change.operations) { - if (operation.content.type === "movable-list-move") { - const containerId = this.#containerKey(operation.container); - replayedMoveContainers.add(containerId); - } - } - } - for (const containerId of replayedMoveContainers) { - if (this.#movableOrderHistory.get(containerId) === undefined) return false; - } - } - const moveSuffixes = new Map< - SequenceElement, - { readonly history: readonly SequenceMoveMeta[]; readonly indices: Set } - >(); + #canTransitionRecords(records: readonly HistoryRecord[]): boolean { + const movableOps = new Map(); // The inserted elements and style anchors each sequence must still hold, // checked once per container below: one tree walk per operation made a // checkout O(operations * elements). @@ -5508,60 +5550,19 @@ export class LoroDoc = Record = Record = Record = Record = Record(); const textAttributeRuns = new Map>(); const textStyleContainers = new Set(); - const movableValues = new Map(); + const movableOps = new Map(); const mapKeys = new Map>(); const treeSubjects = new Map>(); const counters = new Map(); @@ -5726,12 +5715,18 @@ export class LoroDoc = Record = Record = Record = Record = Record - change.operations.flatMap((operation) => - operation.content.type === "movable-list-move" ? [{ change, operation }] : [], - ), - ) - .sort((left, right) => - compareHistoryOperations( - left.change, - left.operation, - right.change, - right.operation, - ), - ); - if (retreat) operations.reverse(); - for (const { change, operation } of operations) { - const content = operation.content; - if (content.type !== "movable-list-move") continue; - const container = this.#containers.get(this.#containerKey(operation.container)); - if (!(container instanceof LoroMovableList)) continue; - const element = container._sequence.findByLamport( - content.elementId.peer, - content.elementId.lamport, - ); - if (element === undefined || element.deleted) continue; - const meta = findSequenceMoveMeta( - element.moveHistory, - operationWriter(change, operation), - ); - if (meta === undefined) continue; - const from = container._sequence.visibleIndexOf(element); - if (from === undefined) continue; - const value = cloneRuntimeValue(element.value); - container._moveToAnchors( - element, - retreat ? meta.beforePrevious : meta.afterPrevious, - retreat ? meta.beforeNext : meta.afterNext, - ); - const to = container._sequence.visibleIndexOf(element); - if (recording === undefined || to === undefined || to === from) continue; - const state = this.#sequenceEventState(recording, container, "list"); - state.diff.delete(from, 1); - state.diff.insertList(to, [value]); - } - } - - #canonicalizeImportedMovableMoves( - records: readonly HistoryRecord[], - recording?: EventRecording, - ): void { - let hasMove = false; - for (const { change } of records) { - for (const operation of change.operations) { - if (operation.content.type !== "movable-list-move") continue; - hasMove = true; - const container = this.#containers.get(this.#containerKey(operation.container)); - if (!(container instanceof LoroMovableList) || !container._moveHistoryComplete) { - return; - } - } - } - if (hasMove) this.#replayMovableMoves([], records, this.#historyVersion(), recording); - } - - #replayMovableMoves( - retreat: readonly HistoryRecord[], - forward: readonly HistoryRecord[], - target: VersionVector, - recording?: EventRecording, - ): void { - const containerIds = new Set(); - for (const { change } of [...retreat, ...forward]) { - for (const operation of change.operations) { - if (operation.content.type === "movable-list-move") { - containerIds.add(this.#containerKey(operation.container)); - } - } - } - - for (const containerId of containerIds) { - const container = this.#containers.get(containerId); - const history = this.#movableOrderHistory.get(containerId); - if (!(container instanceof LoroMovableList) || history === undefined) continue; - const replay = new LoroMovableList(); - const ordered = history - .values() - .map((indexed) => ({ - indexed, - orderRecord: - this.#recordContaining({ - peer: indexed.record.change.id.peer, - counter: indexed.operation.counter, - }) ?? indexed.record, - })) - .sort( - (left, right) => - compareHistoryRecords(left.orderRecord, right.orderRecord) || - left.indexed.operation.counter - right.indexed.operation.counter, - ); - for (const { indexed } of ordered) { - const peer = indexed.record.change.id.peer; - const includedLength = Math.min( - indexed.operation.length, - (target.get(peer) ?? 0) - indexed.operation.counter, - ); - if (includedLength <= 0) continue; - const operation = - includedLength === indexed.operation.length - ? indexed.operation - : sliceOperation(indexed.operation, 0, includedLength); - const content = operation.content; - const operationId = { peer, counter: operation.counter }; - const lamport = - indexed.record.change.lamport + - operation.counter - - indexed.record.change.id.counter; - const causalVersion = this.#causalVersionAt(indexed.record.change.dependencies); - causalVersion.set( - peer, - Math.max(causalVersion.get(peer) ?? 0, operation.counter), - ); - if (content.type === "movable-list-insert") { - replay._insertFugue( - content.position, - content.values.map(() => null), - content.values.map((_, offset) => ({ - peer, - counter: operation.counter + offset, - })), - content.values.map((_, offset) => lamport + offset), - causalVersion, - ); - } else if (content.type === "movable-list-delete") { - replay._deleteIdSpan(content.startId, Number(content.length), operationId); - } else if (content.type === "movable-list-move") { - const element = replay._sequence.findByLamport( - content.elementId.peer, - content.elementId.lamport, - ); - const from = - element === undefined || element.deleted - ? undefined - : replay._sequence.visibleIndexOf(element); - if (from !== undefined) { - replay._applyMove(from, Math.min(content.to, replay.length - 1), { - id: operationId, - lamport, - }); - } - } - } - - const current = container._visibleElements(); - const currentIndex = new Map( - current.map((element, index) => [element, index] as const), - ); - const targetElements = replay._visibleElements().map((replayed) => { - const element = container._sequence.findById(replayed.id); - if (element === undefined || element.deleted) { - throw new Error("movable-list replay produced an unavailable target element"); - } - return element; - }); - for (const replayed of replay._elements) { - const element = container._sequence.findById(replayed.id); - if (element === undefined) continue; - for (const meta of replayed.moveHistory ?? []) { - let history = element.moveHistory; - if (history === undefined) { - history = []; - element.moveHistory = history; - } - const existing = history.findIndex( - ({ id }) => id.peer === meta.id.peer && id.counter === meta.id.counter, - ); - if (existing >= 0) history[existing] = meta; - else { - history.push(meta); - history.sort((left, right) => - compareWriter( - { peer: left.id.peer, lamport: left.lamport }, - { peer: right.id.peer, lamport: right.lamport }, - ), - ); - } - } - } - if ( - targetElements.length !== current.length || - targetElements.some((element) => !currentIndex.has(element)) - ) { - throw new Error("movable-list replay changed the visible element set"); - } - const stableTargetIndices = longestIncreasingSubsequenceIndices( - targetElements.map((element) => currentIndex.get(element)!), - ); - const state = - recording === undefined - ? undefined - : this.#sequenceEventState(recording, container, "list"); - for (let index = targetElements.length - 1; index >= 0; index -= 1) { - if (stableTargetIndices.has(index)) continue; - const element = targetElements[index]!; - const from = container._sequence.visibleIndexOf(element)!; - container._sequence.moveBefore(element, targetElements[index + 1]); - const to = container._sequence.visibleIndexOf(element)!; - if (state !== undefined && from !== to) { - state.diff.delete(from, 1); - state.diff.insertList(to, [cloneRuntimeValue(element.value)]); - } - } - } - } - /** * Resets every container and replays history up to `version`, or the latest * version. Every container that loro.js cannot replay is then rebuilt from @@ -6477,7 +6216,7 @@ export class LoroDoc = Record = Record(); + for (const { change } of records) { + let causalVersion: CausalVersion | undefined; + for (const operation of change.operations) { + const key = this.#containerKey(operation.container); + if (complete.has(key)) continue; + if ( + this.#deferredSnapshotState !== undefined && + !this.#containers.has(key) && + getLazyStateSnapshotContainer( + this.#deferredSnapshotState.store, + operation.container, + ) === undefined + ) { + continue; + } + const container = this.#getOrCreateContainer(operation.container); + const entry = this.#snapshotSequences.get(key); + if ( + entry?.kind !== "hydrated" || + !(container instanceof LoroList || container instanceof LoroText) + ) { + continue; + } + causalVersion ??= this.#causalVersionAt(change.dependencies); + const seen = causalVersion; + const content = operation.content; + if ( + entry.version + ._codecEntriesUnsorted() + .some(({ peer, counter }) => (seen.get(peer) ?? 0) < counter) || + (container instanceof LoroMovableList && + (content.type === "movable-list-move" || + content.type === "movable-list-set") && + container._state.element( + content.elementId.peer, + content.elementId.lamport, + ) === undefined) + ) { + complete.set(key, container); + } + } + } + for (const [key, container] of complete) { + this.#completeSnapshotSequence(container, key, current); + } + } + /** * Readies the containers that `records` touch for a transition from * `current`. A lazily encoded container that no transition has touched still @@ -6561,10 +6360,13 @@ export class LoroDoc = Record counter >= (version.get(peer) ?? 0), - ) || + (!(container instanceof LoroMovableList) && + [...firstCounters].every( + ([peer, counter]) => counter >= (version.get(peer) ?? 0), + )) || !this.#completeSnapshotSequence(container, key, current) ) { continue; @@ -6591,7 +6393,9 @@ export class LoroDoc = Record = Record = Record = Record = Record - compareWriter(left.writer, right.writer), - ); - this.#movableOrderHistory.set(container, history); - } - history.add(indexed); - if (content.type === "movable-list-move") { - let peers = this.#movableMovePeers.get(container); - if (peers === undefined) { - peers = new Set(); - this.#movableMovePeers.set(container, peers); - } - peers.add(record.change.id.peer); - } - } let bySubject: Map | undefined; let subject: string | undefined; let treeOperation = false; @@ -7233,8 +7023,6 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record this.#encodeSnapshotValue(value)); + } if (!(container instanceof LoroList)) { throw new TypeError(`unsupported container kind ${container.kind()}`); } @@ -8255,32 +8051,6 @@ export class LoroDoc = Record this.#encodeSnapshotValue(element.value)), - peers, - items: [ - { - invisibleListItems: 0n, - positionIdEqualsElementId: true, - elementIdEqualsLastSetId: true, - }, - ...visible.map(() => ({ - invisibleListItems: 0n, - positionIdEqualsElementId: true, - elementIdEqualsLastSetId: true, - })), - ], - listItemIds: visible.map((element) => ({ - peerIndex: peerIndex(element.id.peer), - counter: element.id.counter, - lamportSub: element.lamport - element.id.counter, - })), - elementIds: [], - lastSetIds: [], - }; - } return { kind: CodecContainerType.List, values: visible.map((element) => this.#encodeSnapshotValue(element.value)), @@ -8327,10 +8097,13 @@ export class LoroDoc = Record = Record [item.key, item])); for (const [key, value] of state.values) { @@ -8496,18 +8273,11 @@ export class LoroDoc = Record this.#decodeSnapshotValue(value, container)), - ids.map((item) => ({ - peer: state.peers[Number(item.peerIndex)]!, - counter: item.counter, - })), - ids.map((item) => item.counter + item.lamportSub), + container._state.hydrate( + state, + (value) => this.#decodeSnapshotValue(value, container), + shallowRoot, ); - container._valueHistoryComplete = false; - container._moveHistoryComplete = false; } else if (container instanceof LoroList && state.kind === CodecContainerType.List) { container._insertVisible( 0, @@ -8907,50 +8677,110 @@ function changedContainerIds(records: readonly HistoryRecord[]): Set { ); } -function longestIncreasingSubsequenceIndices(values: readonly number[]): Set { - if (values.length === 0) return new Set(); - const tails: number[] = []; - const previous = new Int32Array(values.length); - previous.fill(-1); - for (let index = 0; index < values.length; index += 1) { - let low = 0; - let high = tails.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (values[tails[middle]!]! < values[index]!) low = middle + 1; - else high = middle; +/** + * Rust's `shallow_since_vv` keeps a zero entry for a root at a peer's first op; + * without it Rust treats the imported document as not shallow at all. + */ +function shallowStartEntries( + startVersion: VersionVector, + startFrontiers: readonly CodecId[], +): { peer: bigint; counter: number }[] { + const entries = startVersion.codecEntries(); + for (const frontier of startFrontiers) { + if (!entries.some(({ peer }) => peer === frontier.peer)) { + entries.push({ peer: frontier.peer, counter: 0 }); } - if (low > 0) previous[index] = tails[low - 1]!; - tails[low] = index; } - const indices = new Set(); - let index = tails.at(-1)!; - while (index >= 0) { - indices.add(index); - index = previous[index]!; + return entries.sort((left, right) => + left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0, + ); +} + +function movableTransitionOp( + change: DecodedChange, + operation: DecodedOperation, +): MovableTransitionOp { + const content = operation.content; + const id = { peer: change.id.peer, counter: operation.counter }; + switch (content.type) { + case "movable-list-insert": + return { + type: "insert", + id, + lamport: change.lamport + (operation.counter - change.id.counter), + length: operation.length, + }; + case "movable-list-delete": + return { type: "delete", id, length: operation.length }; + case "movable-list-move": + return { type: "move", id, elementId: content.elementId }; + case "movable-list-set": + return { type: "set", id, elementId: content.elementId }; + default: + throw new Error(`${content.type} is not a movable-list operation`); } - return indices; } -function movableMoveTransitionMode( - retreat: readonly HistoryRecord[], - forward: readonly HistoryRecord[], - movePeers: ReadonlyMap>, -): MovableMoveTransitionMode { - const changedMoveContainers = new Set(); - for (const { change } of [...retreat, ...forward]) { - for (const operation of change.operations) { - if (operation.content.type === "movable-list-move") { - changedMoveContainers.add(formatContainerId(operation.container)); - } +/** + * Transforms list delta `delta` to apply after `over` (both start from the same + * list). At an equal position `delta`'s inserts go first, like Rust's + * `transform(.., left_priority = true)` in undo. + */ +function transformListDelta( + delta: readonly Delta[], + over: readonly Delta[], +): Delta[] { + type Part = + | { readonly kind: "retain" | "delete"; length: number } + | { readonly kind: "insert"; values: unknown[] }; + const parts = (items: readonly Delta[]): Part[] => + items.map((item) => + "insert" in item + ? { kind: "insert", values: [...item.insert] } + : "delete" in item + ? { kind: "delete", length: item.delete } + : { kind: "retain", length: item.retain }, + ); + const left = parts(delta); + const right = parts(over); + const output: Delta[] = []; + const push = (item: Delta): void => { + const last = output.at(-1); + if (last !== undefined && "retain" in last && "retain" in item) { + output[output.length - 1] = { retain: last.retain + item.retain }; + } else if (last !== undefined && "delete" in last && "delete" in item) { + output[output.length - 1] = { delete: last.delete + item.delete }; + } else { + output.push(item); + } + }; + while (left.length > 0 || right.length > 0) { + const a = left[0]; + const b = right[0]; + if (a?.kind === "insert") { + push({ insert: a.values }); + left.shift(); + continue; + } + if (b?.kind === "insert") { + push({ retain: b.values.length }); + right.shift(); + continue; + } + if (a === undefined) break; + const length = Math.min(a.length, b?.length ?? Number.POSITIVE_INFINITY); + if (b === undefined || b.kind === "retain") { + push(a.kind === "delete" ? { delete: length } : { retain: length }); + } + a.length -= length; + if (a.length === 0) left.shift(); + if (b !== undefined) { + b.length -= length; + if (b.length === 0) right.shift(); } } - if (changedMoveContainers.size === 0) return "anchors"; - if (retreat.length > 0 && forward.length > 0) return "replay"; - for (const containerId of changedMoveContainers) { - if ((movePeers.get(containerId)?.size ?? 0) > 1) return "replay"; - } - return "anchors"; + while (output.length > 0 && "retain" in output.at(-1)!) output.pop(); + return output; } function hasMaterializedSequenceInsertions( @@ -8971,12 +8801,12 @@ function hasMaterializedSequenceInsertions( continue; } const container = containers.get(formatContainerId(operation.container)); + const id = { peer: change.id.peer, counter: operation.counter }; if ( - (container instanceof LoroText || container instanceof LoroList) && - container._sequence.findById({ - peer: change.id.peer, - counter: operation.counter, - }) !== undefined + container instanceof LoroMovableList + ? container._state.positions.findById(id) !== undefined + : (container instanceof LoroText || container instanceof LoroList) && + container._sequence.findById(id) !== undefined ) { return true; } @@ -9503,18 +9333,6 @@ function compareHistoryRecords(left: HistoryRecord, right: HistoryRecord): numbe ); } -function compareHistoryOperations( - leftChange: DecodedChange, - leftOperation: DecodedOperation, - rightChange: DecodedChange, - rightOperation: DecodedOperation, -): number { - return compareWriter( - operationWriter(leftChange, leftOperation), - operationWriter(rightChange, rightOperation), - ); -} - function lowerBoundHistory(records: readonly HistoryRecord[], counter: number): number { let low = 0; let high = records.length; @@ -9622,67 +9440,6 @@ function latestIncludedTreePlacement( return latest; } -function latestIncludedSequenceValue( - history: readonly SequenceValueMeta[] | undefined, - version: VersionVector, -): SequenceValueMeta | undefined { - if (history === undefined) return undefined; - for (let index = history.length - 1; index >= 0; index -= 1) { - const meta = history[index]!; - if (meta.id.counter < (version.get(meta.id.peer) ?? 0)) return meta; - } - return undefined; -} - -function hasSequenceValueMeta( - history: readonly SequenceValueMeta[] | undefined, - writer: LastWriter, - peer: bigint, - counter: number, -): boolean { - if (history === undefined) return false; - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - const meta = history[middle]!; - if (compareWriter({ peer: meta.id.peer, lamport: meta.lamport }, writer) < 0) { - low = middle + 1; - } else { - high = middle; - } - } - const meta = history[low]; - return meta?.id.peer === peer && meta.id.counter === counter; -} - -function findSequenceMoveMeta( - history: readonly SequenceMoveMeta[] | undefined, - writer: LastWriter, -): SequenceMoveMeta | undefined { - const index = findSequenceMoveMetaIndex(history, writer); - return index < 0 ? undefined : history![index]; -} - -function findSequenceMoveMetaIndex( - history: readonly SequenceMoveMeta[] | undefined, - writer: LastWriter, -): number { - if (history === undefined) return -1; - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - const meta = history[middle]!; - if (compareWriter({ peer: meta.id.peer, lamport: meta.lamport }, writer) < 0) { - low = middle + 1; - } else { - high = middle; - } - } - return low < history.length ? low : -1; -} - function counterDelta( content: Extract, ): number | undefined { @@ -10397,6 +10154,9 @@ function captureBlueprint(container: Container): ContainerBlueprint { if (container instanceof LoroText) return { kind: "Text", value: container.toDelta() }; if (container instanceof LoroCounter) return { kind: "Counter", value: container.value }; + if (container instanceof LoroMovableList) { + return { kind: "MovableList", value: [...container._rawValues()] }; + } if (container instanceof LoroList) { return { kind: container.kind(), @@ -11288,10 +11048,18 @@ function recoverParentBinding( ): | { readonly kind: "map"; readonly key: string } | { readonly kind: "sequence"; readonly element: SequenceElement } + | { readonly kind: "movable"; readonly element: MovableElement } | { readonly kind: "tree"; readonly record: TreeNodeRecord } | undefined { parent._ensureHydrated(); - if (parent instanceof LoroMap) { + if (parent instanceof LoroMovableList) { + for (const element of parent._state.visibleElements()) { + if (element.value !== child) continue; + const binding = { kind: "movable" as const, element }; + child._setParentBinding(parent, binding); + return binding; + } + } else if (parent instanceof LoroMap) { for (const [key, record] of parent._entries) { if (record.value !== child) continue; const binding = { kind: "map" as const, key }; @@ -11333,6 +11101,10 @@ function containerPath(container: LoroContainer): Path { if (binding?.kind === "map") { path.unshift(binding.key); } + } else if (parent instanceof LoroMovableList) { + const index = + binding?.kind === "movable" ? parent._state.indexOf(binding.element) : undefined; + if (index !== undefined && index >= 0) path.unshift(index); } else if (parent instanceof LoroList) { const index = binding?.kind === "sequence" diff --git a/loro-js/src/runtime/movable-list.ts b/loro-js/src/runtime/movable-list.ts new file mode 100644 index 000000000..c52ed7c0d --- /dev/null +++ b/loro-js/src/runtime/movable-list.ts @@ -0,0 +1,896 @@ +import type { + ListStateItemId, + MovableListStateItem, + MovableListStateLamportId, + MovableListStateSnapshot, +} from "../codec/state-snapshot"; +import { ContainerType as CodecContainerType } from "../codec/types"; +import type { EncodedLoroValue, Id as CodecId } from "../codec/types"; +import { + cloneRuntimeValue, + insertFugueElements, + LoroContainer, + type CausalVersion, + type LastWriter, + type RuntimeValue, + type SequenceElement, +} from "./containers"; +import { SequenceIndex, type SequenceMetrics } from "./sequence-index"; + +/** + * MovableList state with Rust's model: a Fugue sequence of positions plus + * elements whose position and value are separate last-writer-wins registers. + * See context/loro-js-movable-list.md. + */ + +/** A list item. `deleted` means it is not alive; see `MovableListState`. */ +export interface MovablePosition extends SequenceElement { + /** + * The element this position was created for. Undefined for an unpointed + * position read from a snapshot, whose element is not recorded. + */ + element: MovableElement | undefined; +} + +export interface MovableValueCandidate { + readonly writer: LastWriter; + readonly value: RuntimeValue; + /** The writing op, or undefined for a shallow-root seed that every version includes. */ + readonly id: CodecId | undefined; +} + +export interface MovableElement { + readonly peer: bigint; + readonly lamport: number; + value: RuntimeValue; + valueWriter: LastWriter; + /** The winning position. The element is visible iff this position is alive. */ + pos: MovablePosition; + /** Position candidates by `(lamport, peer)`; undefined when history is unknown. */ + positions: MovablePosition[] | undefined; + /** Value candidates by writer; undefined when history is unknown. */ + values: MovableValueCandidate[] | undefined; +} + +export interface MovableElementId { + readonly peer: bigint; + readonly lamport: number; +} + +/** Receives user-visible changes in the order they happen. */ +export interface MovableListEvents { + delete(position: number, length: number): void; + insertList(position: number, values: readonly unknown[]): void; +} + +export type MovableTransitionOp = + | { + readonly type: "insert"; + readonly id: CodecId; + readonly lamport: number; + readonly length: number; + } + | { readonly type: "delete"; readonly id: CodecId; readonly length: number } + | { readonly type: "move"; readonly id: CodecId; readonly elementId: MovableElementId } + | { readonly type: "set"; readonly id: CodecId; readonly elementId: MovableElementId }; + +export class InvalidMovableListOpError extends Error {} + +const POINTED: SequenceMetrics = { utf16: 1, utf8: 1 }; +const UNPOINTED: SequenceMetrics = { utf16: 0, utf8: 0 }; + +/** A position counts as one user-visible item iff its element points at it. */ +function positionMetrics(position: MovablePosition): SequenceMetrics { + return position.element !== undefined && position.element.pos === position + ? POINTED + : UNPOINTED; +} + +export function compareLamportIds( + left: { readonly lamport: number; readonly peer: bigint }, + right: { readonly lamport: number; readonly peer: bigint }, +): number { + return ( + left.lamport - right.lamport || + (left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0) + ); +} + +function positionWriter(position: MovablePosition): LastWriter { + return { peer: position.id.peer, lamport: position.lamport }; +} + +function insertSorted( + items: T[], + item: T, + compare: (left: T, right: T) => number, +): void { + let low = 0; + let high = items.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compare(items[middle]!, item) <= 0) low = middle + 1; + else high = middle; + } + items.splice(low, 0, item); +} + +function latestIncluded( + items: readonly T[], + included: (item: T) => boolean, +): T | undefined { + for (let index = items.length - 1; index >= 0; index -= 1) { + if (included(items[index]!)) return items[index]; + } + return undefined; +} + +export function runtimeValuesEqual(left: RuntimeValue, right: RuntimeValue): boolean { + if (left === right) return true; + if (left instanceof LoroContainer || right instanceof LoroContainer) return false; + if (left instanceof Uint8Array || right instanceof Uint8Array) { + if (!(left instanceof Uint8Array) || !(right instanceof Uint8Array)) return false; + return ( + left.length === right.length && left.every((byte, index) => byte === right[index]) + ); + } + if (Array.isArray(left) || Array.isArray(right)) { + if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) { + return false; + } + return left.every((item, index) => runtimeValuesEqual(item, right[index]!)); + } + if ( + typeof left === "object" && + left !== null && + typeof right === "object" && + right !== null + ) { + const leftKeys = Object.keys(left); + const rightRecord = right as Record; + return ( + leftKeys.length === Object.keys(right).length && + leftKeys.every( + (key) => + Object.prototype.hasOwnProperty.call(right, key) && + runtimeValuesEqual( + (left as Record)[key]!, + rightRecord[key]!, + ), + ) + ); + } + return false; +} + +export class MovableListState { + readonly positions = new SequenceIndex(positionMetrics); + readonly #elements = new Map>(); + /** + * False after hydrating latest state from a snapshot: tombstones, origins and + * candidate history are then unknown, so version transitions and imports that + * are concurrent with that state must replay history first. + */ + historyComplete = true; + + constructor(readonly bindValue: (element: MovableElement) => void) {} + + /** User-visible length. */ + get length(): number { + return this.positions.visibleUtf16Length; + } + + /** Length in op indices: alive positions, pointed or not. */ + get opLength(): number { + return this.positions.visibleLength; + } + + reset(): void { + this.positions.reset(); + this.#elements.clear(); + this.historyComplete = true; + } + + /** + * `element`'s value at the version `includes` describes, or `undefined` when + * the element is not visible there. A state hydrated from a snapshot only + * knows versions from the snapshot's on (`historyComplete`). + */ + visibleValueAt( + element: MovableElement, + includes: (id: CodecId) => boolean, + ): { readonly value: RuntimeValue } | undefined { + const position = latestIncluded(element.positions ?? [element.pos], (candidate) => + includes(candidate.id), + ); + if (position === undefined || this.positions.someDeletion(position, includes)) { + return undefined; + } + const value = + element.values === undefined + ? { value: element.value } + : latestIncluded( + element.values, + (candidate) => candidate.id === undefined || includes(candidate.id), + ); + return value === undefined ? undefined : { value: value.value }; + } + + /** Every element, visible or not. */ + *allElements(): IterableIterator { + for (const byLamport of this.#elements.values()) yield* byLamport.values(); + } + + element(peer: bigint, lamport: number): MovableElement | undefined { + return this.#elements.get(peer)?.get(lamport); + } + + /** The position of the user-visible item at `index`. */ + positionAt(index: number): MovablePosition | undefined { + return this.positions.visibleAtMetricOffset(index, "utf16"); + } + + elementAt(index: number): MovableElement | undefined { + return this.positionAt(index)?.element; + } + + isVisible(element: MovableElement): boolean { + return !element.pos.deleted; + } + + /** User index of a visible element. */ + indexOf(element: MovableElement): number | undefined { + return this.isVisible(element) ? this.userIndexOf(element.pos) : undefined; + } + + /** Number of user-visible items before `position`, alive or not. */ + userIndexOf(position: MovablePosition): number { + return this.positions.visibleMetricOffsetOf(position, "utf16") ?? 0; + } + + /** Op index of an alive position. */ + opIndexOf(position: MovablePosition): number { + const index = this.positions.visibleIndexOf(position); + if (index === undefined || position.deleted) { + throw new Error("movable-list position is not alive"); + } + return index; + } + + /** Op index for a local insertion before the user item at `index`. */ + opIndexForInsert(index: number): number { + return index >= this.length ? this.opLength : this.opIndexOf(this.positionAt(index)!); + } + + visibleElements(): MovableElement[] { + const output: MovableElement[] = []; + this.positions.forEachVisible((position) => { + const element = position.element; + if (element !== undefined && element.pos === position) output.push(element); + }); + return output; + } + + visibleElementsRange(start: number, end: number): MovableElement[] { + const output: MovableElement[] = []; + for (let index = start; index < end; index += 1) { + const element = this.elementAt(index); + if (element !== undefined) output.push(element); + } + return output; + } + + applyInsert( + opIndex: number, + values: readonly RuntimeValue[], + firstId: CodecId, + lamport: number, + causalVersion: CausalVersion, + events?: MovableListEvents, + ): void { + if (values.length === 0) return; + const viewLength = this.#viewLength(causalVersion); + if (!Number.isSafeInteger(opIndex) || opIndex < 0 || opIndex > viewLength) { + throw new InvalidMovableListOpError( + `movable-list insert position ${opIndex} is out of range (length ${viewLength})`, + ); + } + const inserted: MovablePosition[] = []; + for (let offset = 0; offset < values.length; offset += 1) { + const id = { peer: firstId.peer, counter: firstId.counter + offset }; + const writer = { peer: firstId.peer, lamport: lamport + offset }; + const position: MovablePosition = { + id, + lamport: writer.lamport, + value: null, + deleted: false, + originLeft: undefined, + originRight: undefined, + element: undefined, + }; + const value = values[offset]!; + const element: MovableElement = { + peer: writer.peer, + lamport: writer.lamport, + value, + valueWriter: writer, + pos: position, + positions: [position], + values: [{ writer, value, id }], + }; + position.element = element; + this.#registerElement(element); + inserted.push(position); + } + // The origin index misorders a sibling subtree followed by a concurrent + // element that is not its descendant, so positions use the scan. + insertFugueElements(this.positions, opIndex, inserted, causalVersion, false); + for (const position of inserted) this.bindValue(position.element!); + if (events !== undefined) { + events.insertList( + this.userIndexOf(inserted[0]!), + values.map((value) => cloneRuntimeValue(value)), + ); + } + } + + /** Deletes the positions `startId..startId+|length|` (op `deletedBy` onwards). */ + applyDelete( + startId: CodecId, + length: number, + deletedBy: CodecId, + events?: MovableListEvents, + ): void { + const count = Math.abs(length); + for (let offset = 0; offset < count; offset += 1) { + const position = this.positions.findById({ + peer: startId.peer, + counter: startId.counter + offset, + }); + if (position === undefined) continue; + // In a reversed span the k-th atom deletes the k-th item from the end. + const byOffset = length < 0 ? count - 1 - offset : offset; + this.#deletePosition( + position, + { peer: deletedBy.peer, counter: deletedBy.counter + byOffset }, + events, + ); + } + } + + /** + * Rust's tracker deletes the item at `from` in the op's causal view, then + * inserts the new position at `to` in the view after that delete. + */ + applyMove( + from: number, + to: number, + elementId: MovableElementId, + opId: CodecId, + lamport: number, + causalVersion: CausalVersion, + events?: MovableListEvents, + ): void { + const element = this.element(elementId.peer, elementId.lamport); + if (element === undefined) { + throw new InvalidMovableListOpError( + `movable-list move targets unknown element L${elementId.lamport}@${elementId.peer}`, + ); + } + const current = this.positions.isFullyIncluded(causalVersion); + const view = current ? undefined : this.positions.causalView(causalVersion); + const viewLength = view?.length ?? this.positions.visibleLength; + const source = Number.isSafeInteger(from) + ? (view?.at(from) ?? (current ? this.positions.atVisible(from) : undefined)) + : undefined; + if (source === undefined) { + throw new InvalidMovableListOpError( + `movable-list move source ${from} is out of range (length ${viewLength})`, + ); + } + // `to` indexes the view after the source is deleted. + if (!Number.isSafeInteger(to) || to < 0 || to > viewLength - 1) { + throw new InvalidMovableListOpError( + `movable-list move destination ${to} is out of range (length ${viewLength - 1})`, + ); + } + this.#deletePosition(source, opId, events); + const afterDelete = new Map(causalVersion); + afterDelete.set( + opId.peer, + Math.max(afterDelete.get(opId.peer) ?? 0, opId.counter + 1), + ); + const position: MovablePosition = { + id: { ...opId }, + lamport, + value: null, + deleted: false, + originLeft: undefined, + originRight: undefined, + element, + }; + insertFugueElements(this.positions, to, [position], afterDelete, false); + if (element.positions !== undefined) { + insertSorted(element.positions, position, (left, right) => + compareLamportIds(positionWriter(left), positionWriter(right)), + ); + } + if (compareLamportIds(positionWriter(position), positionWriter(element.pos)) > 0) { + this.#setWinnerPosition(element, position, events); + } + } + + applySet( + elementId: MovableElementId, + value: RuntimeValue, + writer: LastWriter, + opId: CodecId | undefined, + events?: MovableListEvents, + local = false, + ): void { + const element = this.element(elementId.peer, elementId.lamport); + if (element === undefined) { + throw new InvalidMovableListOpError( + `movable-list set targets unknown element L${elementId.lamport}@${elementId.peer}`, + ); + } + const candidate = { writer, value, id: opId === undefined ? undefined : { ...opId } }; + if (element.values !== undefined) { + insertSorted(element.values, candidate, (left, right) => + compareLamportIds(left.writer, right.writer), + ); + } + if (compareLamportIds(writer, element.valueWriter) <= 0) return; + this.#setWinnerValue(element, candidate, events, local); + } + + /** + * Moves the state to the version selected by `includes` after the given ops + * were retreated or forwarded. Only the positions and elements those ops + * touch are revisited. + */ + transition( + ops: readonly MovableTransitionOp[], + includes: (id: CodecId) => boolean, + events?: MovableListEvents, + ): void { + const positions = new Set(); + const elements = new Set(); + const addPosition = (position: MovablePosition | undefined): void => { + if (position === undefined) return; + positions.add(position); + if (position.element !== undefined) elements.add(position.element); + }; + for (const op of ops) { + switch (op.type) { + case "insert": + for (let offset = 0; offset < op.length; offset += 1) { + addPosition( + this.positions.findById({ + peer: op.id.peer, + counter: op.id.counter + offset, + }), + ); + } + break; + case "delete": + for (const position of this.positions.elementsDeletedBy( + op.id.peer, + op.id.counter, + op.id.counter + op.length, + )) { + addPosition(position); + } + break; + case "move": + addPosition(this.positions.findById(op.id)); + for (const position of this.positions.elementsDeletedBy( + op.id.peer, + op.id.counter, + op.id.counter + 1, + )) { + addPosition(position); + } + break; + case "set": { + const element = this.element(op.elementId.peer, op.elementId.lamport); + if (element !== undefined) elements.add(element); + break; + } + } + } + for (const element of elements) positions.add(element.pos); + + const before = new Map(); + if (events !== undefined) { + for (const position of positions) { + const element = position.element; + if (!position.deleted && element !== undefined && element.pos === position) { + before.set(position, { + index: this.userIndexOf(position), + value: element.value, + }); + } + } + } + + for (const position of positions) { + const alive = + includes(position.id) && !this.positions.someDeletion(position, includes); + if (alive === position.deleted) this.positions.setDeleted(position, !alive); + } + for (const element of elements) { + const candidates = element.positions; + const values = element.values; + if (candidates === undefined || values === undefined) { + throw new Error("movable-list transition needs complete history"); + } + const winner = + latestIncluded(candidates, (position) => includes(position.id)) ?? candidates[0]!; + if (winner !== element.pos) { + const previous = element.pos; + element.pos = winner; + this.positions.refreshMetrics(previous); + this.positions.refreshMetrics(winner); + positions.add(winner); + } + const value = + latestIncluded( + values, + (candidate) => candidate.id === undefined || includes(candidate.id), + ) ?? values[0]!; + if ( + value.value !== element.value || + compareLamportIds(value.writer, element.valueWriter) !== 0 + ) { + element.value = value.value; + element.valueWriter = value.writer; + this.bindValue(element); + } + } + + if (events === undefined) return; + const after = new Map(); + for (const position of positions) { + const element = position.element; + if (!position.deleted && element !== undefined && element.pos === position) { + after.set(position, { index: this.userIndexOf(position), value: element.value }); + } + } + const removed = [...before] + .filter(([position, state]) => { + const next = after.get(position); + return next === undefined || !runtimeValuesEqual(next.value, state.value); + }) + .map(([, state]) => state.index) + .sort((left, right) => right - left); + for (const index of removed) events.delete(index, 1); + const inserted = [...after] + .filter(([position, state]) => { + const previous = before.get(position); + return previous === undefined || !runtimeValuesEqual(previous.value, state.value); + }) + .map(([, state]) => state) + .sort((left, right) => left.index - right.index); + for (const state of inserted) { + events.insertList(state.index, [cloneRuntimeValue(state.value)]); + } + } + + /** Whether version transitions over `ops` can run without replaying history. */ + canTransition(ops: readonly MovableTransitionOp[]): boolean { + if (!this.historyComplete) return false; + for (const op of ops) { + switch (op.type) { + case "insert": + if (!this.positions.containsIdRuns([{ start: op.id, length: op.length }])) { + return false; + } + break; + case "delete": { + let recorded = 0; + for (const run of this.positions.idRunsDeletedBy( + op.id.peer, + op.id.counter, + op.id.counter + op.length, + )) { + recorded += run.length; + } + if (recorded < op.length) return false; + break; + } + case "move": + if (this.positions.findById(op.id) === undefined) return false; + if ( + this.positions.elementsDeletedBy(op.id.peer, op.id.counter, op.id.counter + 1) + .length === 0 + ) { + return false; + } + break; + case "set": { + const element = this.element(op.elementId.peer, op.elementId.lamport); + if ( + element?.values === undefined || + !element.values.some( + (candidate) => + candidate.id !== undefined && + candidate.id.peer === op.id.peer && + candidate.id.counter === op.id.counter, + ) + ) { + return false; + } + break; + } + } + } + return true; + } + + /** Encodes the state in Rust's `MovableListState` snapshot layout. */ + encodeSnapshot( + encodeValue: (value: RuntimeValue) => EncodedLoroValue, + ): MovableListStateSnapshot { + const peers: bigint[] = []; + const peerIndices = new Map(); + const peerIndex = (peer: bigint): bigint => { + let index = peerIndices.get(peer); + if (index === undefined) { + index = peers.length; + peers.push(peer); + peerIndices.set(peer, index); + } + return BigInt(index); + }; + const values: EncodedLoroValue[] = []; + const items: { + -readonly [Key in keyof MovableListStateItem]: MovableListStateItem[Key]; + }[] = [ + { + invisibleListItems: 0n, + positionIdEqualsElementId: true, + elementIdEqualsLastSetId: true, + }, + ]; + const listItemIds: ListStateItemId[] = []; + const elementIds: MovableListStateLamportId[] = []; + const lastSetIds: MovableListStateLamportId[] = []; + this.positions.forEachVisible((position) => { + const element = position.element; + const listItemId = { + peerIndex: peerIndex(position.id.peer), + counter: position.id.counter, + lamportSub: position.lamport - position.id.counter, + }; + if (element === undefined || element.pos !== position) { + items.at(-1)!.invisibleListItems += 1n; + listItemIds.push(listItemId); + return; + } + const positionIdEqualsElementId = + position.id.peer === element.peer && position.lamport === element.lamport; + const elementIdEqualsLastSetId = + element.valueWriter.peer === element.peer && + element.valueWriter.lamport === element.lamport; + items.push({ + invisibleListItems: 0n, + positionIdEqualsElementId, + elementIdEqualsLastSetId, + }); + listItemIds.push(listItemId); + if (!positionIdEqualsElementId) { + elementIds.push({ peerIndex: peerIndex(element.peer), lamport: element.lamport }); + } + if (!elementIdEqualsLastSetId) { + lastSetIds.push({ + peerIndex: peerIndex(element.valueWriter.peer), + lamport: element.valueWriter.lamport, + }); + } + values.push(encodeValue(element.value)); + }); + return { + kind: CodecContainerType.MovableList, + values, + peers, + items, + listItemIds, + elementIds, + lastSetIds, + }; + } + + /** + * Hydrates Rust's snapshot layout. A shallow root seeds candidate history, + * because no version before the root can be checked out; latest state does + * not, so the state is marked incomplete. + */ + hydrate( + state: MovableListStateSnapshot, + decodeValue: (value: EncodedLoroValue) => RuntimeValue, + shallowRoot: boolean, + ): void { + const positions: MovablePosition[] = []; + const listItemIds = state.listItemIds; + let listIndex = 0; + let elementIndex = 0; + let lastSetIndex = 0; + const peer = (index: bigint): bigint => { + const value = state.peers[Number(index)]; + if (value === undefined) + throw new Error("movable-list state peer index out of range"); + return value; + }; + const nextPosition = (): MovablePosition => { + const item = listItemIds[listIndex++]; + if (item === undefined) + throw new Error("movable-list state is missing a list item ID"); + return { + id: { peer: peer(item.peerIndex), counter: item.counter }, + lamport: item.counter + item.lamportSub, + value: null, + deleted: false, + originLeft: undefined, + originRight: undefined, + element: undefined, + }; + }; + for (let itemIndex = 0; itemIndex < state.items.length; itemIndex += 1) { + const item = state.items[itemIndex]!; + if (itemIndex > 0) { + const position = nextPosition(); + const elementId = item.positionIdEqualsElementId + ? { peer: position.id.peer, lamport: position.lamport } + : readLamportId(state.elementIds[elementIndex++], peer); + const valueWriter = item.elementIdEqualsLastSetId + ? { ...elementId } + : readLamportId(state.lastSetIds[lastSetIndex++], peer); + const encoded = state.values[itemIndex - 1]; + if (encoded === undefined) + throw new Error("movable-list state is missing a value"); + const value = decodeValue(encoded); + const element: MovableElement = { + peer: elementId.peer, + lamport: elementId.lamport, + value, + valueWriter, + pos: position, + // Seeds that every later version includes. Without the snapshot's + // history (`historyComplete`), they are only right from its version on. + positions: [position], + values: [{ writer: valueWriter, value, id: undefined }], + }; + position.element = element; + this.#registerElement(element); + positions.push(position); + } + for (let index = 0n; index < item.invisibleListItems; index += 1n) { + positions.push(nextPosition()); + } + } + if (positions.length > 0) + this.positions.insertAtPhysical(this.positions.allLength, positions); + for (const position of positions) { + if (position.element !== undefined) this.bindValue(position.element); + } + if (!shallowRoot) this.historyComplete = false; + } + + #registerElement(element: MovableElement): void { + let byLamport = this.#elements.get(element.peer); + if (byLamport === undefined) { + byLamport = new Map(); + this.#elements.set(element.peer, byLamport); + } + if (byLamport.has(element.lamport)) { + throw new Error( + `duplicate movable-list element L${element.lamport}@${element.peer}`, + ); + } + byLamport.set(element.lamport, element); + } + + #viewLength(causalVersion: CausalVersion): number { + return this.positions.isFullyIncluded(causalVersion) + ? this.positions.visibleLength + : this.positions.causalView(causalVersion).length; + } + + #deletePosition( + position: MovablePosition, + deletedBy: CodecId, + events: MovableListEvents | undefined, + ): void { + const element = position.element; + const hides = !position.deleted && element !== undefined && element.pos === position; + if (events !== undefined && hides) events.delete(this.userIndexOf(position), 1); + this.positions.deleteElement(position, deletedBy); + } + + /** + * Events follow Rust's `convert_update_to_event_pos`: insert at the new + * index, then delete the old item. When the new item directly follows the + * old one the two cancel and no event is emitted. + */ + #setWinnerPosition( + element: MovableElement, + position: MovablePosition, + events: MovableListEvents | undefined, + ): void { + const previous = element.pos; + element.pos = position; + this.positions.refreshMetrics(previous); + this.positions.refreshMetrics(position); + if (events === undefined) return; + const removed = previous.deleted ? undefined : this.userIndexOf(previous); + if (!position.deleted) { + let inserted = this.userIndexOf(position); + if (removed !== undefined && inserted > removed) inserted += 1; + events.insertList(inserted, [cloneRuntimeValue(element.value)]); + } + if (removed !== undefined) events.delete(removed, 1); + } + + #setWinnerValue( + element: MovableElement, + candidate: MovableValueCandidate, + events: MovableListEvents | undefined, + local: boolean, + ): void { + const changed = local || !runtimeValuesEqual(element.value, candidate.value); + if (events !== undefined && changed && this.isVisible(element)) { + const index = this.userIndexOf(element.pos); + events.delete(index, 1); + events.insertList(index, [cloneRuntimeValue(candidate.value)]); + } + element.value = candidate.value; + element.valueWriter = candidate.writer; + this.bindValue(element); + } +} + +function readLamportId( + id: MovableListStateLamportId | undefined, + peer: (index: bigint) => bigint, +): LastWriter { + if (id === undefined) throw new Error("movable-list state is missing an element ID"); + return { peer: peer(id.peerIndex), lamport: id.lamport }; +} + +/** + * Whether two states hold the same alive positions in the same order, with + * the same elements, values and last writers: a snapshot state against the + * replay of its history. + */ +export function sameMovableListStates( + left: MovableListState, + right: MovableListState, +): boolean { + if (left.positions.visibleLength !== right.positions.visibleLength) return false; + const alive = (state: MovableListState): MovablePosition[] => { + const positions: MovablePosition[] = []; + state.positions.forEachVisible((position) => { + positions.push(position); + }); + return positions; + }; + const rightPositions = alive(right); + return alive(left).every((position, index) => { + const other = rightPositions[index]!; + if (position.id.peer !== other.id.peer || position.id.counter !== other.id.counter) { + return false; + } + const element = position.element?.pos === position ? position.element : undefined; + const otherElement = other.element?.pos === other ? other.element : undefined; + if (element === undefined || otherElement === undefined) { + return element === otherElement; + } + return ( + element.peer === otherElement.peer && + element.lamport === otherElement.lamport && + compareLamportIds(element.valueWriter, otherElement.valueWriter) === 0 && + runtimeValuesEqual(element.value, otherElement.value) + ); + }); +} diff --git a/loro-js/src/runtime/sequence-index.ts b/loro-js/src/runtime/sequence-index.ts index ebf62b745..31bbc3745 100644 --- a/loro-js/src/runtime/sequence-index.ts +++ b/loro-js/src/runtime/sequence-index.ts @@ -814,6 +814,48 @@ export class SequenceIndex { return runs; } + /** + * The visible element whose metric range contains `offset`. With a 0/1 + * metric this is the `offset`-th element that has the metric. + */ + visibleAtMetricOffset(offset: number, metric: Metric): T | undefined { + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + offset >= visibleMetric(this.#root, metric) + ) { + return undefined; + } + let node = this.#root; + let remaining = offset; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftMetric = visibleMetric(node.left, metric); + if (remaining < leftMetric) { + node = node.left; + continue; + } + remaining -= leftMetric; + const ownMetric = ownVisibleMetric(node, metric, this.#metrics); + if (remaining < ownMetric) { + const prefix = + metric === "utf16" ? node.visibleUtf16Prefix : node.visibleUtf8Prefix; + if (prefix === undefined) return nodeElement(node, 0); + let low = 0; + let high = prefix.length - 1; + while (low < high) { + const middle = (low + high) >>> 1; + if (prefix[middle + 1]! > remaining) high = middle; + else low = middle + 1; + } + return nodeElement(node, low); + } + remaining -= ownMetric; + node = node.right; + } + return undefined; + } + visibleIndexAfterLineBreaks(count: number): number | undefined { const total = visibleLineBreakMetric(this.#root); if (!Number.isSafeInteger(count) || count < 0 || count > total) return undefined; @@ -1080,6 +1122,15 @@ export class SequenceIndex { recomputeToRoot(node, this.#metrics, false); } + /** Recomputes cached metrics after `element`'s metrics changed in place. */ + refreshMetrics(element: T): void { + if (this.#hasLazyDeletions) materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + if (node === undefined) return; + recomputeOwn(node, this.#metrics); + recomputeToRoot(node, this.#metrics); + } + deleteElement(element: T, deletedBy?: SequenceId): void { if (this.#hasLazyDeletions) materializeElementDeletion(element, this.#metrics); const node = elementNode(element); diff --git a/loro-js/tests/differential/movable-list.test.ts b/loro-js/tests/differential/movable-list.test.ts index 2228d17a6..42f5ea613 100644 --- a/loro-js/tests/differential/movable-list.test.ts +++ b/loro-js/tests/differential/movable-list.test.ts @@ -115,15 +115,7 @@ const cases: Case[] = [ * an item that moves inside the Fugue sequence. The Rust-compatible model in * context/loro-js-movable-list.md removes them from this set. */ -const KNOWN_DIVERGENT = new Set([ - "insert/delete undo (one peer)", - "move/set", - "move/set with children", - "move/set history", - "move/set undo (one peer)", - "snapshots", - "shallow snapshots", -]); +const KNOWN_DIVERGENT = new Set([]); describe.skipIf(rust === undefined)("MovableList matches the Rust implementation", () => { test.each(cases)( diff --git a/loro-js/tests/fixtures/rust/movable-move-delete.blob b/loro-js/tests/fixtures/rust/movable-move-delete.blob new file mode 100644 index 0000000000000000000000000000000000000000..d8007b2dc2e19538ee6c2e9a8d68b4f93b33e8fd GIT binary patch literal 110 zcmYj}F$#b{5CfAntH^;^_%}O2JK^9!eUU%#gW?ufm?C5tNG?agFYmM5&s-y=Y=$V}ymGrOuqIbBnJJPY(zr literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/rust/movable-snapshot-peer3.blob b/loro-js/tests/fixtures/rust/movable-snapshot-peer3.blob new file mode 100644 index 0000000000000000000000000000000000000000..33e69b34fbc64ea0a7d7179f476e310e9181fe42 GIT binary patch literal 109 zcmYj{!3lss5CfC!t|IsGYZuny!yd$g1NCD!ma1?HhQI_Sxoic$tmAaSXQ>ef*=zVc fSXVqC8af>a&&z(Gr^{gmnWaw7L=#mTEraI`$L|K5 literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/rust/movable-snapshot-peer4.blob b/loro-js/tests/fixtures/rust/movable-snapshot-peer4.blob new file mode 100644 index 0000000000000000000000000000000000000000..1372c36d13f3f50d6ad31af1b0a90cb655086023 GIT binary patch literal 103 zcmd1FFUn^?0-95F-N{BMT!?4J%7dW^oCF e03!=f2QxDR3o|nxHxmmZ0|zrBGdm+IV+8=63kHh- literal 0 HcmV?d00001 diff --git a/loro-js/tests/fixtures/rust/movable-snapshot-updates.blob b/loro-js/tests/fixtures/rust/movable-snapshot-updates.blob new file mode 100644 index 0000000000000000000000000000000000000000..884248f70635b6fa4f63fa2d2be33b5fd67b9284 GIT binary patch literal 291 zcmb7;OA5k35JaoGx)VV#S-A58u02BV0A9h*MhOJe8+bkMGcPF{Ii$C+c#*1z;<)D39IfbwKuLmJ>vlZv8*UT z%xjDqB*BO#3Q>wL|3DOBv}TBCG$2^-?D7h;o@H=Xoi3^xu@GC}QD7(VIIs%_sjtTY zf_-$vgcT`Z7JHyzjzl|h1c8evdhU0k8U@vvqIj#s$5I08ad?Kb;Dx9=5ENS|t(vwl zDtT$cd!G!*#ABK|2(O(2&hd&es|>NuL31I;9$3qz$w=JKdSCc)WcwuYW|3)Xc9Djlr@QTqhiiO=g@iN9MuRjCUQ-?*aXzk zj?UsJ@frhB3gro!L)7n@5|a|>(%Gtb2SbU@kE9N4BStI-`#8OT{VZEF)o7;4RU1v! zpo(xtads+x1?eN}x><);X!cQVnD+7QA)JxLyD3@3dp1)(5YyOa7nk^hmwd}KKb^Cp zaUW`U+|g@+oQN@@+xhzbwiTbBBC$BQylPlCc8p!)Naw{tOBMyf@*JkAngcO}a|4z0 ujH#rmNNkn)qEblM&zCtQ9`)RicVdnN`3AO9eK1{Nh>i3EqH=bNpV8X4;sGBYqS zvM?|*GBP_eC^Bm$Ffh0>7%?$wurqvM&|~0aXk|~%FU>1SEn;P~Wmpe1`G-A&97C&& zFC&AIT~VqTqdzM%BSRio{|9as2L=Wu1|Nn5hQkcPjOL65jO!VHG72zpgPr7LbaOf* Yki`nbybKIl3=EU?SoAplgT#OU07CsE`~Uy| delta 167 zcmeys@q|MpC%-740ST-$?GD~3BFZFa%*4Rp;~(VDz@p?Uk>JpDd$Sc&BV(!nlR1MT zGqVGO0kak(2ZJCJGs6Q0PKH+Y>p79+cD-$=^8Sy^?9VY9r*l>Jigg6TT D-~S(w diff --git a/loro-js/tests/movable-list.test.ts b/loro-js/tests/movable-list.test.ts new file mode 100644 index 000000000..9628fe694 --- /dev/null +++ b/loro-js/tests/movable-list.test.ts @@ -0,0 +1,414 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, test } from "vitest"; + +import { + decodeFastSnapshot, + decodeSstable, + decodeStateSnapshotStore, + encodeStateSnapshotStore, +} from "../src/codec"; +import { + LoroDoc, + LoroList, + LoroMovableList, + LoroText, + UndoManager, + type LoroEventBatch, +} from "../src/index"; + +/** + * MovableList regressions against the Rust model. Expected values were taken + * from the Rust implementation (loro-crdt); see context/loro-js-movable-list.md. + */ + +const fixture = (name: string): Uint8Array => + new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); + +function doc(peer: number): LoroDoc { + const created = new LoroDoc(); + created.setPeerId(peer); + return created; +} + +function metadata(list: LoroMovableList): (string | undefined)[][] { + return Array.from({ length: list.length }, (_, index) => [ + list.getCreatorAt(index), + list.getLastMoverAt(index), + list.getLastEditorAt(index), + ]); +} + +function sync(...docs: LoroDoc[]): void { + for (const source of docs) { + for (const target of docs) { + if (source !== target) target.import(source.export({ mode: "update" })); + } + } +} + +describe("MovableList positions and elements", () => { + test("a concurrent move keeps an element another peer deleted", () => { + const a = doc(1); + const list = a.getMovableList("list"); + for (const value of ["a", "b", "c"]) list.push(value); + a.commit(); + const b = doc(2); + b.import(a.export({ mode: "update" })); + + list.delete(1, 1); + a.commit(); + b.getMovableList("list").move(1, 2); + b.getMovableList("list").set(2, "B"); + b.commit(); + sync(a, b); + + expect(a.toJSON()).toEqual({ list: ["a", "c", "B"] }); + expect(b.toJSON()).toEqual({ list: ["a", "c", "B"] }); + const c = doc(3); + c.import(a.export({ mode: "update" })); + expect(c.toJSON()).toEqual({ list: ["a", "c", "B"] }); + }); + + test("a delete after a move removes the moved element", () => { + const imported = new LoroDoc(); + imported.import(fixture("movable-move-delete.blob")); + expect(imported.toJSON()).toEqual({ list: ["b"] }); + + const local = doc(1); + const list = local.getMovableList("list"); + list.push("a"); + list.push("b"); + local.commit(); + list.move(0, 1); + local.commit(); + list.delete(1, 1); + local.commit(); + const replica = new LoroDoc(); + replica.import(local.export({ mode: "update" })); + expect(replica.toJSON()).toEqual({ list: ["b"] }); + }); + + test("the later of two concurrent moves wins; the loser position stays dead", () => { + const a = doc(1); + const list = a.getMovableList("list"); + for (const value of ["a", "b", "c"]) list.push(value); + a.commit(); + const b = doc(2); + b.import(a.export({ mode: "update" })); + list.move(0, 2); + a.commit(); + b.getMovableList("list").move(0, 1); + b.commit(); + sync(a, b); + expect(a.toJSON()).toEqual(b.toJSON()); + expect(a.toJSON()).toEqual({ list: ["b", "a", "c"] }); + // Inserting at the end goes after the dead position of the losing move. + list.push("d"); + a.commit(); + sync(a, b); + expect(b.toJSON()).toEqual({ list: ["b", "a", "c", "d"] }); + }); + + test("reports the creator, last mover and last editor", () => { + const a = doc(1); + const list = a.getMovableList("list"); + list.push("a"); + list.push("b"); + a.commit(); + const b = doc(2); + b.import(a.export({ mode: "update" })); + const other = b.getMovableList("list"); + other.set(0, "A"); + other.move(1, 0); + b.commit(); + expect(b.toJSON()).toEqual({ list: ["b", "A"] }); + expect(metadata(other)).toEqual([ + ["1", "2", "1"], + ["1", "1", "2"], + ]); + a.import(b.export({ mode: "update" })); + expect(metadata(list)).toEqual(metadata(other)); + }); + + test("records a set even when the value does not change", () => { + const a = doc(1); + const list = a.getMovableList("list"); + list.push("same"); + a.commit(); + const b = doc(2); + b.import(a.export({ mode: "update" })); + b.getMovableList("list").set(0, "same"); + b.commit(); + a.import(b.export({ mode: "update" })); + expect(list.getLastEditorAt(0)).toBe("2"); + }); +}); + +describe("MovableList snapshots use Rust's layout", () => { + test("reads Rust's list item, element and last-set IDs", () => { + const hydrated = new LoroDoc(); + hydrated.import(fixture("movable-snapshot.blob")); + const list = hydrated.getMovableList("list"); + expect(hydrated.toJSON()).toEqual({ list: ["B2", "B", "D", "d"] }); + expect(metadata(list)).toEqual([ + ["1", "1", "1"], + ["1", "1", "2"], + ["1", "3", "3"], + ["1", "1", "1"], + ]); + + // Concurrent edits made on a Rust replica and on a Rust peer that loaded + // the snapshot still converge (the latter replays history first). + hydrated.import(fixture("movable-snapshot-peer4.blob")); + hydrated.import(fixture("movable-snapshot-peer3.blob")); + expect(hydrated.toJSON()).toEqual({ list: ["y", "d", "x", "B", "D"] }); + expect(metadata(list)).toEqual([ + ["3", "3", "3"], + ["1", "4", "1"], + ["4", "4", "4"], + ["1", "1", "2"], + ["1", "3", "3"], + ]); + }); + + test("writes the MovableList state byte-for-byte like Rust", () => { + const rustSnapshot = decodeFastSnapshot(fixture("movable-snapshot.blob")); + const hydrated = new LoroDoc(); + hydrated.import(fixture("movable-snapshot-updates.blob")); + const exported = decodeFastSnapshot(hydrated.export({ mode: "snapshot" })); + const listState = (bytes: Uint8Array): Uint8Array => { + const store = decodeStateSnapshotStore(bytes); + if (store.kind !== "sstable") throw new Error("expected state entries"); + const entry = store.containers.find( + ({ id }) => id.kind === "root" && id.name === "list", + ); + return encodeStateSnapshotStore( + { kind: "sstable", frontiers: undefined, containers: [entry!] }, + { compression: "none" }, + ); + }; + expect(listState(exported.state)).toEqual(listState(rustSnapshot.state)); + expect(decodeSstable(exported.oplog).length).toBeGreaterThan(0); + }); +}); + +describe("MovableList versions, undo and diffs", () => { + test("checkout selects the winning position and value at each version", () => { + const a = doc(1); + const list = a.getMovableList("list"); + for (const value of ["a", "b", "c"]) list.push(value); + a.commit(); + const start = a.frontiers(); + list.move(0, 2); + list.set(0, "B"); + a.commit(); + const moved = a.frontiers(); + list.delete(2, 1); + a.commit(); + a.checkout(start); + expect(a.toJSON()).toEqual({ list: ["a", "b", "c"] }); + expect(metadata(list)).toEqual([ + ["1", "1", "1"], + ["1", "1", "1"], + ["1", "1", "1"], + ]); + a.checkout(moved); + expect(a.toJSON()).toEqual({ list: ["B", "c", "a"] }); + a.checkoutToLatest(); + expect(a.toJSON()).toEqual({ list: ["B", "c"] }); + // Rust: [delete 1, insert B, delete 1, retain 1, insert a], the same delta. + expect(a.diff(start, moved, false)).toEqual([ + [ + "cid:root-list:MovableList", + { + type: "list", + diff: [{ delete: 2 }, { insert: ["B"] }, { retain: 1 }, { insert: ["a"] }], + }, + ], + ]); + }); + + test("revertTo deletes before it inserts, like Rust's apply_delta", () => { + const p3 = doc(3); + p3.getMovableList("list").insert(0, "v2"); + p3.commit(); + const p1 = doc(1); + p1.import(p3.export({ mode: "update" })); + const withV2 = p1.frontiers(); + p1.getMovableList("list").delete(0, 1); + p1.getMovableList("list").insert(0, "v6"); + p1.commit(); + p3.import(p1.export({ mode: "update" })); + p1.revertTo(withV2); + p1.commit(); + p3.getMovableList("list").insert(0, "x"); + p3.commit(); + p1.import(p3.export({ mode: "update" })); + // Since loro-dev/loro#1138 Rust deletes unclaimed elements first, so the + // restored "v2" lands after the deleted positions and "x" before it. + expect(p1.toJSON()).toEqual({ list: ["v2", "x"] }); + }); + + test("undo of a move or set reinserts the old value, like Rust", () => { + const d = doc(1); + const undo = new UndoManager(d, { mergeInterval: 0 }); + const list = d.getMovableList("list"); + for (const value of ["a", "b", "c"]) list.push(value); + d.commit(); + list.move(0, 2); + d.commit(); + list.set(1, "C"); + d.commit(); + expect(d.toJSON()).toEqual({ list: ["b", "C", "a"] }); + undo.undo(); + expect(d.toJSON()).toEqual({ list: ["b", "c", "a"] }); + undo.undo(); + expect(d.toJSON()).toEqual({ list: ["a", "b", "c"] }); + undo.redo(); + expect(d.toJSON()).toEqual({ list: ["b", "c", "a"] }); + }); + + test("a shallow document keeps root-time winners when it retreats", () => { + const d = doc(1); + const list = d.getMovableList("list"); + list.push("a"); + list.push("b"); + d.commit(); + const other = doc(2); + other.import(d.export({ mode: "update" })); + other.getMovableList("list").set(0, "A"); + other.getMovableList("list").move(1, 0); + other.commit(); + d.import(other.export({ mode: "update" })); + list.push("c"); + d.commit(); + const root = d.frontiers(); + list.set(0, "b"); + list.move(0, 2); + d.commit(); + + const shallow = new LoroDoc(); + shallow.import(d.export({ mode: "shallow-snapshot", frontiers: root })); + expect(shallow.toJSON()).toEqual({ list: ["A", "c", "b"] }); + shallow.checkout(root); + expect(shallow.toJSON()).toEqual({ list: ["b", "A", "c"] }); + // The last editor of "b" is its creator. Rust's shallow root seeds the + // position writer instead and reports "2"; the full history agrees with 1. + expect(metadata(shallow.getMovableList("list"))).toEqual([ + ["1", "2", "1"], + ["1", "1", "2"], + ["1", "1", "1"], + ]); + }); + + test("a child container revived by a concurrent move reports its content", () => { + const a = doc(1); + const list = a.getMovableList("list"); + list.push("a"); + const text = list.insertContainer(1, new LoroText()); + text.insert(0, "hi"); + a.commit(); + const b = doc(2); + b.import(a.export({ mode: "update" })); + list.delete(1, 1); + a.commit(); + b.getMovableList("list").move(1, 0); + b.commit(); + const events: LoroEventBatch[] = []; + a.subscribe((event) => events.push(event)); + a.import(b.export({ mode: "update" })); + expect(a.toJSON()).toEqual({ list: ["hi", "a"] }); + const textEvent = events + .flatMap((batch) => batch.events) + .find((event) => event.target === text.id); + expect(textEvent?.diff).toEqual({ type: "text", diff: [{ insert: "hi" }] }); + }); +}); + +describe("MovableList cursors", () => { + test("anchor the list item, which a move leaves in place", () => { + const d = doc(1); + const list = d.getMovableList("list"); + for (const value of ["a", "b", "c"]) list.push(value); + d.commit(); + const cursor = list.getCursor(1)!; + expect(cursor.pos()).toEqual({ peer: "1", counter: 1 }); + expect(d.getCursorPos(cursor)?.offset).toBe(1); + list.move(1, 2); + d.commit(); + // "b"'s old list item was deleted by the move; the cursor now points at "c". + expect(d.getCursorPos(cursor)?.offset).toBe(1); + expect(list.toJSON()).toEqual(["a", "c", "b"]); + const end = list.getCursor(3)!; + expect(end.pos()).toBeUndefined(); + expect(d.getCursorPos(end)?.offset).toBe(3); + }); +}); + +describe("snapshot-hydrated documents replay history for concurrent imports", () => { + test("List and MovableList", () => { + for (const kind of ["List", "MovableList"] as const) { + const get = (target: LoroDoc): LoroList | LoroMovableList => + kind === "List" ? target.getList("list") : target.getMovableList("list"); + const a = doc(1); + for (const value of ["a", "b", "c"]) get(a).push(value); + a.commit(); + const c = doc(3); + c.import(a.export({ mode: "update" })); + get(a).delete(1, 1); + a.commit(); + // Inserted after "b", which the snapshot below no longer contains. + get(c).insert(2, "z"); + c.commit(); + + const hydrated = new LoroDoc(); + hydrated.import(a.export({ mode: "snapshot" })); + hydrated.import(c.export({ mode: "update" })); + a.import(c.export({ mode: "update" })); + expect(hydrated.toJSON()).toEqual(a.toJSON()); + expect(hydrated.toJSON()).toEqual({ list: ["a", "z", "c"] }); + } + }); +}); + +describe("shallow imports", () => { + test("a root at the empty version accepts changes without dependencies", () => { + const empty = doc(1); + const shallow = new LoroDoc(); + shallow.import(empty.export({ mode: "shallow-snapshot", frontiers: [] })); + const other = doc(2); + other.getMovableList("list").push("x"); + other.commit(); + shallow.import(other.export({ mode: "update" })); + expect(shallow.toJSON()).toEqual({ list: ["x"] }); + }); + + test("changes already inside the shallow root are skipped", () => { + const source = doc(3); + source.getMovableList("list").push("v"); + source.commit(); + const shallow = new LoroDoc(); + shallow.import( + source.export({ mode: "shallow-snapshot", frontiers: source.frontiers() }), + ); + shallow.import(source.export({ mode: "snapshot" })); + expect(shallow.toJSON()).toEqual({ list: ["v"] }); + }); + + test("writes a zero start-version entry for a root at a peer's first op", () => { + const source = doc(3); + source.getMovableList("list").push("v"); + source.commit(); + const bytes = source.export({ + mode: "shallow-snapshot", + frontiers: source.frontiers(), + }); + const entries = decodeSstable(decodeFastSnapshot(bytes).oplog); + const startVersion = entries.find( + ({ key }) => new TextDecoder().decode(key) === "sv", + )!.value; + // postcard: one entry, peer 3, counter 0. + expect([...startVersion]).toEqual([1, 3, 0]); + }); +}); diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 5d23e8e6b..5e4079b72 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -1791,6 +1791,12 @@ describe("loro-wasm-compatible runtime", () => { expect(count(() => doc.getCounter("counter").increment(0))).toBe(1); // Attaching a detached counter increments it by its value, even 0. expect(count(() => map.setContainer("child", new LoroCounter()))).toBe(2); + // A movable-list set is recorded even when the value is unchanged. + const movable = doc.getMovableList("movable"); + movable.push("same"); + doc.commit(); + expect(count(() => movable.set(0, "same"))).toBe(1); + expect(movable.toJSON()).toEqual(["same"]); }); test("lets a delete of an absent map key win against a concurrent set", () => { From f5dadabfc124de66a925e564a1e5a204324fdc15 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 01:40:45 +0800 Subject: [PATCH 09/20] test(loro.js): skip final syncs with a peer that left the run Co-Authored-By: Claude Opus 5.5 --- loro-js/tests/differential/fuzz.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index 7a02f3e3a..828ba9268 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -231,7 +231,8 @@ export class FuzzRun { ); for (const from of peers) { for (const to of peers) { - if (from === to) continue; + // A peer can leave the run during these syncs (see applySync). + if (from === to || !this.twins.has(from) || !this.twins.has(to)) continue; yield { kind: "sync", from, to, mode: "update", cross: this.profile.crossEngine }; } } From 0af5b740163d2a2cafe6840ba4ded3efd9757f40 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 03:46:58 +0800 Subject: [PATCH 10/20] fix(loro.js): keep pending changes and follow Rust's shallow import rules Decoding deferred snapshot history dropped pending changes. Shallow imports now check the unknown part of each change like is_before_shallow_root, and an import that only adds pending changes creates their root containers. Co-Authored-By: Claude Opus 5.5 --- context/loro-js-movable-list.md | 94 +++++++++++++++++++++--------- context/loro-js-performance.md | 7 ++- loro-js/src/runtime/document.ts | 79 +++++++++++++++++-------- loro-js/tests/movable-list.test.ts | 62 ++++++++++++++++++++ loro-js/tests/runtime.test.ts | 45 ++++++++++++++ 5 files changed, 233 insertions(+), 54 deletions(-) diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index 9acc89db8..bfb92f11d 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -133,35 +133,55 @@ fails loudly. in descending before-index order, then insert in ascending after-index order. Rust derives an import's event from the net change while loro.js composes the ops, so a multi-op import can show an element that was hidden - and shown again as a delete plus an insert of the same value; the effect is - the same. Getting Rust's exact shape would need an O(n) snapshot per import. -- Revival, as in Rust's `DocState::apply_diff`: in an import or checkout - batch, a child container of an element that was hidden when the batch - started and is visible at its end reports its whole state (and so do its - descendants), for example a Text brought back by a concurrent move. `diff()` - does the same. + and shown again as a delete plus an insert of the same value, and when + several imported moves bring an element back to its index, Rust's net + change cancels and it reports nothing while loro.js reports a delete plus + an insert. The effect is the same. Getting Rust's exact shape would need an + O(n) snapshot per import, and Rust composes element updates in `FxHashMap` + order. +- Revival, as in Rust's `DocState::apply_diff`: a child container that an + import, checkout or `diff()` makes reachable again (for example a Text brought + back by a concurrent move) reports its whole state. This is main's generic + `#completeDiffEntries`; for a MovableList child, reachability at a version + comes from `MovableListState.visibleValueAt` (winning position alive and + winning value is that child). - Snapshot export follows section 8 exactly. Import hydrates alive positions (visible and dead) and visible elements with their real position, element - and last-set IDs. Hydrated state has no tombstones, origins or candidate - history, so the container is marked incomplete. Version transitions on it - replay history once. `LoroDoc.#snapshotStateVersion` records the hydrated - version; `#needsHistoryReplay` replays history before an import (or an - attach) containing a record that does not depend on that whole version, or a - MovableList move/set naming an element the state lacks. Such an op's causal - view can need tombstones the snapshot dropped. The same rule fixes List and - Text, which had the same gap. `#discardDeferredSnapshotState` (also in - loro.js #1126) drops the lazy state first so it is not hydrated again on top - of the replay. + and last-set IDs, each seeded as the element's only candidate. Hydrated + state has no tombstones, origins or older candidates (`historyComplete` is + false), so it is right only from the snapshot's version on. A MovableList + is therefore a snapshot sequence like Text and List (loro.js #1126, + `#markSnapshotSequence`); its snapshot names Rust IDs, so the replay of its + history is comparable to the snapshot state (`sameMovableListStates`). + Before a version transition touches it, `#prepareSnapshotTransition` + rebuilds that one container from its own history (`#completeSnapshotSequence`); + a Text or List only when the transition crosses a snapshot op, a MovableList + always, since its candidates are partial. Before an import, + `#prepareSnapshotImport` does the same for each container that a record + touches concurrently with the snapshot version (its causal view can need + tombstones the snapshot dropped; loro-dev/loro#1163 for List and Text) or, + in a MovableList, whose move/set names an element the state lacks. The + import then applies its records as usual, so its events are the ops' events, + as in Rust. A container whose replay differs from its snapshot state (a + snapshot written by loro.js 0.2) stays `unreplayable`, as #1126 describes. - A shallow root store seeds root-time position and value candidates, the counterpart of `record_shallow_root_state`, so retreat to the root keeps root-time winners (the MovableList part of Rust #1124 / loro.js #1127). +- Shallow imports follow Rust's `is_before_shallow_root` on the part of each + change that is not known yet (`#assertImportsNotOutdated`): a change sliced + at the known version depends on the op before the slice. An import that + applies no change still creates the root containers of its pending changes + (`#materializePendingRoots`, Rust's `pending_root_containers_to_materialize`). + Decoding deferred snapshot history keeps the document's pending changes. ### Undo, `revertTo`, `applyDiff` -`applyDiff` and `revertTo` port Rust's MovableList `apply_delta` -(`applyMovableListDelta` in `document.ts`): insert first, defer deletes, and -turn a reinserted child container into a `move`, following the container -remap chain. +`applyDiff` and `revertTo` follow Rust's MovableList `apply_delta` after +loro-dev/loro#1138 (`#applyMovableListMoves`, main's port on the new model; +[movable-list-apply-diff.md](movable-list-apply-diff.md)): delete unclaimed +elements right to left, then place each insert after its predecessor, and turn +a reinserted child container into a `move`, following the container remap +chain. Undo follows Rust's `undo_internal` for the MovableList containers of an undo item (`#undoMovableLists`). A is the inverse diff from the span's last op back @@ -169,8 +189,13 @@ to its dependencies, B is the change since the span, and A transformed over B (A's inserts first, like `transform(.., left_priority = true)`) is applied with `apply_delta`. Undoing a `move`/`set` therefore inserts the old value as a new element and deletes the current one, and a child container moves back. Other -containers keep loro.js's op-based undo. B also contains local changes that -Rust's per-item remote diff excludes; see the remaining divergences. +containers keep loro.js's op-based undo. Rust's B is its remote diff, the +changes the UndoManager did not make; loro.js transforms a list's inverse only +when such an untracked change touched that list (the UndoManager's +`isTracked`), and then over everything since, which is approximate. An insert +of a child that is still visible outside the delta's deleted ranges is dropped, +Rust's rule for `from_move` inserts that transforms leave behind +(`#withoutLiveChildInserts`). ## Validation and import atomicity @@ -251,16 +276,31 @@ Differences in form, not in effect: order over arena indices, so the two engines write the same values with a different op order when several same-depth containers change. The suite checks such a revert by value on a fork and replicates Rust's ops. -- Event deltas of multi-op imports and of history replays can differ in shape - (see "Versions, events, snapshots"); the suite then checks that each - engine's delta turns the previous value into the new one. +- `diff()` list deltas can put an insert and a delete at one index in the + other order than Rust's `DeltaRope`, which composes element updates one by + one (an insert at or past the end goes after trailing deletes, others before + them) in `FxHashMap` order. `revertTo`, undo and `applyDiff` turn an insert + before a delete into an insert before the deleted positions, so the two + engines can place a restored element on different sides of dead positions: + the reverted value is the same, but later concurrent inserts there can + interleave differently. The suite checks such a revert by value and + replicates Rust's ops. +- Event deltas of multi-op imports can differ in shape, or be a no-op where + Rust reports nothing (see "Versions, events, snapshots"); the suite then + checks that each engine's deltas turn the previous value into the new one. Rust issues found: - On a shallow document, `checkout(root)` reports `getLastEditorAt` from the root list item's peer: `record_shallow_root_state` (`history_cache.rs`) seeds the value writer with the list item ID. The full history reports the - real setter, and so does loro.js. + real setter, and so does loro.js. Imports whose diff crosses the root show + the same wrong writer (for example the last mover), so the suite does not + compare `getLastEditorAt` on shallow documents. +- `updates-in-range` on a shallow document exports nothing for a span that + starts in trimmed history: `ChangeStore::iter_blocks` looks up the block at + the span start and finds none. loro.js exports the part it has. The suite + starts such spans at the shallow root. - An update whose changes partly depend on history before the shallow root applies the other changes and then returns `ImportUpdatesThatDependsOnOutdatedVersion`, so the import is not atomic. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index fd8a8b100..42d522fa7 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -527,9 +527,10 @@ The remaining differences are representation and JavaScript constant factors: emitted output. Without a subscriber, both hide and show transitions use the reversible lazy visibility layer and stay proportional to affected ID runs. - State hydrated from a latest-state snapshot has no tombstones or MovableList - candidate history. The first import that is concurrent with that state, or - names a MovableList element it lacks, replays history once - (`#needsHistoryReplay`); so do version transitions on such a MovableList. + candidate history. The first import that touches such a container + concurrently with the snapshot, or names a MovableList element it lacks, + rebuilds only that container from its own history (`#prepareSnapshotImport`); + so does the first version transition that touches a hydrated MovableList. Later imports and transitions are incremental again. - The million-operation C1.1 concurrent-text trace still exposes a large constant-factor and retained-memory gap. Its local edit phase is about 4x the diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 58262b3bc..ad569051a 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -707,6 +707,9 @@ export class LoroDoc = Record 0 && !this.#detached) { this.#prepareSnapshotImport(added, beforeVersion); const recording = this.#hasEventSubscribers() @@ -914,6 +917,8 @@ export class LoroDoc = Record 0) { this.#prepareSnapshotImport(integration.added, beforeVersion); const recording = this.#hasEventSubscribers() @@ -5429,33 +5434,58 @@ export class LoroDoc = Record changeKey(change.id))); + for (const { change } of imported) { + if (!pendingChanges.has(changeKey(change.id))) continue; + for (const operation of change.operations) { + const id = operation.container; + if (id.kind === "root" && !isMergeableContainerId(id)) + this.#getOrCreateContainer(id); + } + } + } + #assertImportsNotOutdated(records: readonly HistoryRecord[]): void { if (this.#shallowRootStore === undefined) return; - // Like Rust's `is_before_shallow_root`, a root at the empty version trims - // nothing, so a change without dependencies is not outdated. - const trimmed = this.#shallowRootFrontiers.length > 0; + // Rust's `is_before_shallow_root`: a root at the empty version trims + // nothing, so nothing is outdated. (A root at a peer's first op still has a + // zero start-version entry, and then a change without dependencies is.) + if (this.#shallowRootFrontiers.length === 0) return; + const start = this.#shallowStartVersion; + const root = this.#shallowRootFrontiers.map(formatOpId); + const known = this.#historyVersion(); for (const { change } of records) { - // Changes up to the root are already part of the root state; a change that - // straddles the root continues from the root op. - const rootEnd = Math.max( - this.#shallowStartVersion.get(change.id.peer) ?? 0, - this.#shallowRootVersion.get(change.id.peer) ?? 0, - ); - if (change.id.counter < rootEnd) continue; - if (change.dependencies.length === 0 && trimmed) { - throw new Error("cannot import updates that depend on an outdated version"); - } - - const touchesShallowRoot = change.dependencies.some((dependency) => - this.#shallowRootFrontiers.some((frontier) => idsEqual(frontier, dependency)), + // Like Rust, only the part of a change that is not known yet is checked; + // a change sliced at `counter` depends on the op before it. + const { peer } = change.id; + const counter = Math.max( + known.get(peer) ?? 0, + this.#shallowRootVersion.get(peer) ?? 0, ); - if (touchesShallowRoot) continue; - - const dependsOnPrunedHistory = change.dependencies.some( - (dependency) => - dependency.counter < (this.#shallowStartVersion.get(dependency.peer) ?? 0), - ); - if (dependsOnPrunedHistory) { + if (change.id.counter + changeLength(change) <= counter) continue; + const sliced = change.id.counter < counter; + const ids = sliced ? [{ peer, counter: counter - 1 }] : change.dependencies; + const dependencies = ids.map(formatOpId); + // A change needs the state at its dependencies: they must be after the + // root or exactly the root. + if ( + dependencies.length === 0 || + ids.some( + (dependency) => dependency.counter < (start.get(dependency.peer) ?? 0), + ) || + (dependencies.some((dependency) => root.includes(dependency)) && + !frontierSetsEqual(dependencies, root)) + ) { throw new Error("cannot import updates that depend on an outdated version"); } } @@ -7025,7 +7055,8 @@ export class LoroDoc = Record { + const b = doc(2); + const list = b.getMovableList("list"); + list.insertContainer(0, new LoroText()); + list.insert(1, "v8"); + b.commit(); + const c = doc(3); + c.import(b.export({ mode: "update" })); + list.insert(2, 12); + list.move(0, 2); + b.commit(); + const hydrated = new LoroDoc(); + hydrated.import(b.export({ mode: "snapshot" })); + // Concurrent with the snapshot: "v8" moves, but stays at index 0. + c.getMovableList("list").move(1, 0); + c.commit(); + + const events: LoroEventBatch[] = []; + hydrated.subscribe((event) => events.push(event)); + hydrated.import(c.export({ mode: "update" })); + expect(hydrated.toJSON()).toEqual({ list: ["v8", 12, ""] }); + expect(events.map(({ events }) => events.map(({ diff }) => diff))).toEqual([ + [{ type: "list", diff: [{ delete: 1 }, { insert: ["v8"] }] }], + ]); + }); }); describe("shallow imports", () => { @@ -396,6 +422,42 @@ describe("shallow imports", () => { expect(shallow.toJSON()).toEqual({ list: ["v"] }); }); + test("the part of a change after the root must not depend on trimmed history", () => { + const a = doc(3); + a.getMovableList("list").push("a"); + a.commit(); + const b = doc(1); + b.import(a.export({ mode: "update" })); + b.getMovableList("list").push("x"); + b.commit(); + const shallow = new LoroDoc(); + shallow.import(b.export({ mode: "shallow-snapshot", frontiers: b.frontiers() })); + // One change holds `a` (before the root) and `b`, which depends on `a`. + a.getMovableList("list").push("b"); + a.commit(); + expect(a.getAllChanges().get("3")).toHaveLength(1); + expect(() => shallow.import(a.export({ mode: "update" }))).toThrow(/outdated/u); + expect(shallow.toJSON()).toEqual({ list: ["a", "x"] }); + }); + + test("a root at a peer's first op rejects only changes without dependencies", () => { + const a = doc(1); + a.getMovableList("list").push("x"); + a.commit(); + const shallow = new LoroDoc(); + shallow.import(a.export({ mode: "shallow-snapshot", frontiers: a.frontiers() })); + const other = doc(2); + other.getMovableList("list").push("y"); + other.commit(); + expect(() => shallow.import(other.export({ mode: "update" }))).toThrow(/outdated/u); + const next = doc(4); + next.import(a.export({ mode: "update" })); + next.getMovableList("list").push("z"); + next.commit(); + shallow.import(next.export({ mode: "update" })); + expect(shallow.toJSON()).toEqual({ list: ["x", "z"] }); + }); + test("writes a zero start-version entry for a root at a peer's first op", () => { const source = doc(3); source.getMovableList("list").push("v"); diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 5e4079b72..491e1a39e 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -1417,6 +1417,51 @@ describe("loro-wasm-compatible runtime", () => { expect(target.getAllChanges()).toEqual(source.getAllChanges()); }); + test("creates the root containers of an import that is only pending", () => { + const source = new LoroDoc(); + source.setPeerId(1); + source.getText("t").insert(0, "a"); + source.commit(); + const version = source.version(); + source.getMap("m").set("k", 1); + source.getMovableList("l").push(1); + source.commit(); + const update = source.export({ mode: "update", from: version }); + + // Like Rust: the roots exist, empty, until their changes apply. + const target = new LoroDoc(); + expect(target.import(update).pending).not.toBeNull(); + expect(target.toJSON()).toEqual({ l: [], m: {} }); + const edited = new LoroDoc(); + edited.getText("t").insert(0, "q"); + edited.commit(); + edited.import(update); + expect(edited.toJSON()).toEqual({ t: "q", l: [], m: {} }); + }); + + test("keeps pending changes when a lazy snapshot's history is materialized", () => { + const source = new LoroDoc(); + source.setPeerId(9); + source.getText("text").insert(0, "a"); + source.commit(); + const snapshot = source.export({ mode: "snapshot" }); + const version = source.version(); + source.getText("text").push("b"); + source.commit(); + const missing = source.export({ mode: "update", from: version }); + const afterMissing = source.version(); + source.getText("text").push("c"); + source.commit(); + const later = source.export({ mode: "update", from: afterMissing }); + + const target = LoroDoc.fromSnapshot(snapshot); + expect(target.import(later).pending).not.toBeNull(); + // History queries decode the snapshot's history. + expect(target.changeCount()).toBe(1); + target.import(missing); + expect(target.toJSON()).toEqual({ text: "abc" }); + }); + test("keeps deferred history atomic when snapshot metadata is inconsistent", () => { const source = new LoroDoc(); source.setPeerId(11); From a77fd0edeae264ae114d8c8e45e0c8c51bfbb35d Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 04:13:32 +0800 Subject: [PATCH 11/20] test(loro.js): pin diff() of a MovableList child created in the range main's diff() reports such a child by its ops, including map keys set and deleted again, as Rust does. Co-Authored-By: Claude Opus 5.5 --- loro-js/tests/movable-list.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/loro-js/tests/movable-list.test.ts b/loro-js/tests/movable-list.test.ts index 89709827c..57855bd81 100644 --- a/loro-js/tests/movable-list.test.ts +++ b/loro-js/tests/movable-list.test.ts @@ -11,6 +11,7 @@ import { import { LoroDoc, LoroList, + LoroMap, LoroMovableList, LoroText, UndoManager, @@ -229,6 +230,26 @@ describe("MovableList versions, undo and diffs", () => { ]); }); + test("diff reports a child created in the range by its ops", () => { + const a = doc(1); + const list = a.getMovableList("list"); + list.push(0); + a.commit(); + const start = a.frontiers(); + const child = list.insertContainer(0, new LoroMap()); + child.set("a", 1); + a.commit(); + child.set("k", 9); + a.commit(); + child.delete("k"); + a.commit(); + // Rust lists `k`, which was set and deleted again, as deleted. + expect(a.diff(start, a.frontiers(), false)).toEqual([ + ["cid:root-list:MovableList", { type: "list", diff: [{ insert: [child] }] }], + [child.id, { type: "map", updated: { a: 1, k: undefined } }], + ]); + }); + test("revertTo deletes before it inserts, like Rust's apply_delta", () => { const p3 = doc(3); p3.getMovableList("list").insert(0, "v2"); From b254f1588f5ff6c2ee45a9070d2cccba053e39aa Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 28 Sep 2026 04:13:19 +0800 Subject: [PATCH 12/20] perf(loro.js): resolve concurrent MovableList op indices through a tracker A causal view per concurrent op made importing concurrent moves O(n^2). Like Rust's Tracker::checkout, keep a tracker version and a per-position delta metric that moves incrementally between op versions; see context/loro-js-movable-list.md. Co-Authored-By: Claude Opus 5.5 --- context/loro-js-movable-list.md | 24 +++-- context/loro-js-performance.md | 6 +- loro-js/benchmarks/movable-list.mjs | 124 +++++++++++++++++++++ loro-js/package.json | 1 + loro-js/src/runtime/containers.ts | 4 +- loro-js/src/runtime/movable-list.ts | 149 ++++++++++++++++++++++---- loro-js/src/runtime/sequence-index.ts | 91 ++++++++++++++++ loro-js/tests/movable-list.test.ts | 47 ++++++++ 8 files changed, 419 insertions(+), 27 deletions(-) create mode 100644 loro-js/benchmarks/movable-list.mjs diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index bfb92f11d..e237b7a33 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -95,18 +95,30 @@ fails loudly. - `positions: SequenceIndex`: the Fugue sequence of positions. `deleted` means "not alive in the tracker"; deletion records (`deletedBy`) hold the delete or move op that removed it, which gives causal views and - version transitions for free. A position's `utf16`/`utf8` metrics are 1 when - it is pointed and 0 otherwise, so the user-visible count is the sequence's - visible metric and user ↔ op index conversion is O(log n) through the - existing metric queries. `SequenceIndex.refreshMetrics` recomputes one - element's node and its ancestors when pointedness changes. + version transitions for free. A position's `utf16` metric is 1 when it is + pointed and 0 otherwise, so the user-visible count is the sequence's visible + metric and user ↔ op index conversion is O(log n) through the existing + metric queries. `SequenceIndex.refreshMetrics` recomputes one element's node + and its ancestors when pointedness changes. The `utf8` metric holds the + tracker delta (below). - `elements: Map>` keyed by element `IdLp`. Each element keeps its current winner position, value and value writer, plus history arrays sorted by `IdLp` (positions it may point to, and value candidates). Winner selection at a version scans candidates from the greatest `IdLp` down until one is included, which is the same work as Rust's `BTreeSet` range scan. -- Remote ops are applied in causal order: +- Remote ops are applied in causal order. An op's indices count the positions + alive at its causal version. When that is not the current state (the op is + concurrent with ops already applied), the state resolves them through a + tracker, like Rust's `Tracker::checkout`: it keeps a tracker version and + gives each position a delta, (alive at the tracker version) - (alive now), + as its `utf8` metric, so `SequenceIndex.atTracked`/`trackedIndexOf` count + that view in O(log n). Moving the tracker revisits only the positions that + the ops between the two versions created or deleted, and each applied op + moves it past itself, so a run of concurrent ops costs + O((ops + positions they touch) · log n) instead of one O(n) causal view per + op. An op that sees the whole state, and every version transition, clears + the tracker through the set of positions with a nonzero delta. - `Insert`: Fugue insertion at the op index in the op's causal view, then new elements. - `Delete`: deletes the target position IDs. Rust deletes by index; the two diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 42d522fa7..9fe35cf98 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -73,7 +73,11 @@ JavaScript constant factor. and value from its candidates, newest first. It costs O((affected ops + skipped candidates) · log n), like Rust's `last_pos` scan, with no replay. Unrelated document history and container state are not - rebuilt. + rebuilt. Imported ops that are concurrent with applied ones resolve their + indices through a tracker version (a per-position delta metric) that moves + incrementally between op versions, like Rust's `Tracker::checkout`, instead + of building a causal view per op; `pnpm --dir loro-js bench:movable-list` + measures it. - Contiguous Text/List insertion and deletion transitions reuse the physical ID runs and reversible lazy subtree visibility in both directions. Without an event subscriber, hiding or showing one complete run is expected O(log n + diff --git a/loro-js/benchmarks/movable-list.mjs b/loro-js/benchmarks/movable-list.mjs new file mode 100644 index 000000000..3bf21131f --- /dev/null +++ b/loro-js/benchmarks/movable-list.mjs @@ -0,0 +1,124 @@ +/* eslint-disable no-console */ + +// MovableList scaling benchmark. Uses only the public API so that it runs +// against any loro.js build; see context/loro-js-movable-list.md. + +import { performance } from "node:perf_hooks"; + +import { LoroDoc } from "../dist/index.js"; + +const positionalArguments = process.argv.slice(2).filter((argument) => argument !== "--"); +const sizes = (positionalArguments[0] ?? "1000,2000,4000,8000").split(",").map(Number); + +function measure(name, size, callback) { + globalThis.gc?.(); + const start = performance.now(); + const result = callback(); + const milliseconds = performance.now() - start; + console.log(JSON.stringify({ name, size, milliseconds, result })); + return result; +} + +function random(seed) { + let state = seed >>> 0; + return (limit) => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return (((value ^ (value >>> 14)) >>> 0) % limit) | 0; + }; +} + +function filled(peer, size) { + const doc = new LoroDoc(); + doc.setPeerId(peer); + const list = doc.getMovableList("list"); + for (let index = 0; index < size; index += 1) list.push(index); + doc.commit(); + return doc; +} + +/** `count` random moves (and a set every fourth op), committing every 50 ops. */ +function edit(doc, count, seed) { + const list = doc.getMovableList("list"); + const next = random(seed); + for (let index = 0; index < count; index += 1) { + if (index % 4 === 3) list.set(next(list.length), -index); + else list.move(next(list.length), next(list.length)); + if (index % 50 === 49) doc.commit(); + } + doc.commit(); +} + +for (const size of sizes) { + measure("movable-push", size, () => filled(1, size).getMovableList("list").length); + + const base = filled(1, size); + const baseVersion = base.oplogVersion(); + const baseFrontiers = base.frontiers(); + const local = base.fork(); + local.setPeerId(2); + measure("movable-local-move-set", size, () => { + edit(local, size, 7); + return local.getMovableList("list").length; + }); + const update = local.export({ mode: "update", from: baseVersion }); + + measure("movable-import-moves", size, () => { + const target = base.fork(); + target.import(update); + return target.getMovableList("list").length; + }); + + const concurrent = base.fork(); + concurrent.setPeerId(3); + edit(concurrent, size, 11); + const concurrentUpdate = concurrent.export({ mode: "update", from: baseVersion }); + measure("movable-import-concurrent-moves", size, () => { + const target = local.fork(); + target.import(concurrentUpdate); + return target.getMovableList("list").length; + }); + + const merged = local.fork(); + merged.import(concurrentUpdate); + measure( + "movable-snapshot-export", + size, + () => merged.export({ mode: "snapshot" }).length, + ); + const bytes = merged.export({ mode: "snapshot" }); + measure("movable-snapshot-import-to-json", size, () => { + const target = new LoroDoc(); + target.import(bytes); + return target.getMovableList("list").toJSON().length; + }); + measure("movable-snapshot-import-concurrent-update", size, () => { + const target = new LoroDoc(); + target.import(local.export({ mode: "snapshot" })); + target.import(concurrentUpdate); + return target.getMovableList("list").length; + }); + + const latest = merged.frontiers(); + measure("movable-checkout-before-moves", size, () => { + merged.checkout(baseFrontiers); + return merged.getMovableList("list").length; + }); + measure("movable-checkout-latest", size, () => { + merged.checkout(latest); + merged.attach(); + return merged.getMovableList("list").length; + }); + measure("movable-metadata", size, () => { + const list = merged.getMovableList("list"); + let checksum = 0; + for (let index = 0; index < list.length; index += 1) { + checksum += + Number(list.getLastMoverAt(index) ?? 0) + + Number(list.getLastEditorAt(index) ?? 0); + } + return checksum; + }); +} diff --git a/loro-js/package.json b/loro-js/package.json index 2785b414c..7dd1a5e9e 100644 --- a/loro-js/package.json +++ b/loro-js/package.json @@ -46,6 +46,7 @@ "scripts": { "bench:b4": "pnpm build && node --expose-gc benchmarks/b4.mjs", "bench:complexity": "pnpm build && node --expose-gc benchmarks/complexity.mjs", + "bench:movable-list": "pnpm build && node --expose-gc benchmarks/movable-list.mjs", "bench:snapshot-memory": "pnpm build && node --expose-gc benchmarks/snapshot-memory.mjs", "bench:subscribers": "pnpm build && node --expose-gc benchmarks/container-subscriber.mjs", "bench:text-buffer": "pnpm build && node --expose-gc benchmarks/text-buffer.mjs", diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 8c5e85547..d8080965b 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -3531,6 +3531,7 @@ export function insertFugueElements( inserted: T[], causalVersion: CausalVersion, useOriginIndex = true, + positionHint?: FuguePositionHint, ): void { if (inserted.length === 0) return; @@ -3540,6 +3541,7 @@ export function insertFugueElements( inserted[0]!.id, causalVersion, useOriginIndex, + positionHint, ); const { insertIndex, originLeft, originRight } = insertion; @@ -3580,7 +3582,7 @@ interface FugueInsertionResult { readonly indexUpdate?: FugueOriginIndex | undefined; } -interface FuguePositionHint { +export interface FuguePositionHint { readonly current: boolean; readonly left: T | undefined; readonly startIndex?: number | undefined; diff --git a/loro-js/src/runtime/movable-list.ts b/loro-js/src/runtime/movable-list.ts index c52ed7c0d..ecaa9111d 100644 --- a/loro-js/src/runtime/movable-list.ts +++ b/loro-js/src/runtime/movable-list.ts @@ -30,6 +30,8 @@ export interface MovablePosition extends SequenceElement { * position read from a snapshot, whose element is not recorded. */ element: MovableElement | undefined; + /** Alive at the tracker version minus alive now; see `MovableListState`. */ + trackerDelta?: number | undefined; } export interface MovableValueCandidate { @@ -76,14 +78,18 @@ export type MovableTransitionOp = export class InvalidMovableListOpError extends Error {} -const POINTED: SequenceMetrics = { utf16: 1, utf8: 1 }; -const UNPOINTED: SequenceMetrics = { utf16: 0, utf8: 0 }; +/** Indexed by pointedness, then by tracker delta + 1. */ +const POSITION_METRICS: readonly (readonly SequenceMetrics[])[] = [0, 1].map((utf16) => + [-1, 0, 1].map((utf8) => ({ utf16, utf8 })), +); -/** A position counts as one user-visible item iff its element points at it. */ +/** + * A position counts as one user-visible item (`utf16`) iff its element points + * at it. `utf8` holds its tracker delta. + */ function positionMetrics(position: MovablePosition): SequenceMetrics { - return position.element !== undefined && position.element.pos === position - ? POINTED - : UNPOINTED; + const pointed = position.element !== undefined && position.element.pos === position; + return POSITION_METRICS[pointed ? 1 : 0]![(position.trackerDelta ?? 0) + 1]!; } export function compareLamportIds( @@ -172,6 +178,18 @@ export class MovableListState { * are concurrent with that state must replay history first. */ historyComplete = true; + /** + * Rust's tracker (`Tracker::checkout` in the MovableList diff calculator). + * An op's indices count the positions alive at its causal version. Instead + * of building that view per op, the tracker keeps a version and gives each + * position a delta, (alive there) - (alive now), as its `utf8` metric, so + * `positions.atTracked`/`trackedIndexOf` resolve indices in O(log n). + * Moving it to another version revisits only the positions that the ops in + * between created or deleted, and each applied op moves it past itself. + * Undefined when the tracker is at the current state (all deltas zero). + */ + #trackerVersion: Map | undefined; + readonly #trackedPositions = new Set(); constructor(readonly bindValue: (element: MovableElement) => void) {} @@ -186,6 +204,8 @@ export class MovableListState { } reset(): void { + this.#trackerVersion = undefined; + this.#trackedPositions.clear(); this.positions.reset(); this.#elements.clear(); this.historyComplete = true; @@ -289,7 +309,10 @@ export class MovableListState { events?: MovableListEvents, ): void { if (values.length === 0) return; - const viewLength = this.#viewLength(causalVersion); + const tracked = this.#track(causalVersion); + const viewLength = tracked + ? this.positions.trackedLength + : this.positions.visibleLength; if (!Number.isSafeInteger(opIndex) || opIndex < 0 || opIndex > viewLength) { throw new InvalidMovableListOpError( `movable-list insert position ${opIndex} is out of range (length ${viewLength})`, @@ -324,7 +347,15 @@ export class MovableListState { } // The origin index misorders a sibling subtree followed by a concurrent // element that is not its descendant, so positions use the scan. - insertFugueElements(this.positions, opIndex, inserted, causalVersion, false); + insertFugueElements( + this.positions, + opIndex, + inserted, + causalVersion, + false, + tracked ? this.#trackedLeft(opIndex) : undefined, + ); + if (tracked) this.#trackPast(firstId, values.length); for (const position of inserted) this.bindValue(position.element!); if (events !== undefined) { events.insertList( @@ -356,6 +387,7 @@ export class MovableListState { events, ); } + if (this.#trackerVersion !== undefined) this.#trackPast(deletedBy, count); } /** @@ -377,12 +409,13 @@ export class MovableListState { `movable-list move targets unknown element L${elementId.lamport}@${elementId.peer}`, ); } - const current = this.positions.isFullyIncluded(causalVersion); - const view = current ? undefined : this.positions.causalView(causalVersion); - const viewLength = view?.length ?? this.positions.visibleLength; - const source = Number.isSafeInteger(from) - ? (view?.at(from) ?? (current ? this.positions.atVisible(from) : undefined)) - : undefined; + const tracked = this.#track(causalVersion); + const viewLength = tracked + ? this.positions.trackedLength + : this.positions.visibleLength; + const source = tracked + ? this.positions.atTracked(from) + : this.positions.atVisible(from); if (source === undefined) { throw new InvalidMovableListOpError( `movable-list move source ${from} is out of range (length ${viewLength})`, @@ -395,6 +428,8 @@ export class MovableListState { ); } this.#deletePosition(source, opId, events); + // `to` counts the view after the source is deleted. + if (tracked) this.#setTrackedAlive(source, false); const afterDelete = new Map(causalVersion); afterDelete.set( opId.peer, @@ -409,7 +444,15 @@ export class MovableListState { originRight: undefined, element, }; - insertFugueElements(this.positions, to, [position], afterDelete, false); + insertFugueElements( + this.positions, + to, + [position], + afterDelete, + false, + tracked ? this.#trackedLeft(to) : undefined, + ); + if (tracked) this.#trackPast(opId, 1); if (element.positions !== undefined) { insertSorted(element.positions, position, (left, right) => compareLamportIds(positionWriter(left), positionWriter(right)), @@ -454,6 +497,7 @@ export class MovableListState { includes: (id: CodecId) => boolean, events?: MovableListEvents, ): void { + this.#clearTracker(); const positions = new Set(); const elements = new Set(); const addPosition = (position: MovablePosition | undefined): void => { @@ -791,10 +835,77 @@ export class MovableListState { byLamport.set(element.lamport, element); } - #viewLength(causalVersion: CausalVersion): number { - return this.positions.isFullyIncluded(causalVersion) - ? this.positions.visibleLength - : this.positions.causalView(causalVersion).length; + /** + * Moves the tracker to `version`. Returns false, with the tracker cleared, + * when `version` includes every op applied so far (the current state). + */ + #track(version: CausalVersion): boolean { + if (this.positions.isFullyIncluded(version)) { + this.#clearTracker(); + return false; + } + const from = this.#trackerVersion ?? this.positions.idEnds(); + const affected = new Set(); + const visit = (position: MovablePosition): void => { + affected.add(position); + }; + for (const peer of new Set([...from.keys(), ...version.keys()])) { + const start = Math.min(from.get(peer) ?? 0, version.get(peer) ?? 0); + const end = Math.max(from.get(peer) ?? 0, version.get(peer) ?? 0); + if (start === end) continue; + this.positions.forEachWithIdIn(peer, start, end, visit); + for (const position of this.positions.elementsDeletedBy(peer, start, end)) { + affected.add(position); + } + } + const target = new Map(version); + this.#trackerVersion = target; + const includes = (id: CodecId): boolean => id.counter < (target.get(id.peer) ?? 0); + for (const position of affected) { + this.#setTrackedAlive( + position, + includes(position.id) && !this.positions.someDeletion(position, includes), + ); + } + return true; + } + + /** Moves the tracker past the op `id..id+length` it just applied. */ + #trackPast(id: CodecId, length: number): void { + const version = this.#trackerVersion; + if (version === undefined) return; + const next = new Map(version); + next.set(id.peer, Math.max(next.get(id.peer) ?? 0, id.counter + length)); + this.#track(next); + } + + #setTrackedAlive(position: MovablePosition, alive: boolean): void { + const delta = (alive ? 1 : 0) - (position.deleted ? 0 : 1); + if (delta === (position.trackerDelta ?? 0)) return; + position.trackerDelta = delta === 0 ? undefined : delta; + if (delta === 0) this.#trackedPositions.delete(position); + else this.#trackedPositions.add(position); + this.positions.refreshMetrics(position); + } + + #clearTracker(): void { + this.#trackerVersion = undefined; + for (const position of this.#trackedPositions) { + position.trackerDelta = undefined; + this.positions.refreshMetrics(position); + } + this.#trackedPositions.clear(); + } + + /** Fugue origins for an insertion at tracked index `index`. */ + #trackedLeft(index: number): { + readonly current: false; + readonly left: MovablePosition | undefined; + } { + return { + current: false, + left: index === 0 ? undefined : this.positions.atTracked(index - 1), + }; } #deletePosition( diff --git a/loro-js/src/runtime/sequence-index.ts b/loro-js/src/runtime/sequence-index.ts index 31bbc3745..35a660a37 100644 --- a/loro-js/src/runtime/sequence-index.ts +++ b/loro-js/src/runtime/sequence-index.ts @@ -221,6 +221,83 @@ export class SequenceIndex { return visibleCount(this.#root); } + /** + * Visible elements plus the `utf8` metric of every element. A caller that + * keeps a 0/±1 delta in `utf8` (MovableList's tracker; see + * context/loro-js-movable-list.md) indexes a second view with it, where each + * element counts 0 or 1: `trackedLength`, `atTracked` and `trackedIndexOf` + * are O(log n) like their visible counterparts. + */ + get trackedLength(): number { + return trackedCount(this.#root); + } + + atTracked(index: number): T | undefined { + if (!Number.isSafeInteger(index) || index < 0 || index >= this.trackedLength) { + return undefined; + } + let node = this.#root; + let remaining = index; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftCount = trackedCount(node.left); + if (remaining < leftCount) { + node = node.left; + continue; + } + remaining -= leftCount; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + const count = trackedElementCount(node, offset, this.#metrics); + if (remaining < count) return nodeElement(node, offset); + remaining -= count; + } + node = node.right; + } + return undefined; + } + + trackedIndexOf(element: T): number | undefined { + if (this.#hasLazyDeletions) materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + const offset = elementOffset(element); + if (node === undefined || offset === undefined) return undefined; + let index = trackedCount(node.left); + for (let before = 0; before < offset; before += 1) { + index += trackedElementCount(node, before, this.#metrics); + } + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.right) { + index += + trackedCount(current.parent.left) + + ownVisibleCount(current.parent) + + current.parent.ownUtf8; + } + current = current.parent; + } + return current === this.#root ? index : undefined; + } + + /** Visits the elements whose IDs are `peer:start..end`. */ + forEachWithIdIn( + peer: bigint, + start: number, + end: number, + visit: (element: T) => void, + ): void { + this.#locationsByPeer.get(peer)?.forEach(start, end, (location) => { + const element = + typeof location === "number" ? this.#elementAtLocation(location) : location; + if (this.#hasLazyDeletions) materializeElementDeletion(element, this.#metrics); + visit(element); + }); + } + + /** The exclusive end counter of the element and deletion IDs of each peer. */ + idEnds(): Map { + return new Map(this.#maxCounterByPeer); + } + get visibleUtf16Length(): number { return visibleMetric(this.#root, "utf16"); } @@ -2531,6 +2608,20 @@ function visibleCount( return node?.visibleCount ?? 0; } +function trackedCount( + node: SequenceNode | undefined, +): number { + return node === undefined ? 0 : node.visibleCount + node.allUtf8; +} + +function trackedElementCount( + node: SequenceNode, + offset: number, + metrics: (element: T) => SequenceMetrics, +): number { + return (nodeDeleted(node, offset) ? 0 : 1) + nodeMetrics(node, offset, metrics).utf8; +} + function visibleMetric( node: SequenceNode | undefined, metric: Metric, diff --git a/loro-js/tests/movable-list.test.ts b/loro-js/tests/movable-list.test.ts index 57855bd81..ced2d6008 100644 --- a/loro-js/tests/movable-list.test.ts +++ b/loro-js/tests/movable-list.test.ts @@ -132,6 +132,53 @@ describe("MovableList positions and elements", () => { expect(metadata(list)).toEqual(metadata(other)); }); + test("concurrent branches converge in any import order", () => { + let seed = 7; + const next = (limit: number): number => { + seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648; + return seed % limit; + }; + const base = doc(1); + for (let index = 0; index < 20; index += 1) base.getMovableList("list").push(index); + base.commit(); + const version = base.oplogVersion(); + const branches = [2, 3, 4].map((peer) => { + const branch = base.fork(); + branch.setPeerId(peer); + const list = branch.getMovableList("list"); + for (let step = 0; step < 60; step += 1) { + const kind = next(4); + if (kind === 0) list.insert(next(list.length + 1), `${peer}-${step}`); + else if (kind === 1 && list.length > 1) list.delete(next(list.length), 1); + else if (kind === 2) list.set(next(list.length), -step); + else list.move(next(list.length), next(list.length)); + if (step % 7 === 6) branch.commit(); + } + branch.commit(); + return branch.export({ mode: "update", from: version }); + }); + + const results = [ + [0, 1, 2], + [2, 1, 0], + [1, 0, 2], + ].map((order) => { + const target = base.fork(); + for (const index of order) target.import(branches[index]!); + return target; + }); + const batched = base.fork(); + batched.importBatch(branches); + const replayed = new LoroDoc(); + replayed.import(results[0]!.export({ mode: "update" })); + for (const target of [...results, batched, replayed]) { + expect(target.toJSON()).toEqual(results[0]!.toJSON()); + expect(metadata(target.getMovableList("list"))).toEqual( + metadata(results[0]!.getMovableList("list")), + ); + } + }); + test("records a set even when the value does not change", () => { const a = doc(1); const list = a.getMovableList("list"); From 7f56f79824997271f6cc6325f8fe684e0447e6b6 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 01:45:03 +0800 Subject: [PATCH 13/20] chore(loro.js): describe the merged MovableList fixes in the changeset Co-Authored-By: Claude Opus 5.5 --- .changeset/loro-js-movable-list-model.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.changeset/loro-js-movable-list-model.md b/.changeset/loro-js-movable-list-model.md index 963fd96bc..085520794 100644 --- a/.changeset/loro-js-movable-list-model.md +++ b/.changeset/loro-js-movable-list-model.md @@ -7,12 +7,14 @@ - A move of an element that another peer deleted concurrently keeps the element (`["a","c","B"]`, as in Rust), and a delete after a move removes the moved element. - Snapshots write and read Rust's MovableList layout, including list item IDs, element IDs, last-set IDs and dead list items. Before, loro.js wrote every position and last-set ID as the element ID. - `getLastEditorAt` and `getLastMoverAt` return the last setter and mover instead of the creator. -- Checkout, `diff`, `revertTo`, `applyDiff` and undo select winners the way Rust does. `revertTo`/`applyDiff` follow Rust's `apply_delta`, and undoing a `move` or `set` reinserts the old value. -- Import and checkout events report the whole state of a child container that a concurrent move brings back. +- Checkout, `diff`, `revertTo`, `applyDiff` and undo select winners the way Rust does. Undoing a `move` or `set` reinserts the old value, and undo transforms its inverse only over changes the `UndoManager` did not make, as Rust's does. - `set` records an op even when the value is unchanged, and `move(i, i)` is a no-op, as in Rust. `getIdAt` and `getCursor` of a MovableList anchor the list item, like Rust's cursor. +- Importing concurrent moves resolves their indices incrementally, like Rust's tracker, instead of one causal view per op. Also fixed while making loro.js match Rust: -- A document loaded from a snapshot replays history before importing changes that are concurrent with the snapshot. Before, List, Text and MovableList could place such changes at the wrong index. -- Shallow documents accept changes rooted at the empty version when the shallow root is empty, skip changes already inside the shallow root, and a shallow snapshot rooted at a peer's first op now writes the start version that Rust expects. +- A document loaded from a snapshot rebuilds each container that an import touches concurrently with the snapshot from that container's history first (as a checkout already does), so List, Text and MovableList place such changes where Rust does (loro-dev/loro#1163). A MovableList snapshot takes part in this completion like Text and List. +- Changes imported as pending while a snapshot's history was still undecoded were lost once the history was decoded (for example by a checkout or a history query); they now stay pending. +- Shallow documents follow Rust's rule for outdated imports: they accept changes rooted at the empty version when the shallow root is empty, skip changes already inside the shallow root, and reject a change whose part after the root depends on trimmed history. A shallow snapshot rooted at a peer's first op now writes the start version that Rust expects. +- An import that only adds pending changes creates the root containers they edit, as Rust does. - Importing a shallow snapshot emits events for containers that exist only in the shallow root state. From 972f856e04485fc53a107653a6710b7d2c6a4108 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 01:57:22 +0800 Subject: [PATCH 14/20] fix(loro.js): never walk a container's fields when comparing event values A MovableList replaced by a plain value made the comparison enumerate the container and hit its guarded element sequence. Co-Authored-By: Claude Opus 5.5 --- loro-js/src/runtime/containers.ts | 2 ++ loro-js/src/runtime/document.ts | 2 ++ loro-js/tests/movable-list.test.ts | 15 +++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index d8080965b..ebebe8be8 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -2540,6 +2540,8 @@ export class LoroMovableList extends LoroList { // A MovableList keeps positions and elements in `_state`; the inherited // element sequence stays empty. Fail loudly if a list-only path reads it. Object.defineProperty(this, "_sequence", { + // Hidden from generic walks (`Object.entries`), which must not trip it. + enumerable: false, get(): never { throw new Error("LoroMovableList has no element sequence; use _state"); }, diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index ad569051a..bfc58e4ef 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -10586,6 +10586,8 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { return bytesEqual(left, right); } if (isContainer(left) && isContainer(right)) return left.id === right.id; + // A container never equals a plain value; its fields are not its value. + if (isContainer(left) || isContainer(right)) return false; if (Array.isArray(left) && Array.isArray(right)) { return ( left.length === right.length && diff --git a/loro-js/tests/movable-list.test.ts b/loro-js/tests/movable-list.test.ts index ced2d6008..4f4b45028 100644 --- a/loro-js/tests/movable-list.test.ts +++ b/loro-js/tests/movable-list.test.ts @@ -179,6 +179,21 @@ describe("MovableList positions and elements", () => { } }); + test("replacing a MovableList value with a plain object emits its event", () => { + const d = doc(1); + const events: LoroEventBatch[] = []; + d.subscribe((event) => events.push(event)); + const map = d.getMap("m"); + map.setContainer("k", new LoroMovableList()).push(1); + d.commit(); + map.set("k", { n: 2 }); + d.commit(); + expect(d.toJSON()).toEqual({ m: { k: { n: 2 } } }); + expect(events.at(-1)!.events.map(({ diff }) => diff)).toEqual([ + { type: "map", updated: { k: { n: 2 } } }, + ]); + }); + test("records a set even when the value does not change", () => { const a = doc(1); const list = a.getMovableList("list"); From 699b5a5396e215753bf3052284bbd6c388e932ff Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 02:01:06 +0800 Subject: [PATCH 15/20] test(loro.js): fuzz MovableList in the multi-peer differential CI run The Rust-model MovableList passes seeds 0-599 of the #1141 fuzz with all oracles in lockstep with Rust; main failed 144 of seeds 0-299. Co-Authored-By: Claude Opus 5.5 --- context/loro-js-differential-fuzz.md | 16 +++++++++++----- loro-js/tests/differential-fuzz.test.ts | 7 +++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/context/loro-js-differential-fuzz.md b/context/loro-js-differential-fuzz.md index f29a9848c..042def6b7 100644 --- a/context/loro-js-differential-fuzz.md +++ b/context/loro-js-differential-fuzz.md @@ -1,6 +1,6 @@ # loro.js Differential Fuzzing -Verified against code 2026-09-29. +Verified against code 2026-09-30. `loro-js/tests/fuzz/differential.ts` drives several `loro.js` peers through random edits, update exchanges, and checkouts, and checks them against @@ -76,8 +76,9 @@ prints each action as it runs, which finds hangs. The report groups failures by signature (oracle plus first differing root or normalized error) and stores the minimized action list and a trace written as `loro.js` API calls. -CI runs 120 fixed seeds of Map/List/Text/Counter, and 120 more with nested -child containers and snapshot imports, with the replay, checkout, +CI runs 120 fixed seeds of Map/List/Text/Counter, 120 with MovableList and +nested child containers, and 120 more with nested child containers and snapshot +imports, with the replay, checkout, full-snapshot, and event oracles. A seed that fails on `main` for a known reason goes into `pending` with that reason until its fix lands; both lists are empty on 2026-09-29 (`main` at #1131). @@ -98,6 +99,13 @@ re-inserts (#1150). With nested children and snapshot imports, seeds 297, 315, 344, 422, 452, and 559 of 0–599 fail on `main` in the event oracle; #1150 fixes all six. +MovableList, fixed by the Rust-model rewrite (loro-dev/loro#1132 and its +follow-ups, see [loro-js-movable-list.md](loro-js-movable-list.md)): with +`FUZZ_FEATURES=map,list,text,counter,movableList,nested`, all oracles and +lockstep with Rust, 144 of seeds 0–299 failed on `main` on 2026-09-30 (98 +checkout-vs-replay, 29 lockstep); on the rewrite seeds 0–599 pass, and CI runs +120 of them without Rust. + Still open, so `CI_RUNS` does not enable them yet: - Tree convergence: `loro.js` applies tree moves per node, last writer wins, @@ -121,8 +129,6 @@ Still open, so `CI_RUNS` does not enable them yet: (`DocState::get_path`); `loro.js` still sends them. Needs #1147's liveness. - Shallow snapshots in the harness need #1144 (the root check fails first on every seed without it). -- MovableList: `loro.js` skips a movable-list `set` to the current value, - which Rust records. ## Rust-side findings diff --git a/loro-js/tests/differential-fuzz.test.ts b/loro-js/tests/differential-fuzz.test.ts index 04ea92fa1..c89c7ffa0 100644 --- a/loro-js/tests/differential-fuzz.test.ts +++ b/loro-js/tests/differential-fuzz.test.ts @@ -55,6 +55,13 @@ const CI_RUNS: readonly { seeds: Array.from({ length: 120 }, (_, index) => index), pending: new Map(), }, + { + name: "MovableList and nested child containers: replay, checkout, snapshot, and event oracles", + features: { ...MAP_LIST_TEXT_COUNTER, movableList: true, nested: true }, + checks: { events: true, checkout: true, snapshot: true, shallow: false }, + seeds: Array.from({ length: 120 }, (_, index) => index), + pending: new Map(), + }, { name: "nested child containers and snapshot imports: replay, checkout, snapshot, and event oracles", features: { ...MAP_LIST_TEXT_COUNTER, nested: true, snapshotSync: true }, From fe2e6dd628d41167de10a1d72c24e80c52cbbe98 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 02:05:55 +0800 Subject: [PATCH 16/20] fix(loro.js): apply MovableList diffs deletes first, like Rust's apply_delta Without moved children the delta was walked in order, which gave the ops other IDs than Rust's and later deletes by ID missed. Co-Authored-By: Claude Opus 5.5 --- loro-js/src/runtime/document.ts | 12 +++++++----- loro-js/tests/movable-list.test.ts | 31 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index bfc58e4ef..323093a77 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -1338,9 +1338,10 @@ export class LoroDoc = Record = Record { let low = 0; @@ -1405,7 +1408,6 @@ export class LoroDoc = Record { ]); }); + test("applyDiff writes Rust's ops: deletes first, then inserts", () => { + const d = doc(1); + const list = d.getMovableList("l"); + for (const value of ["a", "b", "c"]) list.push(value); + d.commit(); + const version = d.oplogVersion(); + d.applyDiff([ + [ + list.id, + { + type: "list", + diff: [ + { insert: ["x"] }, + { retain: 1 }, + { delete: 1 }, + { retain: 1 }, + { insert: ["y"] }, + ], + }, + ], + ]); + d.commit(); + expect(d.toJSON()).toEqual({ l: ["x", "a", "c", "y"] }); + // Rust's `apply_delta` since loro-dev/loro#1138; op IDs must match its. + expect(d.exportJsonUpdates(version).changes[0]!.ops.map((op) => op.content)).toEqual([ + { type: "delete", pos: 1, len: 1, start_id: "1@0" }, + { type: "insert", pos: 0, value: ["x"] }, + { type: "insert", pos: 3, value: ["y"] }, + ]); + }); + test("revertTo deletes before it inserts, like Rust's apply_delta", () => { const p3 = doc(3); p3.getMovableList("list").insert(0, "v2"); From 22d811fe331172671e3ef3c43aefe266f1ca4bf0 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 02:07:51 +0800 Subject: [PATCH 17/20] test(loro.js): skip map entries that only delete keys when comparing diff() A child created in the range whose keys were all set and deleted again is listed by Rust and omitted by loro.js, also on main and for any parent; see context/loro-js-movable-list.md. Co-Authored-By: Claude Opus 5.5 --- context/loro-js-movable-list.md | 4 ++++ loro-js/tests/differential/fuzz.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index a2119d294..26ff714f3 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -225,6 +225,10 @@ them (see the comments in `harness.ts` and `fuzz.ts`): snapshot already contains (loro-dev/loro#1163). - Some shallow-snapshot imports that Rust accepts are rejected by loro.js with "cannot import updates that depend on an outdated version". +- `diff()` of a child container created in the range omits a map whose keys + were all set and deleted again in that range; Rust lists those keys as + deleted. It happens under List, Map and MovableList parents alike. The suite + skips such map entries when it compares `diff()`. - Multi-blob `importBatch` events are labeled `by: "checkout"` by Rust and `"import"` by loro.js; checkout events carry origin `"checkout"` only in Rust. Rust's `oplogVersion()` also counts the pending transaction. diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index 828ba9268..3015cd013 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -508,6 +508,10 @@ export class FuzzRun { .diff(from, to, false) .filter(([id]) => reachable === undefined || reachable.has(id)) .map(([id, diff]) => [id, canonicalDiff(diff)] as const) + // A child created in the range whose map keys were all set and + // deleted again: Rust lists them as deleted, loro.js (List, Map and + // MovableList parents alike, on `main` too) omits the child. + .filter(([, diff]) => !onlyDeletedMapKeys(diff)) .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)), ); if (results !== undefined && !isDeepStrictEqual(results.rust, results.js)) { @@ -847,6 +851,14 @@ function orderedDiff(diff: unknown): unknown { return { type: "list", diff: output }; } +function onlyDeletedMapKeys(diff: unknown): boolean { + const typed = diff as { type: string; updated?: Record }; + return ( + typed.type === "map" && + Object.values(typed.updated ?? {}).every((value) => value === undefined) + ); +} + export interface SavedTrace { readonly profile: FuzzProfile; readonly seed: number; From 224b0a43cebca7524c8d04a9c6378ec382c04cc8 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 02:47:40 +0800 Subject: [PATCH 18/20] docs(loro.js): record snapshot completion on import and its cost The performance context still described main's MovableList snapshot special case and #1163 as open. Measurements are in context/loro-js-performance.md. Co-Authored-By: Claude Opus 5.5 --- context/loro-js-performance.md | 62 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 9fe35cf98..adedb741a 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -1,6 +1,6 @@ # loro-js Performance Architecture -Verified against code 2026-09-28. +Verified against code 2026-09-30. The pure TypeScript runtime lives in `loro-js/src/runtime`. Its performance target is the asymptotic behavior of the Rust runtime, while accepting a larger @@ -118,8 +118,10 @@ JavaScript constant factor. index only when a concurrent/future interval needs ordering. Consecutive IDs keep their single-child edge implicit, and `SequenceIndex` can skip an entire future ID run while finding the next causally included element. Ordinary local - edits keep the smaller unindexed path. MovableList continues to use the scan - because moves break the origin-tree physical preorder. Sibling subtrees are + edits keep the smaller unindexed path. MovableList positions keep the scan + (`MovableListState`, `useOriginIndex = false`): the origin index misorders + a sibling subtree followed by a concurrent position that is not its + descendant. Sibling subtrees are contiguous, so the gap between two direct children belongs to the earlier child. After the last child the interval can also hold concurrent elements whose origin is left of `originLeft`; Rust's scan stops before them. The index @@ -171,7 +173,11 @@ JavaScript constant factor. - Text, List, and MovableList state hydrated from a snapshot (eager, lazy, or a shallow root) has no tombstones, deletion index, or style, value, and move history for the snapshot's operations. `LoroDoc.#snapshotSequences` records - each such container with its snapshot version. Before `checkout`, + each such container with its snapshot version. Before an import, + `#prepareSnapshotImport` completes (below) each such container that an + imported record touches concurrently with the snapshot version, since that + record's causal view can need tombstones the snapshot dropped + (loro-dev/loro#1163). Before `checkout`, `checkoutToLatest`, `diff`, or a detached snapshot export (`#encodeLatestState`) transitions, `#prepareSnapshotTransition` looks only at the containers the transition @@ -208,9 +214,8 @@ JavaScript constant factor. whole-container values when style operations are crossed, since their ranges come from positions. - What that guarantees: the latest state, imports, and exports equal the - snapshot state plus the later operations, as loro.js applies them. A later - operation concurrent with a delete that the snapshot already applied can - still land at another position than in Rust, as on main (loro-dev/loro#1163). + snapshot state plus the later operations, as loro.js applies them; a later + operation concurrent with the snapshot runs on the completed container. An older version is approximate: the snapshot state cannot restore text deleted before it. In the round-3 review's random Rust histories, older versions and `revertTo` differed from Rust more often than on main (437 vs @@ -222,33 +227,22 @@ JavaScript constant factor. (plain text too; loro-dev/loro#1136 fixes the plain-text case), and a shallow export that throws midway leaves the live document at the root or in between, since `#encodeShallowSnapshot` rebuilds it without a restore. -- A MovableList is not a snapshot sequence at all (`#markSnapshotSequence`); - it behaves as on main. Its snapshot state names each element by its Rust - position id (`#hydrateContainerState` takes `listItemIds` in order, and after - a move they include invisible positions), while a replay names elements by - their insert ids, and Rust encodes element ids as (peer, lamport), so no - comparison with a replay is meaningful. The hydrated state also has no move - or value history (`_moveHistoryComplete`, `_valueHistoryComplete`), so - `#canTransitionRecords` refuses to cross its snapshot moves, sets, and - deletes, and the transition replays to the target; later transitions are - incremental, and a later move or set by element id resolves. Replaying the - list to the current version and then retreating instead (as round 3 did) - carried loro.js's MovableList non-convergence at the latest version into - older versions. Completing only lists whose snapshot names elements by insert - ids was also tried: it makes a snapshot document transition like a - full-history one, and loro.js's incremental MovableList transitions have - their own bugs with concurrent moves and sets (a full-history document on - main shows them too), so random Rust histories got 167 mismatching versions - against main's 91; leaving every MovableList on main's path gives 56. Until - the MovableList model work (loro-dev/loro#1132 and follow-ups) hydrates - element ids, a MovableList whose loro.js replay differs from Rust can change - its latest state at the first such transition, also as on main. Once the - list is replayed, later transitions are incremental and can still show a - wrong older version where main, which replays on every such checkout, is - right (round-4 review, seed 22: after checking out v11, v12, then v13, the - list shows `[11]` where Rust, main, and a full-history document show `[9]`; - the latest state and `revertTo` are right). This is loro.js's MovableList - transition gap, left to loro-dev/loro#1132. +- A MovableList is a snapshot sequence like Text and List. Its snapshot + state names Rust's position, element and last-set ids, so a replay of its + history is comparable to it (`sameMovableListStates`); its hydrated + candidates are partial, so every transition that touches it completes it. + Model details: [loro-js-movable-list.md](loro-js-movable-list.md). +- Cost of completion on import: the first import concurrent with the snapshot + pays one replay of that container's history, the work a document loaded + from updates did at load time. Measured 2026-09-30: with 2000 random Text + edits per peer, the concurrent import after a snapshot takes 675 ms, as + after a full-history update import on main (667 ms); main's snapshot path + took 235 ms because it skipped the tombstones (loro-dev/loro#1163). With a + MovableList of 8000 items and 8000 moves/sets per peer it takes 92 ms, 60 ms + after an update import (main: 35 ms and 96 ms; main's snapshot path skipped + the history and diverged from Rust). Text concurrent imports stay + superlinear, as on main, since each op computes its causal view + (`causalView`). - A full `#rebuildFromHistory` (the non-incremental fallback, shallow export, `forkAt`) rebuilds unreplayable containers the same way. It no longer checks snapshot-hydrated styled Text first (`#checkSnapshotSequences`, removed in From 06a5fba8247a25568e3919919c757e0654755388 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 03:22:33 +0800 Subject: [PATCH 19/20] perf(loro.js): check MovableList transition inserts in one tree walk canTransition walked the position tree once per insert op, which made checkout, diff and revertTo O(insert ops * positions). Collect the runs and check them in one containsIdRuns call, as the List branch does. Co-Authored-By: Claude Opus 5.5 --- loro-js/src/runtime/movable-list.ts | 15 +++++++++----- loro-js/tests/movable-list.test.ts | 31 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/loro-js/src/runtime/movable-list.ts b/loro-js/src/runtime/movable-list.ts index ecaa9111d..5a30f2eaf 100644 --- a/loro-js/src/runtime/movable-list.ts +++ b/loro-js/src/runtime/movable-list.ts @@ -15,7 +15,11 @@ import { type RuntimeValue, type SequenceElement, } from "./containers"; -import { SequenceIndex, type SequenceMetrics } from "./sequence-index"; +import { + SequenceIndex, + type SequenceIdRun, + type SequenceMetrics, +} from "./sequence-index"; /** * MovableList state with Rust's model: a Fugue sequence of positions plus @@ -624,12 +628,13 @@ export class MovableListState { /** Whether version transitions over `ops` can run without replaying history. */ canTransition(ops: readonly MovableTransitionOp[]): boolean { if (!this.historyComplete) return false; + // Checked in one tree walk below; one walk per insert made a checkout + // O(operations * positions). + const inserted: SequenceIdRun[] = []; for (const op of ops) { switch (op.type) { case "insert": - if (!this.positions.containsIdRuns([{ start: op.id, length: op.length }])) { - return false; - } + inserted.push({ start: op.id, length: op.length }); break; case "delete": { let recorded = 0; @@ -669,7 +674,7 @@ export class MovableListState { } } } - return true; + return this.positions.containsIdRuns(inserted); } /** Encodes the state in Rust's `MovableListState` snapshot layout. */ diff --git a/loro-js/tests/movable-list.test.ts b/loro-js/tests/movable-list.test.ts index 21cf37b75..8fdff61c5 100644 --- a/loro-js/tests/movable-list.test.ts +++ b/loro-js/tests/movable-list.test.ts @@ -438,6 +438,37 @@ describe("MovableList versions, undo and diffs", () => { .find((event) => event.target === text.id); expect(textEvent?.diff).toEqual({ type: "text", diff: [{ insert: "hi" }] }); }); + + test("checkout and diff stay linear in the number of inserts", () => { + /** Best of three checkout round trips plus a diff over n scattered inserts. */ + function transitionsMs(size: number): number { + const a = doc(1); + const list = a.getMovableList("list"); + list.push(-1); + a.commit(); + const start = a.frontiers(); + for (let index = 0; index < size; index += 1) { + list.insert((index * 7_919) % (list.length + 1), index); + } + a.commit(); + const latest = a.frontiers(); + let best = Infinity; + for (let run = 0; run < 3; run += 1) { + const started = performance.now(); + a.checkout(start); + expect(a.getMovableList("list").length).toBe(1); + a.checkoutToLatest(); + expect(a.diff(start, latest).length).toBe(1); + best = Math.min(best, performance.now() - started); + } + expect(a.frontiers()).toEqual(latest); + return best; + } + // Each insert op walked the whole position tree to check that the + // transition could skip the replay (8x the inserts took about 90x the time). + const ratio = transitionsMs(8_000) / transitionsMs(1_000); + expect(ratio).toBeLessThan(24); + }, 60_000); }); describe("MovableList cursors", () => { From 4e99ba64882146a5acf49a0508280c6fccf945e4 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Wed, 30 Sep 2026 03:23:54 +0800 Subject: [PATCH 20/20] test(loro.js): fix the suite's changeset and ignore empty diff() deltas The changeset named a test:differential script that does not exist. Rust's diff() can list a sequence with an empty delta where loro.js omits it (context/loro-js-movable-list.md). Co-Authored-By: Claude Opus 5.5 --- .changeset/loro-js-differential-suite.md | 2 +- context/loro-js-movable-list.md | 3 +++ loro-js/tests/differential/fuzz.ts | 11 +++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.changeset/loro-js-differential-suite.md b/.changeset/loro-js-differential-suite.md index ac05280fe..4aa735d9e 100644 --- a/.changeset/loro-js-differential-suite.md +++ b/.changeset/loro-js-differential-suite.md @@ -1,4 +1,4 @@ --- --- -Add a differential test suite that compares loro.js with the Rust implementation (`pnpm --dir loro-js test:differential`). Tests and scripts only; the published package does not change. +Add a MovableList differential test suite (`loro-js/tests/differential/`) that compares loro.js with the Rust implementation. It runs in `pnpm test` when the Rust WASM build is present. Tests only; the published package does not change. diff --git a/context/loro-js-movable-list.md b/context/loro-js-movable-list.md index 26ff714f3..3ea78a33b 100644 --- a/context/loro-js-movable-list.md +++ b/context/loro-js-movable-list.md @@ -242,6 +242,9 @@ Differences in form, not in effect, which the suite checks by value: - Event deltas can differ in shape, or be a no-op where Rust reports nothing; the suite then checks that each engine's deltas turn the previous value into the new one. +- Rust's `diff()` can list a sequence whose delta is empty (two peers + concurrently moving one element), where loro.js omits the container. The + suite ignores such entries. Rust issues the suite works around: diff --git a/loro-js/tests/differential/fuzz.ts b/loro-js/tests/differential/fuzz.ts index 3015cd013..bff356ae3 100644 --- a/loro-js/tests/differential/fuzz.ts +++ b/loro-js/tests/differential/fuzz.ts @@ -510,8 +510,10 @@ export class FuzzRun { .map(([id, diff]) => [id, canonicalDiff(diff)] as const) // A child created in the range whose map keys were all set and // deleted again: Rust lists them as deleted, loro.js (List, Map and - // MovableList parents alike, on `main` too) omits the child. - .filter(([, diff]) => !onlyDeletedMapKeys(diff)) + // MovableList parents alike, on `main` too) omits the child. Rust + // can also list a sequence whose delta is empty (two concurrent + // moves of one element); loro.js omits it. + .filter(([, diff]) => !onlyDeletedMapKeys(diff) && !emptyDelta(diff)) .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)), ); if (results !== undefined && !isDeepStrictEqual(results.rust, results.js)) { @@ -851,6 +853,11 @@ function orderedDiff(diff: unknown): unknown { return { type: "list", diff: output }; } +function emptyDelta(diff: unknown): boolean { + const typed = diff as { type: string; diff?: unknown }; + return Array.isArray(typed.diff) && typed.diff.length === 0; +} + function onlyDeletedMapKeys(diff: unknown): boolean { const typed = diff as { type: string; updated?: Record }; return (