diff --git a/.changeset/loro-js-shallow-root-winner.md b/.changeset/loro-js-shallow-root-winner.md new file mode 100644 index 000000000..d7986cf6d --- /dev/null +++ b/.changeset/loro-js-shallow-root-winner.md @@ -0,0 +1,5 @@ +--- +"loro.js": patch +--- + +Fix `checkout`, `diff`, and undo on a `loro.js` shallow document when a map value or tree node placement was written in the shallow root commit and changed afterwards. When the root commit held more than one op, retreating to the root dropped the map key or tree node instead of restoring its root-time value, and `getLastEditor` returned `undefined`. A tree node deleted by a retained delete whose placement was trimmed from the shallow history now stays deleted. diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 1ea6a0e95..d6e5d4e7e 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -237,6 +237,20 @@ JavaScript constant factor. needs no replay: 57–61 ms versus 152 ms, with the same 234 MiB peak RSS as before the fix (earlier 284 MiB). The coalesced inserts also make importing the B4 trace as one update about 30% faster (about 195 versus 275 ms). +- A shallow history trims the ops that wrote root-time Map values and Tree + placements (the root commit's other ops). When a Map or Tree retreat finds no + retained winner at or below the target, it uses the shallow root state entry + (`#shallowRootMapRecord`, `#shallowRootTreeNode`) instead of dropping the key + or node; a retained Tree delete whose placement was trimmed takes the root + placement and stays deleted. The root store entry for a container comes from + `#shallowRootEntryIndex`: the key index the import builds to merge the root + and latest states, or while hydrating the root store for a replay (Rust omits + the latest state for a short retained tail). Each container's key/node index + is built on first lookup. A retreat therefore touches only the maps and trees + it changes, as in Rust's per-map checkout index seeding (loro-dev/loro#1120, + #1124): the first such retreat in a shallow doc with 32,768 child Maps takes + under 1 ms for both loro.js- and Rust-written snapshots, flat from 1,024 + Maps, and later ones about 0.02–0.03 ms. When an element's deleted flag, tree parent/position, or map visibility changes, mutate it through its owning index helper. Direct mutation leaves subtree or diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index ee3eed668..3c41efd94 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -69,6 +69,7 @@ import { type Container, type CausalVersion, type LastWriter, + type MapRecord, type RuntimeValue, type SequenceElement, type SequenceMoveMeta, @@ -339,6 +340,9 @@ export class LoroDoc = Record | undefined; + // Root-time map entries and tree nodes of a shallow doc, indexed per + // container on first lookup. + #shallowRootIndexes = new WeakMap(); #textStyles = new Map([ ["bold", "after"], ["italic", "after"], @@ -1164,24 +1168,11 @@ export class LoroDoc = Record = Record = Record | undefined { + const index = this.#shallowRootIndex(); + const id = tree._codecId; + if (index === undefined || id === undefined) return undefined; + const containerKey = this.#containerKey(id); + let nodes = index.trees.get(containerKey); + if (nodes === undefined) { + nodes = new Map(); + const state = this.#shallowRootContainerState(containerKey); + if (state?.kind === CodecContainerType.Tree) { + const ids = state.nodes.map((node) => ({ + peer: state.peers[Number(node.peerIndex)]!, + counter: node.counter, + })); + for (const [position, node] of state.nodes.entries()) { + const lastMoveId = { + peer: state.peers[Number(node.lastSetPeerIndex)]!, + counter: node.lastSetCounter, + }; + nodes.set(formatTreeId(ids[position]!), { + id: ids[position]!, + parent: + node.parentIndexPlusTwo >= 2n + ? ids[Number(node.parentIndexPlusTwo - 2n)] + : undefined, + position: state.positions[node.fractionalIndexIndex]!, + deleted: node.parentIndexPlusTwo === 1n, + writer: { + peer: lastMoveId.peer, + lamport: node.lastSetCounter + node.lastSetLamportSub, + }, + lastMoveId, + }); + } + } + index.trees.set(containerKey, nodes); + } + return nodes.get(nodeKey); + } + + #shallowRootMapRecord(map: LoroMap, key: string): MapRecord | undefined { + const index = this.#shallowRootIndex(); + const id = map._codecId; + if (index === undefined || id === undefined) return undefined; + const containerKey = this.#containerKey(id); + let entries = index.maps.get(containerKey); + if (entries === undefined) { + const state = this.#shallowRootContainerState(containerKey); + entries = new Map(); + if (state?.kind === CodecContainerType.Map) { + const values = new Map(state.values); + for (const item of state.metadata) { + entries.set(item.key, { + value: values.get(item.key), + writer: { + peer: state.peers[Number(item.peerIndex)]!, + lamport: Number(item.lamport), + }, + }); + } + } + index.maps.set(containerKey, entries); + } + const entry = entries.get(key); + if (entry === undefined) return undefined; + if (entry.value === undefined) { + return { + value: undefined, + rawValue: undefined, + deleted: true, + writer: entry.writer, + }; + } + const rawValue = this.#decodeSnapshotValue(entry.value, map); + return { + value: this.#materializeMapValue(map, key, rawValue), + rawValue, + deleted: false, + writer: entry.writer, + }; + } + #previousMapOperation( containerId: CodecContainerId, key: string, @@ -3928,13 +4019,8 @@ export class LoroDoc = Record(); if (movableMoveMode === "replay") { for (const { change } of records) { @@ -4081,35 +4167,12 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record() + : undefined; + for (const entry of store.containers) { + rootEntries?.set(this.#getOrCreateContainer(entry.id, undefined, false).id, entry); } + if (rootEntries !== undefined) this.#shallowRootEntries = rootEntries; for (const { id, wrapper } of store.containers) { const container = this.#getOrCreateContainer(id, undefined, false); const parent = @@ -7806,6 +7933,17 @@ function lowerBoundWriter( return low; } +interface ShallowRootIndex { + readonly maps: Map< + string, + Map< + string, + { readonly value: EncodedLoroValue | undefined; readonly writer: LastWriter } + > + >; + readonly trees: Map>>; +} + function latestIncludedOperation( history: IndexedSubjectHistory | undefined, version: VersionVector, diff --git a/loro-js/tests/fixtures/rust/shallow-empty-state.blob b/loro-js/tests/fixtures/rust/shallow-empty-state.blob new file mode 100644 index 000000000..e841b8fd1 Binary files /dev/null and b/loro-js/tests/fixtures/rust/shallow-empty-state.blob differ diff --git a/loro-js/tests/shallow-root-map-winner.test.ts b/loro-js/tests/shallow-root-map-winner.test.ts new file mode 100644 index 000000000..9340f6425 --- /dev/null +++ b/loro-js/tests/shallow-root-map-winner.test.ts @@ -0,0 +1,305 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, test } from "vitest"; + +import { LoroDoc, LoroMap, UndoManager, type Frontiers } from "../src/index"; + +/** + * Port of crates/loro/tests/shallow_checkout_equal_value.rs (loro-dev/loro#1124). + * + * The root commit holds several ops, so the op that wrote a key's root-time + * value is trimmed from the shallow history. Retreating to the root must then + * fall back to the shallow root state instead of dropping the key. MovableList + * is left out: its position/last-set metadata is a separate known gap. + */ +const PADS = [0, 300]; + +function buildFixture(pad: number, revert: boolean) { + const doc = new LoroDoc(); + doc.setPeerId(1); + const map = doc.getMap("map"); + map.set("x", "root"); + map.set("untouched", "root"); + const child = map.setContainer("child", new LoroMap()); + child.set("x", 1); + const sub = map.setContainer("sub", new LoroMap()); + sub.set("k", "v"); + const mergeable = map.ensureMergeableMap("merge"); + mergeable.set("x", "root"); + const list = doc.getList("list"); + list.push("a"); + const text = doc.getText("text"); + text.insert(0, "ab"); + const tree = doc.getTree("tree"); + const n1 = tree.createNode().id; + const n2 = tree.createNode().id; + tree.getNodeByID(n1)!.data.set("x", "root"); + const counter = doc.getCounter("counter"); + counter.increment(1); + doc.commit(); + const root = doc.frontiers(); + const versions: Frontiers[] = [root]; + const commit = (): void => { + doc.commit(); + versions.push(doc.frontiers()); + }; + + map.set("x", "later"); + map.set("sub", "not a map"); + child.set("x", 2); + mergeable.set("x", "later"); + list.delete(0, 1); + list.push("a"); + text.delete(0, 1); + text.insert(0, "a"); + tree.move(n1, n2); + tree.getNodeByID(n1)!.data.set("x", "later"); + counter.increment(2); + commit(); + + const padMap = doc.getMap("pad"); + for (let index = 0; index < pad; index += 1) padMap.set("k", index); + commit(); + + if (revert) { + doc.revertTo(root); + } else { + map.set("x", "root"); + child.set("x", 1); + mergeable.set("x", "root"); + tree.move(n1); + tree.getNodeByID(n1)!.data.set("x", "root"); + counter.decrement(2); + } + commit(); + map.set("tail", 1); + commit(); + return { doc, root, versions }; +} + +const importBytes = (bytes: Uint8Array): LoroDoc => { + const doc = new LoroDoc(); + doc.import(bytes); + return doc; +}; + +const valueAt = (doc: LoroDoc, version: Frontiers): unknown => { + doc.checkout(version); + const value = doc.toJSON(); + doc.checkoutToLatest(); + return value; +}; + +function expectSameHistory( + full: LoroDoc, + shallow: LoroDoc, + versions: readonly Frontiers[], + context: string, +): void { + // The context is part of the compared value so a failure names its step. + expect({ context, value: shallow.toJSON() }).toEqual({ context, value: full.toJSON() }); + for (const version of versions) { + expect({ context, version, value: valueAt(shallow, version) }).toEqual({ + context, + version, + value: valueAt(full, version), + }); + } + // Walk back and forth so later checkouts use incremental transitions. + for (const version of [...versions].reverse()) { + shallow.checkout(version); + full.checkout(version); + expect({ context, version, value: shallow.toJSON() }).toEqual({ + context, + version, + value: full.toJSON(), + }); + } + shallow.checkoutToLatest(); + full.checkoutToLatest(); + for (const from of versions) { + for (const to of versions) { + const applied = (doc: LoroDoc): unknown => { + const diff = doc.diff(from, to); + const fork = full.forkAt(from); + fork.setDetachedEditing(true); + fork.applyDiff(diff); + return fork.toJSON(); + }; + expect({ context, from, to, value: applied(shallow) }).toEqual({ + context, + from, + to, + value: applied(full), + }); + } + } +} + +describe("shallow checkout of a value rewritten after the root", () => { + test("keeps a root-time map value whose writer was trimmed", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const map = doc.getMap("map"); + map.set("a", 1); + map.set("z", 0); + doc.commit(); + const root = doc.frontiers(); + map.set("a", 2); + doc.commit(); + const latest = doc.frontiers(); + + const shallow = importBytes( + doc.export({ mode: "shallow-snapshot", frontiers: root }), + ); + for (let round = 0; round < 2; round += 1) { + shallow.checkout(root); + expect(shallow.toJSON()).toEqual({ map: { a: 1, z: 0 } }); + expect(shallow.getMap("map").getLastEditor("a")).toBe("1"); + shallow.checkout(latest); + expect(shallow.toJSON()).toEqual({ map: { a: 2, z: 0 } }); + } + }); + + test("reads root-time values from a Rust snapshot without a latest state", () => { + // Rust shallow export, peer 1 at the root: map {a:1, z:0, child:{x:"root"}}; + // peer 2 then writes a=2 and child.x="later". The short retained tail makes + // Rust omit the latest state, so the import replays from the root state. + const shallow = importBytes( + new Uint8Array( + readFileSync( + new URL("./fixtures/rust/shallow-empty-state.blob", import.meta.url), + ), + ), + ); + const root: Frontiers = [{ peer: "1", counter: 3 }]; + const latest = shallow.frontiers(); + expect(shallow.toJSON()).toEqual({ map: { a: 2, z: 0, child: { x: "later" } } }); + for (let round = 0; round < 2; round += 1) { + shallow.checkout(root); + expect(shallow.toJSON()).toEqual({ map: { a: 1, z: 0, child: { x: "root" } } }); + expect(shallow.getMap("map").getLastEditor("a")).toBe("1"); + shallow.checkout(latest); + expect(shallow.getMap("map").getLastEditor("a")).toBe("2"); + } + }); + + test("keeps a node deleted by a retained delete whose placement was trimmed", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const first = tree.createNode().id; + const second = tree.createNode().id; + doc.commit(); + const root = doc.frontiers(); + tree.delete(first); + doc.commit(); + const deleted = doc.frontiers(); + + const shallow = importBytes( + doc.export({ mode: "shallow-snapshot", frontiers: root }), + ); + for (let round = 0; round < 2; round += 1) { + shallow.checkout(root); + expect( + shallow + .getTree("tree") + .roots() + .map((node) => node.id), + ).toEqual([first, second]); + shallow.checkout(deleted); + expect( + shallow + .getTree("tree") + .roots() + .map((node) => node.id), + ).toEqual([second]); + expect(shallow.getTree("tree").isNodeDeleted(first)).toBe(true); + } + }); + + for (const pad of PADS) { + for (const revert of [false, true]) { + test(`matches full history (pad=${pad}, revert=${revert})`, () => { + const { doc, root, versions } = buildFixture(pad, revert); + const full = importBytes(doc.export({ mode: "snapshot" })); + const shallow = importBytes( + doc.export({ mode: "shallow-snapshot", frontiers: root }), + ); + expect((valueAt(shallow, root) as { map: { x: string } }).map.x).toBe("root"); + expectSameHistory(full, shallow, versions, `pad=${pad} revert=${revert}`); + }); + } + + test(`re-exports keep the root-time values (pad=${pad})`, () => { + const { doc, root, versions } = buildFixture(pad, true); + const full = importBytes(doc.export({ mode: "snapshot" })); + let shallow = importBytes( + doc.export({ mode: "shallow-snapshot", frontiers: root }), + ); + for (let hop = 0; hop < 2; hop += 1) { + shallow.checkout(root); + shallow.checkoutToLatest(); + shallow = importBytes( + shallow.export({ mode: "shallow-snapshot", frontiers: root }), + ); + expectSameHistory(full, shallow, versions, `pad=${pad} hop=${hop}`); + } + const later = importBytes( + shallow.export({ mode: "shallow-snapshot", frontiers: versions[1]! }), + ); + expectSameHistory(full, later, versions.slice(1), `pad=${pad} later root`); + }); + + test(`undo matches full history (pad=${pad})`, () => { + const { doc, root } = buildFixture(pad, true); + const replicas = [ + importBytes(doc.export({ mode: "snapshot" })), + importBytes(doc.export({ mode: "shallow-snapshot", frontiers: root })), + ]; + const values = replicas.map((replica) => { + replica.setPeerId(2); + const undo = new UndoManager(replica, {}); + replica.getMap("map").set("x", "local"); + replica.commit(); + undo.undo(); + const afterUndo = replica.toJSON(); + undo.redo(); + return [afterUndo, replica.toJSON()]; + }); + expect(values[1]).toEqual(values[0]); + expect((values[0]![0] as { map: { x: string } }).map.x).toBe("root"); + }); + + test(`reports the root-time writer of an equal value (pad=${pad})`, () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const map = doc.getMap("map"); + map.set("x", "root"); + map.set("y", "root"); + doc.commit(); + const root = doc.frontiers(); + map.set("x", "later"); + for (let index = 0; index < pad; index += 1) doc.getMap("pad").set("k", index); + doc.commit(); + doc.setPeerId(2); + map.set("x", "root"); + doc.commit(); + + const full = importBytes(doc.export({ mode: "snapshot" })); + const shallow = importBytes( + doc.export({ mode: "shallow-snapshot", frontiers: root }), + ); + const expected = valueAt(full, root); + for (const replica of [full, shallow]) { + for (let round = 0; round < 2; round += 1) { + replica.checkout(root); + expect(replica.getMap("map").getLastEditor("x")).toBe("1"); + expect(replica.toJSON()).toEqual(expected); + replica.checkoutToLatest(); + expect(replica.getMap("map").getLastEditor("x")).toBe("2"); + } + } + }); + } +});