diff --git a/.changeset/loro-js-richtext-anchors.md b/.changeset/loro-js-richtext-anchors.md new file mode 100644 index 000000000..dd032e016 --- /dev/null +++ b/.changeset/loro-js-richtext-anchors.md @@ -0,0 +1,19 @@ +--- +"loro.js": major +--- + +Rich text now uses Rust's position model. A mark's start and end anchors are elements of the text sequence, as in Rust (`loro-crdt`), so every insert, delete, and mark position loro.js writes or reads counts them. + +**Breaking:** loro.js reads rich-text history written by earlier versions with Rust's positions. Earlier versions wrote and read positions without the anchors, so every edit made after a mark in a document edited only with loro.js can now apply elsewhere, and styles can cover other text (for example, after marking `ab` in `abcd`, inserting `X` at 3 and deleting 1 gave `acXd` in earlier versions; those ops now read as `bXcd`, as Rust has always read them). Documents shared with `loro-crdt` peers had already diverged there (Rust read `bXcd` from loro.js's updates, loro.js read `acdX` from Rust's); they now agree with Rust. There is no version marker in the data: read "Upgrading from 0.2" in the README before upgrading. + +Behavior now follows Rust: + +- Text inserted where a mark starts or ends goes inside or outside the mark according to the mark's expand setting, as Rust decides it; concurrent edits at a mark boundary converge with Rust. +- **Breaking:** `mark` and `unmark` throw when `start >= end`, and skip a mark that would change nothing, as Rust does. +- **Breaking:** internal `_`-prefixed members of `LoroText` that were visible in the type declarations (`_insertVisible`, `_applyMark`, `_styleRuns`, `_validateInsertPosition`, `_unicodePosition`, `_detachedStyleCounter`) were removed. +- `applyDelta` drops a style that inserted text would inherit when the insert does not list it, like Rust's `apply_delta`. +- Undo and redo restore the attributes a mark replaced. +- Snapshots write and read the anchors like Rust, so checking out an older version of a styled text gives the same result in both runtimes. Shallow snapshots null the values of styles that no text is left in, like Rust. +- Inserting into a nested text that a snapshot loaded lazily no longer throws or drops the styles at the insert position, and checking out a version that has a mark's start but not its end no longer reports the whole text as inserted. + +Upgrading: a snapshot written by an earlier version keeps its current rich text, but its history reads like Rust, so a document loaded from it must not stay in use. Read the final state (`text.toDelta()`) with the old version and build a new document from it (reading it with 1.0 right after importing an old snapshot only works when no updates were stored after that snapshot: with a snapshot of bold `ab` in `abcd` and a later update that inserted `X` at 3 and deleted 1, the old version shows `[a]cXd` and 1.0 and Rust show `[bX]cd`), or import the old updates everywhere and accept Rust's reading. Exporting a shallow snapshot from a document loaded from an old snapshot switches that document itself to Rust's reading. diff --git a/AGENTS.md b/AGENTS.md index d4c8b63da..024ee3ff6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,8 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [context/loro-js-performance.md](context/loro-js-performance.md). - loro.js vs Rust randomized Text differential tests and the Rust behaviors they work around: [context/loro-js-rust-differential.md](context/loro-js-rust-differential.md). +- loro.js rich-text style anchors, entity positions, and Rust's insert/delete/mark + position rules: [context/loro-js-richtext-anchors.md](context/loro-js-richtext-anchors.md). - WASM panic/OOM reporting channels, `__wbindgen_start` glue invariant, and trap-testing recipes: [context/wasm-error-reporting.md](context/wasm-error-reporting.md). diff --git a/context/loro-js-performance.md b/context/loro-js-performance.md index 38450b831..830ba979e 100644 --- a/context/loro-js-performance.md +++ b/context/loro-js-performance.md @@ -40,12 +40,19 @@ JavaScript constant factor. - `ordered-index.ts` is the ordered rank index used for map keys and tree children. Insert/delete/rank lookup are expected O(log n), while ordered iteration is O(n). +- Rich-text style anchors are zero-width elements of the Text sequence, as in + Rust ([loro-js-richtext-anchors.md](loro-js-richtext-anchors.md)). Each + subtree also counts its elements without UTF-16 width, so Unicode and entity + positions convert in O(log n); a sequence without anchors takes the old fast + paths. - `text-style-index.ts` stores style histories in disjoint operation-ID ranges, - separately from scalar Text elements. Applying, unapplying, or checking a - style range is expected O(log style-runs + affected style-runs). Full-range - marks and their subscribed checkout events no longer write or inspect every - character. Delta and snapshot output reuse a run-local style resolver so - their work remains linear in returned text and style runs. + separately from scalar Text elements. A style covers the elements physically + between its anchors, so applying one is expected O(log n + ID runs in its + range). Checking or undoing a style range is expected O(log style-runs + + affected style-runs). Full-range marks and their subscribed checkout events + no longer write or inspect every character. Delta and snapshot output reuse a + run-local style resolver so their work remains linear in returned text and + style runs. - `LoroDoc` maintains per-peer change arrays, end counters, operation counts, current frontiers, sorted-history cache, and per-change dependency-version caches. Latest version/frontier lookup is O(peer/frontier count), and @@ -171,12 +178,16 @@ JavaScript constant factor. copies every untouched entry and rewrites only the touched ones. A delete transition is still refused unless the deletion index recorded that delete, for example one imported while detached. -- loro.js does not count Rust rich-text style anchors in Text operation - positions, so a replay of Rust-created styled text can differ from its - snapshot state (for example, an insert right after a mark's end anchor). The - completion therefore compares the replay of a Text or List with the snapshot - state (visible ids, plus the delta when styled). When they differ, the replay - is discarded and the container becomes `unreplayable`: it keeps its snapshot +- The completion compares the replay of a Text or List with the snapshot state + (visible ids, plus the values). With style anchors in the sequence + (loro-dev/loro#1137) a replay of Rust or loro.js history, styled or not, + equals its snapshot state; before, loro.js did not count the anchors and a + replay of Rust-created styled text could differ. A replay now differs only + when the snapshot state disagrees with its own history: a snapshot written by + loro.js 0.2 (see `loro-js/README.md`, "Upgrading from 0.2"), or a loro.js + shallow snapshot with the retained-range gap described in + [loro-js-rust-differential.md](loro-js-rust-differential.md). When they + differ, the replay is discarded and the container becomes `unreplayable`: it keeps its snapshot state, encoded once, and is never given a replay. A transition that touches it runs without its operations and then moves it separately (`#planSnapshotStates`): when the installed state already has every forward @@ -230,13 +241,13 @@ JavaScript constant factor. the latest state and `revertTo` are right). This is loro.js's MovableList transition gap, left to loro-dev/loro#1132. - A full `#rebuildFromHistory` (the non-incremental fallback, shallow export, - `forkAt`) first checks the snapshot-hydrated Text containers that have or - had styles, including lazily encoded ones, and then rebuilds unreplayable - containers the same way. "Had" matters: when every marked character was - deleted, the snapshot state has no style, but the mark's anchors still shift - later Rust positions, so the replay differs (`#hasStyleHistory`: a mark - operation in the history, indexed as `#markedTexts`, or styles in the shallow - root state). Plain text is not checked, so it pays no extra replay. Only the root state of a shallow export uses the replay, since the + `forkAt`) rebuilds unreplayable containers the same way. It no longer checks + snapshot-hydrated styled Text first (`#checkSnapshotSequences`, removed in + loro-dev/loro#1137): that extra replay per styled Text existed because the + anchors shifted Rust positions, which the anchor model now counts. So styled + and plain Text behave alike: a hydrated container that no transition has + completed takes the replay of its history, which for a 0.2 snapshot is Rust's + reading. Only the root state of a shallow export uses the replay, since the snapshot state is later than the root. `forkAt` keeps a snapshot state only in a fork whose version includes that state's version; an older fork has none of the operations needed to undo later ones in that state, so it keeps @@ -356,7 +367,10 @@ heap and 322.1 MB RSS. The original array implementation was estimated at 30–50 minutes. Prefix measurements from 20k through the full trace scale approximately linearly. The matching Rust Criterion benchmark has a 47.711 ms point estimate on the same -machine, so TypeScript is about 7.4x slower in absolute time. B4 leaves 182,315 +machine, so TypeScript is about 7.4x slower in absolute time. Merging +consecutive inserts of a transaction into one op, as Rust does, shrank the B4 +update export from 1,153,540 to 274,574 bytes, and merging contiguous text in +the Text state shrank the snapshot from 309,780 to 206,553 bytes. B4 leaves 182,315 scalar objects but packs them into 13,613 TypeScript treap nodes. The same run measured snapshot export at 162.4 ms, update export at 129.3 ms, snapshot import at 161.5 ms, and @@ -416,6 +430,48 @@ isolated repeated probe. Retreating/restoring that full-range mark takes about now scale with ID/style runs and emitted formatting ranges rather than the 64k characters. +With style anchors in the sequence (September 28, Node 22, best of 7 on a +loaded machine, same process for both revisions): the 64k full-range mark takes +0.07–0.08 ms (0.06–0.08 before), its retreat/restore 0.06–0.07 ms (about 0.01 +before), and the subscribed restore 0.06 ms. Typing 1,000 characters inside the +bold range takes 3.1–3.3 ms (1.7–2.1 before): each insert intersects the style +memberships of its two physical neighbors. Marking the same range again and +again nests anchors: the n-th mark's range contains the n-1 earlier start +anchors as separate ID runs, so applying it and moving the version across it +are O(n), as in Rust, whose `StyleRangeMap` has one segment per anchor there. +`text-repeated-mark-tail-{retreat,restore}` therefore grows with its size +(0.5/1.1/2.2/6.9 ms at 1k/2k/4k/8k, versus 0.16–0.29 ms before); the Rust WASM +build takes 7.1/19/362 ms to retreat 1k/4k/16k such marks, and building the 16k +history takes 402 s in loro.js and 542 s in Rust. Every other +`bench:complexity` entry stays flat from 1k to 8k. + +The zero-width counters are four more fields in every treap node, maintained +on every update even for text without anchors, so plain Text edits pay a +constant cost: 8,000 subscribed middle inserts take 15.4–15.8 ms against +14.3–14.4 ms without them, and `text-subscribed-batch`, `history-commit`, and +`history-update-batch-import` are 12–20% slower at 8k (September 29, Node 22, +alternating runs at 1-minute load 6–8). No entry grows with size because of +them. Moving them into a sidecar that only anchored sequences allocate, like +the line-break totals, would remove that cost. + +Reading a Rust-written styled Text from a snapshot (16k characters, 200 marks, +2k later edits) and checking out a middle version takes 24.7–25.0 ms the +first time, 4.1–4.3 ms back to the latest, and 11.8–12.2 ms for a fork; `main` +takes 35.9–37.2, 3.1–3.3, and 29.8–30.5 ms but shows wrong text, because it +marks the text unreplayable and toggles its snapshot state. + +These costs were reviewed and accepted (loro-dev/loro#1137). The review measured, +on Node 26 at 1-minute load 25–40, typing 1,000 characters inside a 64k bold +range at 2.9–4.4 ms (1.9 ms on `main`) and the 8k repeated-mark tail retreat and +restore at 8.0–8.1 and 11.6–13.2 ms (0.4–0.6 ms on `main`). Re-measured after the +review fixes (two runs of two interleaved rounds with `main`, Node 22, 1-minute +load 5–25): the repeated-mark tail retreat takes 0.51–0.71/1.18–1.27/2.25–2.80/ +6.11–8.25 ms and its restore 0.47–0.53/1.16–1.35/2.08–2.57/5.89–7.75 ms at +1k/2k/4k/8k (0.08–0.43 ms on `main`); a 64k full-range mark applies in 0.11–0.17 +ms (0.06–0.12), retreats in 0.22–0.35 ms (0.11–0.14), and restores in 0.11–0.14 +ms (0.02–0.03); typing 1,000 characters inside it varies with load (2.4–6.6 ms, +2.6–4.3 ms on `main` in the same alternating runs). + A subscribed forward checkout that combines a full-range delete and mark takes 0.41 ms at 1k characters and 0.16 ms at 8k after warmup. Historical mark positions are converted directly to causal ID runs, and removed ID runs are diff --git a/context/loro-js-richtext-anchors.md b/context/loro-js-richtext-anchors.md new file mode 100644 index 000000000..5cb4188c4 --- /dev/null +++ b/context/loro-js-richtext-anchors.md @@ -0,0 +1,178 @@ +# loro-js Rich-Text Anchors + +Verified against code 2026-09-29. + +loro.js keeps rich-text style anchors in the text sequence, as Rust does, so a +Text op position means the same thing in both runtimes. This article covers the +model, the position rules, and how the indexes keep the +[complexity contract](loro-js-performance.md). + +## Why anchors are sequence elements + +Rust writes a mark as two ops: `StyleStart { start, end, key, value, info }` at +counter `c` and `StyleEnd` at `c + 1`. Each inserts a zero-width anchor into the +Fugue sequence (`RichtextDiffCalculator::apply_crdt_op_to_tracker` in +`crates/loro-internal/src/diff_calc.rs`): the start anchor at entity position +`start`, then the end anchor at `end + 1`, in a causal view that now contains the +start anchor. Every Text op position (`insert.pos`, `delete.pos`, +`mark.start/end`, binary and JSON alike) is an *entity index*: Unicode scalars +plus visible anchors, in the op's causal view (`container/richtext.rs`). + +Converting positions only when importing or exporting cannot be exact. A +concurrent insert can have an anchor as its Fugue origin, and whether text is +inside a style depends on its Fugue order relative to the anchors. So loro.js +stores the anchors in `LoroText._sequence`. + +## Data model + +- An anchor is a `TextElement` with `value: ""` and `anchor: { style, isEnd }` + (`loro-js/src/runtime/containers.ts`). The two anchors of a mark share one + `TextStyle`: `startId`, `lamport`, `info`, `value`, `key`, and the op's `end`. + The start anchor has the `StyleStart` ID and lamport, the end anchor the next + counter and lamport. +- Anchors use single-element storage. `TextSequenceSpan` never holds one; + `LoroText.compact` keeps 32-element chunks with anchors as element arrays. +- `SequenceIndex` counts the elements with no UTF-16 width in each node + (`ownZeroWidth`, `ownVisibleZeroWidth`, `visibleZeroWidth`, `allZeroWidth`). + Lists never have one. A sequence without anchors pays one addition per + recompute. +- Styles stay in `TextStyleIndex`, keyed by element ID, but membership now comes + from the anchors: a style covers every element physically between its start + and end anchors, both included. That includes hidden elements and elements + inserted there later, and it never changes once the elements exist. A version + only filters which styles apply. +- `LoroText` keeps each peer's start-anchor counters in a sorted array + (`#anchorStarts`), so an ID run can be split at anchors without looking up its + elements. + +## Positions + +| Unit | How loro.js converts it | +| --- | --- | +| UTF-16 / UTF-8 | The existing prefix metrics. Anchors have width 0, so every entity index in a run of anchors maps to the same offset. | +| Unicode | `entity - zeroWidthBeforeVisibleIndex(entity)`; back with `visibleIndexOfWidthElement(unicode)`. O(log n). | +| Entity | The visible index of `SequenceIndex`. Op positions use it. | + +Local edits follow Rust's `TextHandler`: + +- **Insert** (`LoroText.#insertPosition`, Rust `find_best_insert_pos`): start + right after the scalar that ends at the UTF-16 position (or at 0), then skip + the anchors there until the first one that must stay after the new text: a + start anchor of an unmark (`null` or `false` value), a start anchor of a style + that does not expand before, or an end anchor of a style that expands after. + Rust's insert-position cache used to bypass this rule at text-leaf boundaries + (fixed in loro-dev/loro#1135; + `insert_position_near_style_anchors_does_not_depend_on_cursor_cache` in + `crates/loro/tests/contracts/text_richtext_advanced.rs`). +- **Delete** (`LoroText._deleteRuns`, Rust `get_text_entity_ranges`): the text + from the scalar containing the first UTF-16 unit to the one containing the + last, as runs that are consecutive in both entity position and ID. Anchors are + never deleted; they split runs. Ops are written last run first, so each + recorded position is still valid when applied. +- **Mark** (`LoroText.mark`): both ends use the insert rule. Like Rust's + `mark_with_txn`, the mark is skipped when every entity position in the range + already resolves the key to the value, or when it is an unmark and no position + has the key. `start >= end` throws, as in Rust. Detached texts insert anchors + with Rust's `TextStyleInfoFlag::BOLD` info. +- **applyDelta** follows Rust's `apply_delta`: inserted text without an + attribute explicitly drops each style it would inherit (the styles shared by + both visible neighbors), and all marks run after the whole delta. + +Remote ops: + +- Text inserts and anchors use the same Fugue insertion at the entity position + in the op's causal view. +- Deletes are resolved by position in the causal view, like Rust's tracker, not + by `start_id` (`LoroText._deleteTargets`). +- A new element takes every style shared by its physical neighbors, which is + exactly the set of styles whose anchors enclose it (`#inheritGapStyles`). A + new end anchor applies its style to the physical range between its anchors + (`LoroText._applyStyleEnd`). + +## Styles, versions, events, and undo + +- The value of a key is the covering style with the greatest `(lamport, peer)`; + `null` means the attribute is absent (Rust `StyleValue::get`). +- A style applies once its end anchor is included in a version: + `startId.counter + 1 < version` (`latestIncluded` in `text-style-index.ts`). +- A mark's event is emitted with its end anchor and covers the text whose + resolved value changed (`LoroText._styleChangeRuns`). +- Checkout toggles anchors like inserted text and reports style changes through + `TextStyleIndex.transitions` over the style's member runs. Anchors take the + bulk path of `#applyVersionTransition`, which records no event, so the text's + event baseline is started when an anchor op is collected; without it, a + version with a start anchor but no end anchor was diffed against an empty + text and reported the whole text as inserted. +- Anchors are sequence elements, so a checkout that leaves them hidden in the + state must not insert them again. When a transition cannot run incrementally + (a movable-list set, for example) and would apply the forward records + directly, `hasMaterializedSequenceInsertions` in `document.ts` counts + `text-mark`/`text-mark-end` ops with the inserts and sends the checkout to a + rebuild; `diff` rebuilds its `from` state in the same case. Missing that + threw `duplicate sequence id` on `attach` after a snapshot import and two + checkouts. +- Undoing a mark gives back, wherever it changed the value, the value from just + before its op (`LoroText._undoStyle`, through `mark`). Known gap: `mark` puts + the restoring start anchor after a before- or both-expand start anchor, so + after undoing such a mark, text typed at its old start still gets the undone + value; Rust leaves it unstyled. A fix (start the restoring mark right before + the undone style's start anchor) is commit d1663c9e, left for a follow-up. + loro.js undo also does not transform against remote edits the way Rust's + `UndoManager` does. + +## Snapshots + +`LoroDoc.#containerState` writes Rust's text state layout (`encode_snapshot_fast` +in `crates/loro-internal/src/state/richtext_state.rs`): text spans and anchors in +document order. A start anchor is a zero-length span followed by its mark entry; +an end anchor is a span of length -1 at the start counter + 1 that keeps the +start's lamport offset. Contiguous text is merged into one span. +`#hydrateContainerState` rebuilds the same elements and each style's range +(`LoroText._appendElements`). A Text loaded from a snapshot hydrates lazily; +every position lookup (`#entityFrom`, `#insertPosition`) hydrates first, because +the anchors decide where inserted text goes. + +Shallow snapshots null the value of every style with no text between its anchors +at the shallow root, except both-expand styles, and null the same styles in the +latest state (`redactDeadStyleValues` in `document.ts`, Rust's +`redact_dead_style_values`; see +[shallow-snapshot-style-redaction.md](shallow-snapshot-style-redaction.md)). +Anchors and IDs stay, so positions do not move. + +## Complexity + +- Without anchors, every conversion short-circuits on + `visibleZeroWidthLength === 0`, and delete runs use `visibleIdRuns` directly. +- Unicode conversions, the insert rule, delete runs, and applying a style are + O(log n + anchors at the position + returned runs). Applying a style to a + contiguous range is O(log n + ID runs), as before. +- Inserting inside styled text intersects two style histories per insert, which + is proportional to the number of styles covering that position. + +## Data written by loro.js 0.2 + +loro.js 0.2 wrote and read Text positions without the anchors. Decision +(2026-09-28): later versions read all data with Rust's positions and add no +version marker, like the plain-text changes in +[loro-js-rust-differential.md](loro-js-rust-differential.md). Reasons: a Text +op position must mean the same thing in both runtimes; documents shared with +`loro-crdt` peers had already diverged (each runtime read the other's ops at +other positions); and a marker would need an encoding change that Rust does +not have. The cost: in a document edited only with 0.2, every op after a mark +can apply elsewhere, and a 0.2 snapshot keeps its 0.2 state while its history +reads like Rust (checkouts keep that state through the `unreplayable` fallback +of loro-dev/loro#1126 and show older versions approximately; a fork or shallow +export rebuilds it from history, which is Rust's reading). The +package version is major; `loro-js/README.md` ("Upgrading from 0.2") gives the +migration paths, and `loro-js/tests/legacy-data.test.ts` pins the readings of +0.2 fixtures in both runtimes. + +## Testing + +- `loro-js/tests/richtext-anchors.test.ts` compares loro.js with Rust fixtures + (`loro-js/scripts/write-rust-richtext-fixtures.cjs`): the JSON ops loro.js + writes, imports of Rust updates and snapshots, and checkouts. +- `crates/loro/tests/loro_js_interop.rs` imports loro.js rich-text updates and + snapshots written by `loro-js/scripts/rewrite-rust-fixture.mjs`. +- The randomized Rust differential suite is described in + [loro-js-rust-differential.md](loro-js-rust-differential.md). diff --git a/context/loro-js-rust-differential.md b/context/loro-js-rust-differential.md index c417d8c68..696207f12 100644 --- a/context/loro-js-rust-differential.md +++ b/context/loro-js-rust-differential.md @@ -1,6 +1,6 @@ # loro-js Differential Tests Against Rust -Verified against code 2026-09-28. +Verified against code 2026-09-29. `loro-js/tests/richtext-differential.test.ts` runs random multi-peer Text scenarios in loro.js and in Rust (`loro-crdt`, nodejs WASM build) side by side @@ -38,6 +38,20 @@ per runtime with the same peer ID. Actions: split a change or an op (a mark with only its start anchor, for example), so transitions also run between historical and mid-change versions; - full and shallow snapshot export from either runtime, imported into both; + a full snapshot's copies then check out three versions in a row (picked like + `checkout`'s), diff the last two, and attach, so an imported state moves back + and forward over elements it already holds. The loro.js diff must turn the + `from` text into the `to` text; Rust's is only logged, because between + concurrent versions it can put a style value on the wrong side (seed 162 at + 120 steps, September 29: its diff sets `hl:false` on text its own checkout + shows with `hl:"x"`); +- movable-list inserts and sets, written by Rust and imported by loro.js. The + list is not compared; its set ops keep checkouts off the incremental path, so + they replay the text's records (the path of the `duplicate sequence id` + regression that seed 19 catches). Moves and deletes are left out: loro.js and + Rust disagree on such lists (loro-dev/loro#1132), and Rust panics + (`movable_list_state.rs` "consistency check failed") when it checks out a + snapshot loro.js wrote with the differing state; - cursor creation, then cursor resolution after every later step; - `revertTo` and undo/redo. @@ -80,8 +94,18 @@ tolerates them, so it also runs against older reference builds: can diverge from its own full-history import when they are wrong. The harness compares loro.js with the source document instead. -Snapshot-import checkout (`snapshotCheckout`) is off by default until loro.js -can check out a document imported from a snapshot (loro-dev/loro#1126). +Checkout after a full snapshot import (`snapshotCheckout`) is on. Checkout into +the retained range of a shallow snapshot (`shallowCheckout`) is off: loro.js +still diverges there when it wrote the snapshot. `#calculateShallowStart` in +`loro-js/src/runtime/document.ts` uses the requested frontiers as the root, +while Rust moves the root back to the nearest critical version, and retained +ops concurrent with a non-critical root do not replay from its state. The +harness visits only a few versions per export and fails 6 of 40 rich and 5 of +20 plain 120-step seeds (September 29). An exhaustive check of every retained +version from every commit root (second review of loro-dev/loro#1136/#1137) +finds most multi-peer histories affected: over half of the non-critical-root +versions diverge, and none with a critical root. The fix is to choose the root +like Rust; it is not done yet. ## Data written by loro.js 0.2 diff --git a/crates/loro/tests/loro_js_interop.rs b/crates/loro/tests/loro_js_interop.rs index 1aea9892a..60fb124b3 100644 --- a/crates/loro/tests/loro_js_interop.rs +++ b/crates/loro/tests/loro_js_interop.rs @@ -39,6 +39,16 @@ const TS_LEGACY_TREE_MOVE_SNAPSHOT: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/legacy-tree-move-snapshot.ts.blob"); const TS_LEGACY_TREE_MOVE_SHALLOW_SNAPSHOT: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/legacy-tree-move-shallow.ts.blob"); +const TS_RICHTEXT_BASE: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/richtext-base.ts.blob"); +const TS_RICHTEXT_LEFT: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/richtext-left.ts.blob"); +const TS_RICHTEXT_RIGHT: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/richtext-right.ts.blob"); +const TS_RICHTEXT_SNAPSHOT: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/richtext-snapshot.ts.blob"); +const TS_RICHTEXT_EXPECTED_JSON: &[u8] = + include_bytes!("../../../loro-js/tests/fixtures/rust/richtext.expected.json"); const TS_CURSOR: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/cursor.ts.blob"); const TS_AWARENESS: &[u8] = include_bytes!("../../../loro-js/tests/fixtures/rust/awareness.ts.blob"); @@ -157,6 +167,83 @@ fn imports_typescript_shallow_snapshot_and_checks_out_its_root() { ); } +/// A Quill delta with null attributes dropped and equal neighbors merged, so +/// deltas compare by content rather than by how each runtime splits them. +fn normalized_delta(value: serde_json::Value) -> serde_json::Value { + let mut output: Vec = Vec::new(); + for item in value.as_array().expect("a delta is an array") { + let insert = item["insert"].as_str().expect("a text insert").to_string(); + let attributes: serde_json::Map = item + .get("attributes") + .and_then(|attributes| attributes.as_object()) + .map(|attributes| { + attributes + .iter() + .filter(|(_, value)| !value.is_null()) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() + }) + .unwrap_or_default(); + if let Some(previous) = output.last_mut() { + let previous_attributes = previous + .get("attributes") + .cloned() + .unwrap_or(serde_json::Value::Object(Default::default())); + if previous_attributes == serde_json::Value::Object(attributes.clone()) { + let merged = format!("{}{}", previous["insert"].as_str().unwrap(), insert); + previous["insert"] = merged.into(); + continue; + } + } + let mut entry = serde_json::json!({ "insert": insert }); + entry["attributes"] = serde_json::Value::Object(attributes); + output.push(entry); + } + serde_json::Value::Array(output) +} + +fn richtext_delta(doc: &LoroDoc) -> serde_json::Value { + normalized_delta(doc.get_text("text").get_richtext_value().to_json_value()) +} + +#[test] +fn imports_typescript_rich_text_with_anchor_positions() { + let expected: serde_json::Value = + serde_json::from_slice(TS_RICHTEXT_EXPECTED_JSON).expect("valid rich-text fixture"); + let expected_delta = normalized_delta(expected["delta"].clone()); + + let doc = LoroDoc::new(); + for update in [TS_RICHTEXT_BASE, TS_RICHTEXT_LEFT, TS_RICHTEXT_RIGHT] { + doc.import(update) + .expect("Rust should import TypeScript rich-text updates"); + } + assert_eq!(richtext_delta(&doc), expected_delta); + + let snapshot = LoroDoc::new(); + snapshot + .import(TS_RICHTEXT_SNAPSHOT) + .expect("Rust should import a TypeScript rich-text snapshot"); + assert_eq!(richtext_delta(&snapshot), expected_delta); + let frontiers: Vec = expected["base"]["frontiers"] + .as_array() + .unwrap() + .iter() + .map(|id| { + ID::new( + id["peer"].as_str().unwrap().parse().unwrap(), + id["counter"].as_i64().unwrap() as i32, + ) + }) + .collect(); + snapshot + .checkout(&frontiers.into()) + .expect("the snapshot history should check out"); + assert_eq!( + richtext_delta(&snapshot), + normalized_delta(expected["base"]["delta"].clone()) + ); +} + #[test] fn decodes_typescript_cursor_with_rust() { let cursor = Cursor::decode(TS_CURSOR).expect("Rust should decode a TypeScript cursor"); diff --git a/loro-js/AGENTS.md b/loro-js/AGENTS.md index 61232601b..8b6be0101 100644 --- a/loro-js/AGENTS.md +++ b/loro-js/AGENTS.md @@ -9,6 +9,9 @@ the same binary and JSON data as Rust (`loro-crdt`) and give the same results. [context/loro-js-performance.md](../context/loro-js-performance.md). - Randomized Text comparison with Rust (build the nodejs WASM package first): [context/loro-js-rust-differential.md](../context/loro-js-rust-differential.md). +- Rich text: style anchors live in the Text sequence and op positions count + them. Read [context/loro-js-richtext-anchors.md](../context/loro-js-richtext-anchors.md) + before changing Text positions, marks, or Text snapshot state. - Encoding formats: [context/internal-encoding.md](../context/internal-encoding.md) and `docs/encoding*.md`. diff --git a/loro-js/README.md b/loro-js/README.md index 65b80487d..9468bf0fd 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -94,9 +94,11 @@ The following paths are checked against the Rust implementation: - JSON update import/export and the postcard Awareness/EphemeralStore protocols. - Rust-produced fixtures imported by TypeScript and TypeScript-produced fixtures imported by Rust. -- Randomized multi-peer text edits run side by side with the `loro-crdt` WASM build - (`tests/richtext-differential.test.ts`): positions, deletes, concurrency, checkout, - snapshots, cursors, events, and `revertTo`. +- Randomized multi-peer text and rich-text edits run side by side with the + `loro-crdt` WASM build (`tests/richtext-differential.test.ts`): positions, deletes, + marks with every expand mode, concurrency, checkout, snapshots, cursors, events, + and `revertTo`. Style anchors are text-sequence elements, as in Rust, so every op + position counts them. This is not yet a claim of complete behavioral equivalence with every `loro-crdt` edge case. Important current limits are: @@ -109,11 +111,11 @@ edge case. Important current limits are: importing into a new document or using `LoroDoc.fromSnapshot()`. - `diff()` preserves parent-before-child ordering, but independent containers at the same depth can appear in a different order than Rust's internal hash-map iteration. -- `UndoManager` uses ID-span-based semantic undo for sequence edits and common - map/tree/counter changes. Style-only rich-text edits and movable-list move/set undo - still use simplified behavior. -- The hardest overlapping rich-text-anchor and concurrent movable-list cases still use - simplified metadata compared with the Rust implementation. +- `UndoManager` uses ID-span-based semantic undo for sequence edits, marks, and + common map/tree/counter changes. It does not transform an undo against later remote + edits like Rust's, and movable-list move/set undo still uses simplified behavior. +- The hardest concurrent movable-list cases still use simplified metadata compared + with the Rust implementation. - `subscribeJsonpath()` deliberately uses broad invalidation, so its callback can have false positives. @@ -130,6 +132,11 @@ can read differently after the upgrade: - **Concurrent text inserts.** 0.2 could order an insert after a sibling's subtree differently from Rust. 1.0 orders it like Rust, so such a history can give other text (`bé9a` in 0.2, `béa9` in 1.0 and Rust). +- **Rich text.** 0.2 counted text positions without a mark's start and end anchors; + Rust and 1.0 count them. Every insert, delete, and mark op that 0.2 wrote after a + mark can therefore apply elsewhere (with bold on `ab` in `abcd`, inserting `X` at + 3 and deleting 1 gave `acXd` in 0.2; its ops read as `bXcd` in 1.0 and Rust), and + styles can cover other text. - **Cursors.** 1.0 resolves cursors like Rust: a cursor reports its target's own offset. A cursor encoded by 0.2 with side 1, or at the end of the text, therefore resolves one character earlier, whatever the characters are (on `abc`, the end @@ -139,19 +146,19 @@ can read differently after the upgrade: Documents also edited by `loro-crdt` peers already disagreed with Rust in these cases; 1.0 makes them agree. -How 1.0 reads data written by 0.2 (measured on the example above): +How 1.0 reads data written by 0.2 (measured on the examples above): -| 0.2 data | 1.0 reads | -| --------------------- | ----------------------------------------------------------------------- | -| Updates, JSON updates | Rust's reading: `béa9` | -| Full/shallow snapshot | The 0.2 text (`bé9a`) as current state, but the history reads like Rust | +| 0.2 data | 1.0 reads | +| --------------------- | ------------------------------------------------------------------------------- | +| Updates, JSON updates | Rust's reading: `béa9`, `bXcd` | +| Full/shallow snapshot | The 0.2 text (`bé9a`, `acXd`) as current state, but the history reads like Rust | A document loaded from a 0.2 snapshot therefore holds a current state that its own history disagrees with. Its current state is safe to read right after the import, but do not keep using the document: exporting it as updates gives `béa9`, a replica loaded from updates holds other text and stays different after new edits -(`béQ9aZ` and `béQa9Z`), older versions it checks out can be approximate, and a -fork can rebuild it from its history, which switches it to Rust's reading. +(`béQ9aZ` and `béQa9Z`), older versions it checks out are approximate (`acd` for +`abcd`), and a fork can rebuild it from its history, which switches it to Rust's reading. Exporting a shallow snapshot from it (`export({ mode: "shallow-snapshot" })`) does the same to the document itself: its current state becomes Rust's reading (`bé9a` turns into `béa9`). Rust can even show text that matches neither reading after a @@ -165,7 +172,9 @@ checkout there and back. every replica. Do not import old updates or snapshots into it. Reading the state with 1.0 right after importing a 0.2 snapshot gives the same result only when that snapshot is the final version: if updates were stored after it, 1.0 (like - Rust) reads those updates the Rust way. + Rust) reads those updates the Rust way. For a 0.2 snapshot of bold `ab` in + `abcd` followed by the update that inserted `X` at 3 and deleted 1, 0.2 shows + `[a]cXd` and 1.0 and Rust show `[bX]cd`; read such data with 0.2. 2. **Keep the history, accept Rust's reading.** Import the 0.2 updates (not snapshots) on every replica, and check the content, because it can differ from what 0.2 showed. diff --git a/loro-js/scripts/rewrite-rust-fixture.mjs b/loro-js/scripts/rewrite-rust-fixture.mjs index aa2aab24b..616c0820d 100644 --- a/loro-js/scripts/rewrite-rust-fixture.mjs +++ b/loro-js/scripts/rewrite-rust-fixture.mjs @@ -221,6 +221,68 @@ writeFileSync( cursorDoc.getText("text").getCursor(1).encode(), ); +// Rich text: concurrent edits at style boundaries after marks and unmarks. Op +// positions count style anchors, so Rust must read them the same way. +const richStyles = { + bold: { expand: "after" }, + link: { expand: "none" }, + em: { expand: "before" }, + hl: { expand: "both" }, +}; +const richLeft = new LoroDoc(); +richLeft.setPeerId(21); +richLeft.configTextStyle(richStyles); +richLeft.getText("text").insert(0, "Hello 😀 World"); +richLeft.getText("text").mark({ start: 0, end: 5 }, "bold", true); +richLeft.getText("text").mark({ start: 6, end: 8 }, "link", "https://loro.dev"); +richLeft.commit(); +const richVersion = richLeft.frontiers(); +const richBase = richLeft.export({ mode: "update" }); +const richRight = new LoroDoc(); +richRight.setPeerId(22); +richRight.configTextStyle(richStyles); +richRight.import(richBase); +richLeft.getText("text").insert(5, "!"); +richLeft.getText("text").unmark({ start: 1, end: 3 }, "bold"); +richLeft.getText("text").mark({ start: 3, end: 10 }, "hl", "yellow"); +richLeft.commit(); +richRight.getText("text").insert(0, ">"); +richRight.getText("text").insert(6, "?"); +richRight.getText("text").delete(7, 3); +richRight.getText("text").mark({ start: 2, end: 4 }, "em", true); +richRight.commit(); +const richLeftUpdate = richLeft.export({ + mode: "update", + from: richRight.oplogVersion(), +}); +const richRightUpdate = richRight.export({ + mode: "update", + from: richLeft.oplogVersion(), +}); +richLeft.import(richRightUpdate); +richRight.import(richLeftUpdate); +if ( + JSON.stringify(richLeft.getText("text").toDelta()) !== + JSON.stringify(richRight.getText("text").toDelta()) +) { + throw new Error("TypeScript rich-text replicas did not converge"); +} +writeFileSync(fixtureUrl("richtext-base.ts.blob"), richBase); +writeFileSync(fixtureUrl("richtext-left.ts.blob"), richLeftUpdate); +writeFileSync(fixtureUrl("richtext-right.ts.blob"), richRightUpdate); +writeFileSync( + fixtureUrl("richtext-snapshot.ts.blob"), + richLeft.export({ mode: "snapshot" }), +); +const richLatest = richLeft.getText("text").toDelta(); +richLeft.checkout(richVersion); +const richBaseDelta = richLeft.getText("text").toDelta(); +richLeft.checkoutToLatest(); +writeFileSync( + fixtureUrl("richtext.expected.json"), + `${JSON.stringify({ delta: richLatest, base: { frontiers: richVersion, delta: richBaseDelta } }, null, 2)}\n`, +); + const awareness = new AwarenessWasm("123", 30_000); awareness.setLocalState({ status: "typing", position: 3 }); writeFileSync(fixtureUrl("awareness.ts.blob"), awareness.encodeAll()); diff --git a/loro-js/scripts/write-legacy-fixtures.mjs b/loro-js/scripts/write-legacy-fixtures.mjs index 1b807cdde..80209f531 100644 --- a/loro-js/scripts/write-legacy-fixtures.mjs +++ b/loro-js/scripts/write-legacy-fixtures.mjs @@ -62,4 +62,24 @@ function sync(from, to) { expected["cursor-emoji"] = owner.getCursorPos(cursor).offset; } +// Marks whose anchors 0.2 left out of text positions: with bold on "ab" in +// "abcd", inserting X at 3 and deleting 1 gave "acXd" in 0.2 (and Rust), but +// the ops 0.2 wrote mean "bXcd" to Rust. +{ + const owner = doc(1); + owner.configTextStyle({ bold: { expand: "after" } }); + const text = owner.getText("t"); + text.insert(0, "abcd"); + owner.commit(); + text.mark({ start: 0, end: 2 }, "bold", true); + owner.commit(); + text.insert(3, "X"); + text.delete(1, 1); + owner.commit(); + write("richtext-positions.update.blob", owner.export({ mode: "update" })); + write("richtext-positions.snapshot.blob", owner.export({ mode: "snapshot" })); + write("richtext-positions.json", JSON.stringify(owner.exportJsonUpdates(), null, 2)); + expected["richtext-positions"] = text.toDelta(); +} + write("expected.json", `${JSON.stringify(expected, null, 2)}\n`); diff --git a/loro-js/scripts/write-rust-richtext-fixtures.cjs b/loro-js/scripts/write-rust-richtext-fixtures.cjs new file mode 100644 index 000000000..4dee4eb8a --- /dev/null +++ b/loro-js/scripts/write-rust-richtext-fixtures.cjs @@ -0,0 +1,81 @@ +/* eslint-disable no-console */ +// Writes the Rust (loro-crdt) side of the rich-text interop fixtures used by +// tests/richtext-anchors.test.ts. Build the nodejs WASM package first +// (`pnpm -C crates/loro-wasm build-dev`) or point LORO_WASM_NODEJS at one. +const { writeFileSync } = require("node:fs"); +const path = require("node:path"); + +const wasmPath = + process.env.LORO_WASM_NODEJS ?? + path.resolve(__dirname, "../../crates/loro-wasm/nodejs/index.js"); +const { LoroDoc } = require(wasmPath); +const fixture = (name) => path.resolve(__dirname, "../tests/fixtures/rust", name); + +function newDoc() { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.configTextStyle({ bold: { expand: "after" }, link: { expand: "none" } }); + return doc; +} + +const scenarios = { + // loro-dev/loro#1133: positions written after a mark count its anchors. + "richtext-mark-positions"(doc) { + const text = doc.getText("text"); + text.insert(0, "abcd"); + doc.commit(); + text.mark({ start: 0, end: 2 }, "bold", true); + doc.commit(); + text.insert(3, "X"); + doc.commit(); + text.delete(1, 1); + doc.commit(); + }, + "richtext-unmark-emoji"(doc) { + const text = doc.getText("text"); + text.insert(0, "a😀bc"); + doc.commit(); + text.mark({ start: 0, end: 3 }, "bold", true); + doc.commit(); + text.unmark({ start: 0, end: 1 }, "bold"); + doc.commit(); + text.delete(3, 1); + doc.commit(); + }, + "richtext-checkout-mark"(doc, versions) { + const text = doc.getText("text"); + text.insert(0, "abc"); + doc.commit(); + versions.push(doc.frontiers()); + text.mark({ start: 0, end: 3 }, "bold", true); + text.insert(3, "d"); + doc.commit(); + }, +}; + +for (const [name, edit] of Object.entries(scenarios)) { + const doc = newDoc(); + const versions = []; + edit(doc, versions); + const checkouts = versions.map((frontiers) => { + doc.checkout(frontiers); + const delta = doc.getText("text").toDelta(); + doc.checkoutToLatest(); + return { frontiers, delta }; + }); + writeFileSync( + fixture(`${name}.json`), + `${JSON.stringify( + { + delta: doc.getText("text").toDelta(), + checkouts, + updates: doc.exportJsonUpdates(), + }, + null, + 2, + )}\n`, + ); + writeFileSync(fixture(`${name}.blob`), doc.export({ mode: "update" })); + writeFileSync(fixture(`${name}.snapshot.blob`), doc.export({ mode: "snapshot" })); + console.log(name, JSON.stringify(doc.getText("text").toDelta())); +} diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index c9794f1de..e94b407e6 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -96,10 +96,10 @@ interface MovableListState extends ListState { interface TextState extends SequenceContainerState { readonly sequence: SequenceIndex; readonly detachedCounter: number; - readonly detachedStyleCounter: number; readonly attributeHistoryComplete: boolean; - readonly styleIndex: TextStyleIndex; + readonly styleIndex: TextStyleIndex; readonly styleVersion: CausalVersion | undefined; + readonly anchorStarts: Map; } interface ParentLink { @@ -633,6 +633,8 @@ export class LoroList extends LoroContainer { export interface TextElement extends SequenceElement { value: string; + /** Set on a zero-width rich-text style anchor. */ + anchor?: TextAnchor | undefined; attributes?: Map | undefined; attributeMeta?: Map | undefined; attributeHistory?: Map | undefined; @@ -1404,15 +1406,39 @@ export interface TextStyleMeta { readonly value: RuntimeValue; } +/** + * A rich-text mark. As in Rust, its start and end anchors are zero-width + * elements of the text sequence, and the style covers every element between + * them, including elements inserted there later. See + * context/loro-js-richtext-anchors.md. + */ +export interface TextStyle extends TextStyleMeta { + readonly key: string; + /** + * The mark op's end entity position in its causal view; the end anchor goes + * to `end + 1` once the start anchor is in place. + */ + readonly end: number; +} + +export interface TextAnchor { + readonly style: TextStyle; + readonly isEnd: boolean; +} + +/** Style info byte of detached marks: Rust's `TextStyleInfoFlag::BOLD`. */ +const DETACHED_STYLE_INFO = 0x84; + export class LoroText extends LoroContainer { _sequence = createTextSequence(); _detachedCounter = 0; - _detachedStyleCounter = 0; _attributeHistoryComplete = true; - _styleIndex = new TextStyleIndex(); + _styleIndex = new TextStyleIndex(); _styleVersion: CausalVersion | undefined; + /** Start-anchor counters by peer, ascending; the end anchor is the next counter. */ + #anchorStarts = new Map(); readonly #attributeValuesCache = new WeakMap< - ReadonlyMap, + ReadonlyMap, ReadonlyMap >(); @@ -1424,7 +1450,11 @@ export class LoroText extends LoroContainer { /** Rebuilds fragmented physical text spans without changing CRDT history. */ compact(): void { this._ensureHydrated(); - this._sequence.compact((elements) => TextSequenceSpan.fromElements(elements)); + this._sequence.compact((elements) => + elements.some((element) => element.anchor !== undefined) + ? undefined + : TextSequenceSpan.fromElements(elements), + ); } kind(): "Text" { @@ -1448,10 +1478,10 @@ export class LoroText extends LoroContainer { if (!Number.isSafeInteger(line) || line < 0 || line >= this.lineCount) { return undefined; } - const unicodePosition = this._sequence.visibleIndexAfterLineBreaks(line); - return unicodePosition === undefined + const entity = this._sequence.visibleIndexAfterLineBreaks(line); + return entity === undefined ? undefined - : this._sequence.metricOffsetAtVisibleIndex(unicodePosition, "utf16"); + : this._sequence.metricOffsetAtVisibleIndex(entity, "utf16"); } /** Returns the zero-based line containing a UTF-16 position. */ @@ -1460,10 +1490,10 @@ export class LoroText extends LoroContainer { if (!Number.isSafeInteger(position) || position < 0 || position > this.length) { return undefined; } - const unicodePosition = this.convertPos(position, "utf16", "unicode"); - return unicodePosition === undefined + const entity = this.#entityAt(position); + return entity === undefined ? undefined - : this._sequence.lineBreakOffsetAtVisibleIndex(unicodePosition); + : this._sequence.lineBreakOffsetAtVisibleIndex(entity); } /** Returns a line without its LF or CRLF terminator. */ @@ -1481,6 +1511,7 @@ export class LoroText extends LoroContainer { return this._stringRange(0, this._sequence.visibleLength); } + /** Text of the visible entity range; style anchors contribute nothing. */ _stringRange(start: number, end: number): string { this._ensureHydrated(); const chunks: string[] = []; @@ -1540,6 +1571,7 @@ export class LoroText extends LoroContainer { if (!(span instanceof TextSequenceSpan)) { for (let offset = spanStart; offset < spanEnd; offset += 1) { const element = span.elementAt(offset); + if (element.anchor !== undefined) continue; if (!append(element.value, element.id.peer, element.id.counter)) { return false; } @@ -1575,13 +1607,15 @@ export class LoroText extends LoroContainer { previousRunCounter, ); }, - (element) => append(element.value, element.id.peer, element.id.counter), + (element) => + element.anchor !== undefined || + append(element.value, element.id.peer, element.id.counter), ); if (!stopped && chunk.length > 0) callback(chunk.join("")); } insert(pos: number, text: string): void { - const unicodePosition = this._validateInsertPosition(pos); + const entity = this.#insertPosition(pos); if (text.length === 0) return; if (this._doc === undefined) { const span = TextSequenceSpan.fromText( @@ -1592,10 +1626,11 @@ export class LoroText extends LoroContainer { undefined, ); this._detachedCounter += span.length; - this._sequence.insertSpanAtVisible(unicodePosition, span); + this._sequence.insertSpanAtVisible(entity, span); + this.#inheritGapStyles(span.idAt(0), span.length); return; } - this._doc._textInsert(this, unicodePosition, text); + this._doc._textInsert(this, entity, text); } push(text: string): void { @@ -1611,14 +1646,21 @@ export class LoroText extends LoroContainer { delete(pos: number, len: number): void { validateRange(pos, len, this.length); if (len === 0) return; - const start = this._unicodePosition(pos); - const end = this._unicodePosition(pos + len); + this.#requireBoundary(pos); + this.#requireBoundary(pos + len); if (this._doc === undefined) { - for (const element of this._sequence.visibleRange(start, end)) - this._sequence.setDeleted(element, true); + for (const { run } of this._deleteRuns(pos, pos + len)) { + for (let offset = 0; offset < run.length; offset += 1) { + const element = this._sequence.findById({ + peer: run.start.peer, + counter: run.start.counter + offset, + }); + if (element !== undefined) this._sequence.setDeleted(element, true); + } + } return; } - this._doc._textDelete(this, start, end - start); + this._doc._textDelete(this, pos, len); } deleteUtf8(index: number, length: number): void { @@ -1631,14 +1673,14 @@ export class LoroText extends LoroContainer { slice(start: number, end: number): string { validateRange(start, end - start, this.length); - const unicodeStart = this._unicodePosition(start); - const unicodeEnd = this._unicodePosition(end); - return this._stringRange(unicodeStart, unicodeEnd); + return this._stringRange(this.#requireBoundary(start), this.#requireBoundary(end)); } charAt(pos: number): string { if (pos === this.length) return ""; - return this._sequence.atVisible(this._unicodePosition(pos))?.value ?? ""; + this.#requireBoundary(pos); + const entity = this._sequence.visibleIndexOfMetricUnit(pos, "utf16"); + return entity === undefined ? "" : (this._sequence.atVisible(entity)?.value ?? ""); } splice(pos: number, len: number, text: string): string { @@ -1648,65 +1690,71 @@ export class LoroText extends LoroContainer { return removed; } + /** + * Marks a UTF-16 range like Rust's `TextHandler::mark`: both ends go where + * text inserted there would go, and a mark that would not change any + * position's value (or an unmark of a key the range never had) is skipped. + */ mark(range: { start: number; end: number }, key: string, value: unknown): void { - validateRange(range.start, range.end - range.start, this.length); - const start = this._unicodePosition(range.start); - const end = this._unicodePosition(range.end); + this._ensureHydrated(); + const { start, end } = range; + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start >= end) { + throw new RangeError("mark start must be less than its end"); + } + validateRange(start, end - start, this.length); + const entityStart = this.#insertPosition(start); + const entityEnd = this.#insertPosition(end); + const normalized = normalizeDetachedValue(value); + if (this.#markIsRedundant(entityStart, entityEnd, key, normalized)) return; if (this._doc === undefined) { - this._applyMark(start, end, key, normalizeDetachedValue(value)); + this.#markDetached(entityStart, entityEnd, key, normalized); return; } - this._doc._textMark(this, start, end, key, value); + this._doc._textMark(this, entityStart, entityEnd, key, value); } unmark(range: { start: number; end: number }, key: string): void { - validateRange(range.start, range.end - range.start, this.length); - const start = this._unicodePosition(range.start); - const end = this._unicodePosition(range.end); - if ( - !this._styleIndex.rangeHasKey(this._styleRuns(start, end), key, this._styleVersion) - ) { - return; - } this.mark(range, key, null); } + /** + * Applies a Quill delta like Rust's `apply_delta`: inserted text without an + * attribute explicitly drops the style it would inherit, and every attribute + * is marked after the whole delta is applied. + */ applyDelta(delta: readonly Delta[]): void { let position = 0; const marks: { readonly start: number; readonly end: number; - readonly attributes: Readonly>; + readonly attributes: readonly (readonly [string, unknown])[]; }[] = []; for (const operation of delta) { if ("insert" in operation) { - this.insert(position, operation.insert); const length = operation.insert.length; - if (operation.attributes !== undefined) { - marks.push({ - start: position, - end: position + length, - attributes: operation.attributes, - }); + if (length === 0) continue; + if (position > this.length) { + throw new RangeError(`delta position ${position} is out of range`); } + const attributes = operation.attributes ?? {}; + const overrides = this.#insertOverrides(position, attributes); + this.insert(position, operation.insert); + marks.push({ start: position, end: position + length, attributes: overrides }); position += length; } else if ("delete" in operation) { this.delete(position, operation.delete); } else { - if (operation.attributes !== undefined) { - marks.push({ - start: position, - end: position + operation.retain, - attributes: operation.attributes, - }); + const end = position + operation.retain; + if (end > this.length) { + throw new RangeError(`delta range ${position}..${end} is out of range`); } - position += operation.retain; + const entries = Object.entries(operation.attributes ?? {}); + if (entries.length > 0) marks.push({ start: position, end, attributes: entries }); + position = end; } } for (const { start, end, attributes } of marks) { - for (const [key, value] of Object.entries(attributes)) { - this.mark({ start, end }, key, value); - } + for (const [key, value] of attributes) this.mark({ start, end }, key, value); } } @@ -1717,10 +1765,11 @@ export class LoroText extends LoroContainer { sliceDelta(start: number, end: number): Delta[] { this._ensureHydrated(); validateRange(start, end - start, this.length); - const unicodeStart = this._unicodePosition(start); - const unicodeEnd = this._unicodePosition(end); return textElementsToDelta( - this._sequence.visibleRange(unicodeStart, unicodeEnd), + this._sequence.visibleRange( + this.#requireBoundary(start), + this.#requireBoundary(end), + ), this.#attributeResolver(), ); } @@ -1738,25 +1787,8 @@ export class LoroText extends LoroContainer { convertPos(index: number, from: TextPosType, to: TextPosType): number | undefined { this._ensureHydrated(); if (!isTextPosType(from) || !isTextPosType(to)) return undefined; - const visibleLength = this._sequence.visibleLength; - const directMetric = - from === "unicode" || - (from === "utf16" && this._sequence.visibleUtf16Length === visibleLength) || - (from === "utf8" && this._sequence.visibleUtf8Length === visibleLength); - const unicodeIndex = directMetric - ? Number.isSafeInteger(index) && index >= 0 && index <= visibleLength - ? index - : undefined - : this._sequence.visibleIndexAtMetricOffset(index, from); - if (unicodeIndex === undefined) return undefined; - if ( - to === "unicode" || - (to === "utf16" && this._sequence.visibleUtf16Length === visibleLength) || - (to === "utf8" && this._sequence.visibleUtf8Length === visibleLength) - ) { - return unicodeIndex; - } - return this._sequence.metricOffsetAtVisibleIndex(unicodeIndex, to); + const entity = this.#entityFrom(index, from); + return entity === undefined ? undefined : this.#offsetAt(entity, to); } /** @@ -1770,21 +1802,14 @@ export class LoroText extends LoroContainer { const length = this.length; if (length === 0) return new Cursor(this.id, undefined, side === 0 ? -1 : side, 0); if (pos >= length) return new Cursor(this.id, undefined, 1, length); + const element = this.#elementAt(pos); const unicodePosition = this.convertPos(pos, "utf16", "unicode"); - if (unicodePosition === undefined) return undefined; - return new Cursor( - this.id, - this._sequence.atVisible(unicodePosition)!.id, - side, - unicodePosition, - ); + if (element === undefined || unicodePosition === undefined) return undefined; + return new Cursor(this.id, element.id, side, unicodePosition); } getEditorOf(pos: number): string | undefined { - const unicodePosition = this.convertPos(pos, "utf16", "unicode"); - return unicodePosition === undefined - ? undefined - : this._sequence.atVisible(unicodePosition)?.id.peer.toString(); + return this.#elementAt(pos)?.id.peer.toString(); } update(text: string, _options?: TextUpdateOptions): void { @@ -1803,6 +1828,7 @@ export class LoroText extends LoroContainer { return this.toString(); } + /** Visible elements in order, style anchors included. */ _visibleElements(): TextElement[] { this._ensureHydrated(); return this._sequence.visible(); @@ -1818,23 +1844,32 @@ export class LoroText extends LoroContainer { return this._sequence.atVisible(position); } - _insertVisible( - position: number, - characters: readonly string[], - ids: readonly CodecId[], - lamports: readonly number[], - ): void { - this._sequence.insertAtVisible( - position, - characters.map((value, index) => ({ - value, - id: ids[index]!, - lamport: lamports[index]!, - deleted: false, - originLeft: undefined, - originRight: undefined, - })), - ); + /** + * Appends snapshot state: text scalars and style anchors in document order, + * all visible. Styles cover the elements between their anchors. + */ + _appendElements(elements: readonly TextElement[]): void { + if (elements.length === 0) return; + const base = this._sequence.allLength; + this._sequence.insertAtPhysical(base, elements); + const starts = new Map(); + for (let index = 0; index < elements.length; index += 1) { + const element = elements[index]!; + const anchor = element.anchor; + if (anchor === undefined) continue; + this.#registerAnchor(element); + if (!anchor.isEnd) { + starts.set(anchor.style, base + index); + continue; + } + const start = starts.get(anchor.style); + if (start === undefined) continue; + this._styleIndex.add( + this._sequence.physicalIdRuns(start, base + index + 1), + anchor.style.key, + anchor.style, + ); + } } _insertFugue( @@ -1844,37 +1879,13 @@ export class LoroText extends LoroContainer { lamport: number, causalVersion: CausalVersion, ): void { - const current = this._sequence.isFullyIncluded(causalVersion); - const authored = current ? undefined : this._sequence.causalView(causalVersion); - const authoredLength = current ? this._sequence.visibleLength : authored!.length; - const authoredPosition = Math.min(position, authoredLength); - const currentContext = current - ? this._sequence.visibleInsertionContext(authoredPosition) - : undefined; - const authoredAt = (index: number): TextElement | undefined => - current ? this._sequence.atVisible(index) : authored!.at(index); - const authoredLeft = currentContext?.left ?? authoredAt(authoredPosition - 1); const insertion = fugueInsertion( this._sequence, position, startId, causalVersion, true, - currentContext ?? { current: false, left: authoredLeft }, ); - let inherited: Map | undefined; - if (!this._styleIndex.isEmpty) { - inherited = new Map(); - for (const [key, meta] of this._attributeMetasAt(authoredLeft, causalVersion)) { - if ((meta.info & 0b100) !== 0) inherited.set(key, meta); - } - for (const [key, meta] of this._attributeMetasAt( - authoredAt(authoredPosition), - causalVersion, - )) { - if ((meta.info & 0b010) !== 0) inherited.set(key, meta); - } - } const firstCodePoint = text.codePointAt(0)!; const singleScalar = text.length === 1 || (text.length === 2 && firstCodePoint > 0xffff); @@ -1903,14 +1914,120 @@ export class LoroText extends LoroContainer { recordFugueInsertionRun(this._sequence, insertion, startId); insertedLength = span.length; } - if (inherited !== undefined && inherited.size > 0) { - const insertedRun = [{ start: { ...startId }, length: insertedLength }]; - for (const [key, meta] of inherited) { - this._styleIndex.add(insertedRun, key, meta); + this.#inheritGapStyles(startId, insertedLength, insertion.insertIndex); + } + + /** Inserts a mark's start anchor at its entity position in the causal view. */ + _applyStyleStart( + position: number, + style: TextStyle, + lamport: number, + causalVersion: CausalVersion, + ): void { + this.#insertAnchor( + position, + { + value: "", + id: { peer: style.startId.peer, counter: style.startId.counter }, + lamport, + deleted: false, + originLeft: undefined, + originRight: undefined, + anchor: { style, isEnd: false }, + }, + causalVersion, + ); + } + + /** + * Inserts a mark's end anchor at `end + 1` in the causal view (which now + * contains the start anchor) and applies the style to everything between. + */ + _applyStyleEnd( + id: CodecId, + lamport: number, + causalVersion: CausalVersion, + ): TextStyle | undefined { + const start = this._sequence.findById({ peer: id.peer, counter: id.counter - 1 }); + const anchor = start?.anchor; + if (start === undefined || anchor === undefined || anchor.isEnd) return undefined; + const style = anchor.style; + const end: TextElement = { + value: "", + id: { peer: id.peer, counter: id.counter }, + lamport, + deleted: false, + originLeft: undefined, + originRight: undefined, + anchor: { style, isEnd: true }, + }; + this.#insertAnchor(style.end + 1, end, causalVersion); + this.#applyStyleRange(start, end, style); + if (this._styleVersion !== undefined) { + // A detached document edited in place keeps showing its own new style. + const next = new Map(this._styleVersion); + next.set(id.peer, Math.max(next.get(id.peer) ?? 0, id.counter + 1)); + this._styleVersion = next; + } + return style; + } + + /** ID runs of every element a style covers, its anchors included. */ + _styleMemberRuns(style: TextStyle): SequenceIdRun[] { + const start = this._sequence.findById(style.startId); + const end = this._sequence.findById({ + peer: style.startId.peer, + counter: style.startId.counter + 1, + }); + if (start === undefined || end === undefined) return []; + const from = this._sequence.physicalIndexOf(start); + const to = this._sequence.physicalIndexOf(end); + if (from === undefined || to === undefined || from > to) return []; + return this._sequence.physicalIdRuns(from, to + 1); + } + + /** Runs of the elements whose attribute an applied style changes. */ + _styleChangeRuns(style: TextStyle): SequenceIdRun[] { + return this._styleIndex.winningRuns( + this._styleMemberRuns(style), + style.key, + style, + this._styleVersion, + styleValuesEqual, + ); + } + + /** + * Undoes a style like Rust's undo of its op: wherever the style changed the + * resolved value (from `before` to `after`, the versions around its op), the + * text gets the `before` value back (`null` drops the attribute). + */ + _undoStyle(style: TextStyle, before: CausalVersion, after: CausalVersion): void { + const changes = this._styleIndex + .transitions(this._styleMemberRuns(style), style.key, before, after) + .filter( + ({ before: previous, after: next }) => + !styleValuesEqual(previous?.value ?? null, next?.value ?? null), + ) + .flatMap(({ run, before: previous }) => + this._sequence + .visibleMetricRangesForIdRuns([run], "utf16") + .map((range) => ({ range, value: previous?.value ?? null })), + ) + .sort((left, right) => left.range.start - right.range.start); + for (const { range, value } of changes) { + if (range.start < range.end) { + this.mark({ start: range.start, end: range.end }, style.key, value); } } } + /** The anchored style whose start anchor has this ID. */ + _styleAt(startId: CodecId): TextStyle | undefined { + const anchor = this._sequence.findById(startId)?.anchor; + return anchor === undefined || anchor.isEnd ? undefined : anchor.style; + } + _deleteIdSpan(startId: CodecId, length: number, deletedBy?: CodecId): void { if (Math.abs(length) === 1) { const element = this._sequence.findById(startId); @@ -1979,54 +2096,68 @@ export class LoroText extends LoroContainer { } } - _applyMark( - start: number, - end: number, - key: string, - value: RuntimeValue, - meta?: TextStyleMeta, - causalVersion?: CausalVersion, - ): void { - const appliedMeta = - meta ?? - ({ - startId: { peer: -1n, counter: this._detachedStyleCounter }, - lamport: this._detachedStyleCounter++, - info: 0, - value, - } satisfies TextStyleMeta); - this._styleIndex.add(this._styleRuns(start, end, causalVersion), key, appliedMeta); - if (this._styleVersion !== undefined) { - const next = new Map(this._styleVersion); - next.set( - appliedMeta.startId.peer, - Math.max( - next.get(appliedMeta.startId.peer) ?? 0, - appliedMeta.startId.counter + 1, - ), - ); - this._styleVersion = next; + /** + * Text runs covering a UTF-16 range, like Rust's `get_text_entity_ranges`: + * each run is consecutive in both entity position and ID. Style anchors are + * never deleted and split runs. Positions are entity indexes in the current + * state. + */ + _deleteRuns(start: number, end: number): { position: number; run: SequenceIdRun }[] { + if (start >= end) return []; + const sequence = this._sequence; + if (sequence.visibleZeroWidthLength === 0) { + const first = this.#entityAt(start); + const stop = this.#entityAt(end); + if (first === undefined || stop === undefined) return []; + let position = first; + return sequence.visibleIdRuns(first, stop).map((run) => { + const entry = { position, run }; + position += run.length; + return entry; + }); } - } - - _styleRuns(start: number, end: number, causalVersion?: CausalVersion): SequenceIdRun[] { - if (causalVersion === undefined || this._sequence.isFullyIncluded(causalVersion)) { - return this._sequence.visibleIdRuns(start, end); + const first = sequence.visibleIndexOfMetricUnit(start, "utf16"); + const last = sequence.visibleIndexOfMetricUnit(end - 1, "utf16"); + if (first === undefined || last === undefined) return []; + const output: { position: number; run: SequenceIdRun }[] = []; + let position = first; + for (const run of sequence.visibleIdRuns(first, last + 1)) { + const peer = run.start.peer; + let counter = run.start.counter; + const runEnd = counter + run.length; + for (const anchor of this.#anchorCounters(peer, counter, runEnd)) { + if (anchor > counter) { + output.push({ + position, + run: { start: { peer, counter }, length: anchor - counter }, + }); + position += anchor - counter; + } + position += 1; + counter = anchor + 1; + } + if (counter < runEnd) { + output.push({ + position, + run: { start: { peer, counter }, length: runEnd - counter }, + }); + position += runEnd - counter; + } } - return this._sequence.causalView(causalVersion).idRuns(start, end); + return output; } _attributeHistoryAt( element: TextElement, key: string, - ): readonly TextStyleMeta[] | undefined { + ): readonly TextStyle[] | undefined { return this._styleIndex.historyAt(element.id, key); } _attributeMetasAt( element: TextElement | undefined, version: CausalVersion | undefined = this._styleVersion, - ): ReadonlyMap { + ): ReadonlyMap { return element === undefined ? new Map() : this._styleIndex.metasAt(element.id, version); @@ -2034,7 +2165,7 @@ export class LoroText extends LoroContainer { _attributeMetasResolver( version: CausalVersion | undefined = this._styleVersion, - ): (element: TextElement) => ReadonlyMap { + ): (element: TextElement) => ReadonlyMap { const metasAt = this._styleIndex.resolver(version); return (element) => metasAt(element.id); } @@ -2047,13 +2178,27 @@ export class LoroText extends LoroContainer { this._styleVersion = version === undefined ? undefined : new Map(version); } + _ensureLineIndex(): void { + this._ensureHydrated(); + this._sequence.enableLineBreaks(); + } + + _reset(): void { + this._sequence.reset(); + this._detachedCounter = 0; + this._attributeHistoryComplete = true; + this._styleIndex.reset(); + this._styleVersion = undefined; + this.#anchorStarts.clear(); + } + #attributeResolver(): (element: TextElement) => ReadonlyMap { const metasAt = this._attributeMetasResolver(); return (element) => this.#attributeValues(metasAt(element)); } #attributeValues( - metas: ReadonlyMap, + metas: ReadonlyMap, ): ReadonlyMap { const cached = this.#attributeValuesCache.get(metas); if (cached !== undefined) return cached; @@ -2065,36 +2210,296 @@ export class LoroText extends LoroContainer { return attributes; } - _validateInsertPosition(position: number): number { + /** An entity index at a UTF-16 boundary; any one within a run of anchors. */ + #entityAt(position: number): number | undefined { + return this.#entityFrom(position, "utf16"); + } + + #entityFrom(index: number, from: TextPosType): number | undefined { this._ensureHydrated(); - const unicodePosition = this.convertPos(position, "utf16", "unicode"); - if (unicodePosition === undefined) { - throw new RangeError(`text position ${position} is out of range`); + const sequence = this._sequence; + const length = sequence.visibleLength; + if (sequence.visibleZeroWidthLength === 0) { + const direct = + from === "unicode" || + (from === "utf16" && sequence.visibleUtf16Length === length) || + (from === "utf8" && sequence.visibleUtf8Length === length); + if (direct) { + return Number.isSafeInteger(index) && index >= 0 && index <= length + ? index + : undefined; + } + } + return from === "unicode" + ? sequence.visibleIndexOfWidthElement(index) + : sequence.visibleIndexAtMetricOffset(index, from); + } + + #offsetAt(entity: number, to: TextPosType): number | undefined { + const sequence = this._sequence; + if (to === "unicode") { + return sequence.visibleZeroWidthLength === 0 + ? entity + : entity - sequence.zeroWidthBeforeVisibleIndex(entity)!; + } + if ( + sequence.visibleZeroWidthLength === 0 && + (to === "utf16" ? sequence.visibleUtf16Length : sequence.visibleUtf8Length) === + sequence.visibleLength + ) { + return entity; } - return unicodePosition; + return sequence.metricOffsetAtVisibleIndex(entity, to); } - _unicodePosition(position: number): number { + #requireBoundary(position: number): number { this._ensureHydrated(); - const unicodePosition = this.convertPos(position, "utf16", "unicode"); - if (unicodePosition === undefined) { + const entity = this.#entityAt(position); + if (entity === undefined) { throw new RangeError(`text position ${position} is not on a UTF-16 boundary`); } - return unicodePosition; + return entity; } - _ensureLineIndex(): void { + /** The text scalar that starts at a UTF-16 position. */ + #elementAt(position: number): TextElement | undefined { this._ensureHydrated(); - this._sequence.enableLineBreaks(); + if (this.#entityAt(position) === undefined) return undefined; + const entity = this._sequence.visibleIndexOfMetricUnit(position, "utf16"); + return entity === undefined ? undefined : this._sequence.atVisible(entity); } - _reset(): void { - this._sequence.reset(); - this._detachedCounter = 0; - this._detachedStyleCounter = 0; - this._attributeHistoryComplete = true; - this._styleIndex.reset(); - this._styleVersion = undefined; + /** + * The entity index where text inserted at a UTF-16 position goes: Rust's + * `find_best_insert_pos`. Among the style anchors at that position, the text + * goes before the first anchor that must stay after it (see + * `anchorStaysAfterInsert`), or after all of them. + */ + #insertPosition(position: number): number { + // Hydrates a lazily loaded text; its anchors decide where the text goes. + const entity = this.#entityAt(position); + if (entity === undefined) { + throw new RangeError(`text position ${position} is out of range`); + } + const sequence = this._sequence; + if (sequence.visibleZeroWidthLength === 0) return entity; + let index = + position === 0 ? 0 : sequence.visibleIndexOfMetricUnit(position - 1, "utf16")! + 1; + sequence.forEachVisibleRange(index, sequence.visibleLength, (element) => { + const anchor = element.anchor; + if (anchor === undefined || anchorStaysAfterInsert(anchor)) return false; + index += 1; + return undefined; + }); + return index; + } + + /** + * Whether a mark would change nothing, as Rust checks before writing one: + * every entity position of the range already resolves the key to the value, + * or it is an unmark and no position has the key at all. + */ + #markIsRedundant( + start: number, + end: number, + key: string, + value: RuntimeValue, + ): boolean { + const runs = this._sequence.visibleIdRuns(start, end); + const version = this._styleVersion; + if ( + this._styleIndex.everyWinner(runs, key, version, (meta) => + styleValuesEqual(meta.value, value), + ) + ) { + return true; + } + return value === null && !this._styleIndex.someHasKey(runs, key, version); + } + + /** Detached text marks like Rust's `mark_for_detached`, with local anchors. */ + #markDetached(start: number, end: number, key: string, value: RuntimeValue): void { + const counter = this._detachedCounter; + this._detachedCounter += 2; + const style: TextStyle = { + startId: { peer: 0n, counter }, + lamport: counter, + info: DETACHED_STYLE_INFO, + value, + key, + end, + }; + const startAnchor: TextElement = { + value: "", + id: { peer: 0n, counter }, + lamport: counter, + deleted: false, + originLeft: undefined, + originRight: undefined, + anchor: { style, isEnd: false }, + }; + const endAnchor: TextElement = { + value: "", + id: { peer: 0n, counter: counter + 1 }, + lamport: counter + 1, + deleted: false, + originLeft: undefined, + originRight: undefined, + anchor: { style, isEnd: true }, + }; + this._sequence.insertAtVisible(end, [endAnchor]); + this._sequence.insertAtVisible(start, [startAnchor]); + this.#registerAnchor(startAnchor); + this.#registerAnchor(endAnchor); + this.#inheritGapStyles(startAnchor.id, 1); + this.#inheritGapStyles(endAnchor.id, 1); + this.#applyStyleRange(startAnchor, endAnchor, style); + } + + /** + * Rust's `insert_with_txn_and_attr`: the styles text inserted at a position + * would get, compared with the requested attributes. Returns the marks that + * make the inserted text carry exactly those attributes. + */ + #insertOverrides( + position: number, + attributes: Readonly>, + ): [string, unknown][] { + const entity = this.#insertPosition(position); + const inherited = new Map(); + if (entity > 0 && entity < this._sequence.visibleLength) { + // Rust's `get_styles_for_insert`: the styles shared by both visible + // neighbors, resolved per key. + const left = this._styleIndex.membershipAt( + this._sequence.atVisible(entity - 1)!.id, + ); + const right = this._styleIndex.membershipAt(this._sequence.atVisible(entity)!.id); + if (left !== undefined && right !== undefined) { + const version = this._styleVersion; + for (const [key, history] of intersectHistories(left, right)) { + for (let index = history.length - 1; index >= 0; index -= 1) { + const style = history[index]!; + if ( + version === undefined || + style.startId.counter + 1 < (version.get(style.startId.peer) ?? 0) + ) { + inherited.set(key, style.value); + break; + } + } + } + } + } + const overrides: [string, unknown][] = []; + for (const [key, value] of inherited) { + const requested = attributes[key]; + if ( + Object.hasOwn(attributes, key) && + styleValuesEqual(value, normalizeDetachedValue(requested)) + ) { + continue; + } + overrides.push([key, requested ?? null]); + } + for (const [key, value] of Object.entries(attributes)) { + if (!inherited.has(key)) overrides.push([key, value]); + } + return overrides; + } + + #insertAnchor( + position: number, + element: TextElement, + causalVersion: CausalVersion, + ): void { + const insertion = fugueInsertion( + this._sequence, + position, + element.id, + causalVersion, + true, + ); + element.originLeft = insertion.originLeft; + element.originRight = insertion.originRight; + this._sequence.insertAtPhysical(insertion.insertIndex, [element]); + recordFugueInsertion(this._sequence, insertion, [element.id]); + this.#registerAnchor(element); + this.#inheritGapStyles(element.id, 1, insertion.insertIndex); + } + + #registerAnchor(element: TextElement): void { + if (element.anchor === undefined || element.anchor.isEnd) return; + let starts = this.#anchorStarts.get(element.id.peer); + if (starts === undefined) { + starts = []; + this.#anchorStarts.set(element.id.peer, starts); + } + const counter = element.id.counter; + let low = 0; + let high = starts.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (starts[middle]! < counter) low = middle + 1; + else high = middle; + } + if (starts[low] !== counter) starts.splice(low, 0, counter); + } + + /** Counters of the style anchors of `peer` in `[start, end)`, ascending. */ + #anchorCounters(peer: bigint, start: number, end: number): number[] { + const starts = this.#anchorStarts.get(peer); + if (starts === undefined) return []; + let low = 0; + let high = starts.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (starts[middle]! + 1 < start) low = middle + 1; + else high = middle; + } + const output: number[] = []; + for (let index = low; index < starts.length && starts[index]! < end; index += 1) { + const counter = starts[index]!; + if (counter >= start) output.push(counter); + if (counter + 1 >= start && counter + 1 < end) output.push(counter + 1); + } + return output; + } + + /** + * Gives newly inserted elements every style that covers their physical gap: + * the styles shared by the physical neighbors, which is exactly the set of + * styles whose start anchor precedes and whose end anchor follows the gap. + */ + #inheritGapStyles(startId: CodecId, length: number, insertIndex?: number): void { + if (this._styleIndex.isEmpty) return; + const sequence = this._sequence; + let index = insertIndex; + if (index === undefined) { + const first = sequence.findByIdRaw(startId); + index = first === undefined ? undefined : sequence.physicalIndexOf(first); + if (index === undefined) return; + } + if (index === 0) return; + const left = sequence.atPhysicalRaw(index - 1); + const right = sequence.atPhysicalRaw(index + length); + if (left === undefined || right === undefined) return; + const leftStyles = this._styleIndex.membershipAt(left.id); + const rightStyles = this._styleIndex.membershipAt(right.id); + if (leftStyles === undefined || rightStyles === undefined) return; + const shared = + leftStyles === rightStyles + ? leftStyles + : intersectHistories(leftStyles, rightStyles); + if (shared.size > 0) { + this._styleIndex.addMembership([{ start: startId, length }], shared); + } + } + + #applyStyleRange(start: TextElement, end: TextElement, style: TextStyle): void { + const from = this._sequence.physicalIndexOf(start); + const to = this._sequence.physicalIndexOf(end); + if (from === undefined || to === undefined || from > to) return; + this._styleIndex.add(this._sequence.physicalIdRuns(from, to + 1), style.key, style); } /** Installs `state`, or an empty state, and returns the replaced state. */ @@ -2102,18 +2507,18 @@ export class LoroText extends LoroContainer { const previous: TextState = { sequence: this._sequence, detachedCounter: this._detachedCounter, - detachedStyleCounter: this._detachedStyleCounter, attributeHistoryComplete: this._attributeHistoryComplete, styleIndex: this._styleIndex, styleVersion: this._styleVersion, + anchorStarts: this.#anchorStarts, }; const next = state as TextState | undefined; this._sequence = next?.sequence ?? createTextSequence(); this._detachedCounter = next?.detachedCounter ?? 0; - this._detachedStyleCounter = next?.detachedStyleCounter ?? 0; this._attributeHistoryComplete = next?.attributeHistoryComplete ?? true; - this._styleIndex = next?.styleIndex ?? new TextStyleIndex(); + this._styleIndex = next?.styleIndex ?? new TextStyleIndex(); this._styleVersion = next?.styleVersion; + this.#anchorStarts = next?.anchorStarts ?? new Map(); return previous; } } @@ -2128,6 +2533,74 @@ function createTextSequence(): SequenceIndex { ); } +/** + * Rust's `prefer_insert_before` plus its first rule: text inserted at an + * anchor's position goes before a start anchor of an unmark (a null or false + * value), before a start anchor whose style does not expand before, and before + * an end anchor whose style expands after. + */ +function anchorStaysAfterInsert(anchor: TextAnchor): boolean { + const { style, isEnd } = anchor; + if (isEnd) return (style.info & 0b100) !== 0; + return style.value === null || style.value === false || (style.info & 0b010) === 0; +} + +function intersectHistories( + left: ReadonlyMap, + right: ReadonlyMap, +): Map { + const shared = new Map(); + for (const [key, history] of left) { + const other = right.get(key); + if (other === undefined) continue; + const members = new Set(other); + const common = history.filter((meta) => members.has(meta)); + if (common.length > 0) shared.set(key, common); + } + return shared; +} + +/** Deep equality of style values; `undefined` equals `null`. */ +function styleValuesEqual(left: unknown, right: unknown): boolean { + if (left === undefined) left = null; + if (right === undefined) right = null; + if (left === right) return true; + if (left instanceof Uint8Array && right instanceof Uint8Array) { + return ( + left.length === right.length && left.every((byte, index) => byte === right[index]) + ); + } + if (Array.isArray(left) && Array.isArray(right)) { + return ( + left.length === right.length && + left.every((item, index) => styleValuesEqual(item, right[index])) + ); + } + if ( + typeof left === "object" && + typeof right === "object" && + left !== null && + right !== null && + !Array.isArray(left) && + !Array.isArray(right) && + !(left instanceof Uint8Array) && + !(right instanceof Uint8Array) && + !isContainer(left) && + !isContainer(right) + ) { + const leftEntries = Object.entries(left); + const rightRecord = right as Record; + return ( + leftEntries.length === Object.keys(rightRecord).length && + leftEntries.every( + ([key, value]) => + Object.hasOwn(rightRecord, key) && styleValuesEqual(value, rightRecord[key]), + ) + ); + } + return false; +} + export class LoroMovableList extends LoroList { _valueHistoryComplete = true; _moveHistoryComplete = true; @@ -3561,6 +4034,7 @@ function textElementsToDelta( values = []; }; for (const element of elements) { + if (element.anchor !== undefined) continue; const nextAttributes = attributesAt(element); if (values.length > 0 && !textAttributesEqual(attributes, nextAttributes)) { flush(); @@ -3589,6 +4063,7 @@ function isTextPosType(value: string): value is TextPosType { } function utf8CodePointLength(value: string): number { + if (value.length === 0) return 0; const codePoint = value.codePointAt(0) ?? 0; if (codePoint <= 0x7f) return 1; if (codePoint <= 0x7ff) return 2; diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 247d4dd90..c30fe62e1 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -9,11 +9,7 @@ import { type DecodedOperationContent, } from "../codec/change-block-codec"; import type { ChangeLoroValue, ChangeValue } from "../codec/change-value"; -import { - containerTypeFromRawByte, - containerTypeToRawByte, - decodeContainerId, -} from "../codec/container-id"; +import { containerTypeFromRawByte, containerTypeToRawByte } from "../codec/container-id"; import { decodeDocument, decodeFastSnapshotBody, @@ -28,7 +24,6 @@ import { import { decodeChangeBlockKey, encodeChangeBlockKey } from "../codec/id"; import { decodeSstable, encodeSstable, type SstableEntry } from "../codec/sstable"; import { - decodeContainerStateWrapper, decodeLazyStateSnapshotStore, decodeStateSnapshotStore, encodeStateSnapshotStore, @@ -75,7 +70,7 @@ import { type SequenceMoveMeta, type SequenceValueMeta, type TextElement, - type TextStyleMeta, + type TextStyle, type TreeNodeRecord, } from "./containers"; import { SequenceEventDiff } from "./event-diff"; @@ -291,9 +286,6 @@ export class LoroDoc = Record>(); #movableMovePeers = new Map>(); #containersWithOperations = new Set(); - // Text containers with a mark operation in their history - // (#checkSnapshotSequences). - #markedTexts = new Set(); #containerKeys = new WeakMap(); #pendingHistory = new Map(); #deferredSnapshotHistory: DeferredSnapshotHistory | undefined; @@ -1176,7 +1168,13 @@ export class LoroDoc = Record